cluster/tier: Handle FOPs on files being migrated

Determine which DHT level is responsible for
handling fops on a file undergoing migration based
on the name of the the linkto xattr set on the file
 being migrated and process accordingly.

Change-Id: I82772e39314d4fe7f2ba0dcf22de0c6a374ee139
BUG: 1254428
Signed-off-by: N Balachandran <nbalacha@redhat.com>
Signed-off-by: Nithya Balachandran <nbalacha@redhat.com>
Reviewed-on: http://review.gluster.org/12090
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>

7 files changed, 596 insertions, 87 deletions

diff --git a/tests/basic/tier/fops-during-migration.t b/tests/basic/tier/fops-during-migration.t
new file mode 100755
index 00000000000..ce25f0acc32
--- /dev/null
+++ b/tests/basic/tier/fops-during-migration.t
@@ -0,0 +1,121 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+
+NUM_BRICKS=3
+DEMOTE_FREQ=5
+PROMOTE_FREQ=5
+
+TEST_STR="Testing write and truncate fops on tier migration"
+
+function is_sticky_set () {
+        echo $1
+        if [ -k $1 ];
+        then
+                echo "yes"
+        else
+                echo "no"
+        fi
+}
+
+
+# Creates a tiered volume with pure distribute hot and cold tiers
+# Both hot and cold tiers will have an equal number of bricks.
+
+function create_dist_tier_vol () {
+        mkdir $B0/cold
+        mkdir $B0/hot
+        TEST $CLI volume create $V0 $H0:$B0/cold/${V0}{0..$1}
+        TEST $CLI volume set $V0 performance.quick-read off
+        TEST $CLI volume set $V0 performance.io-cache off
+        TEST $CLI volume set $V0 features.ctr-enabled on
+        TEST $CLI volume start $V0
+        TEST $CLI volume attach-tier $V0 $H0:$B0/hot/${V0}{0..$1}
+        TEST $CLI volume set $V0 cluster.tier-demote-frequency $DEMOTE_FREQ
+        TEST $CLI volume set $V0 cluster.tier-promote-frequency $PROMOTE_FREQ
+        TEST $CLI volume set $V0 cluster.read-freq-threshold 0
+        TEST $CLI volume set $V0 cluster.write-freq-threshold 0
+}
+
+
+# Checks that the contents of the file matches the input string
+#$1 : file_path
+#$2 : comparison string
+
+function check_file_content () {
+        contents=`cat $1`
+        echo $contents
+        if [ "$contents" = "$2" ]; then
+                echo "1"
+        else
+                echo "0"
+        fi
+}
+
+
+cleanup;
+
+#Basic checks
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+
+
+#Create and start a tiered volume
+create_dist_tier_vol $NUM_BRICKS
+
+# Mount FUSE
+TEST glusterfs -s $H0 --volfile-id $V0 $M0
+
+TEST mkdir $M0/dir1
+
+# Create a large file (200MB), so that rebalance takes time
+# The file will be created on the hot tier
+
+dd if=/dev/zero of=$M0/dir1/FILE1 bs=64k count=5120
+
+# Get the path of the file on the hot tier
+HPATH=`find $B0/hot/ -name  FILE1`
+echo "File path on hot tier: "$HPATH
+
+
+# Wait for the tier process to demote the file
+EXPECT_WITHIN $REBALANCE_TIMEOUT "yes" is_sticky_set $HPATH
+
+# Get the path of the file on the cold tier
+CPATH=`find $B0/cold/ -name  FILE1`
+echo "File path on cold tier: "$CPATH
+
+# Test setxattr
+TEST setfattr -n "user.test_xattr" -v "qwerty" $M0/dir1/FILE1
+
+# Test hard link creation
+TEST ln $M0/dir1/FILE1 $M0/dir1/lnk1
+TEST ln $M0/dir1/FILE1 $M0/lnk2
+
+# Change the file contents while it is being migrated
+echo $TEST_STR > $M0/dir1/FILE1
+
+
+# The file contents should have changed even if the file
+# is not done migrating
+EXPECT "1" check_file_content $M0/dir1/FILE1 "$TEST_STR"
+
+
+# Wait for the tier process to finish migrating the file
+EXPECT_WITHIN $REBALANCE_TIMEOUT "no" is_sticky_set $CPATH
+
+# The file contents should have changed
+EXPECT "1" check_file_content $M0/dir1/FILE1 "$TEST_STR"
+
+
+linkcountsrc=$(stat -c %h $M0/dir1/FILE1)
+echo $linkcountsrc
+TEST [[ $linkcountsrc == 3 ]]
+
+TEST getfattr -n "user.test_xattr" $M0/dir1/FILE1
+
+cleanup;
+
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index ef1776beb30..e2c9f5547ac 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -26,13 +26,16 @@
 
 int run_defrag = 0;
 
-int dht_link2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame);
+int dht_link2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame,
+               int ret);
 
 int
-dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame);
+dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
+                  int ret);
 
 int
-dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame);
+dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
+               int ret);
 
 int
 dht_aggregate_quota_xattr (dict_t *dst, char *key, data_t *value)
@@ -3379,9 +3382,10 @@ dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        local->op_ret = 0;
-
+        local->op_ret = op_ret;
         local->rebalance.target_op_fn = dht_setxattr2;
+        if (xdata)
+                local->rebalance.xdata = dict_ref (xdata);
 
         /* Phase 2 of migration */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
@@ -3398,7 +3402,7 @@ dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                                   &subvol1, &subvol2);
                 if (!dht_mig_info_is_invalid (local->cached_subvol,
                                               subvol1, subvol2)) {
-                        dht_setxattr2 (this, subvol2, frame);
+                        dht_setxattr2 (this, subvol2, frame, 0);
                         return 0;
                 }
 
@@ -3408,8 +3412,6 @@ dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 out:
-        if (local->rebalance.xdata)
-                dict_unref (local->rebalance.xdata);
 
         if (local->fop == GF_FOP_SETXATTR) {
                 DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL);
@@ -3482,7 +3484,7 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
         } else {
 
                 local->call_cnt = 1;
-                local->rebalance.xdata = dict_ref (xattr);
+                local->rebalance.xattr = dict_ref (xattr);
                 local->rebalance.flags = flags;
 
                 xdata = xdata ? dict_ref (xdata) : dict_new ();
@@ -3565,27 +3567,42 @@ out:
 
 
 int
-dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t  *local  = NULL;
         int          op_errno = EINVAL;
 
-        if (!frame || !frame->local || !subvol)
+        if (!frame || !frame->local)
                 goto err;
 
         local = frame->local;
 
+        if (we_are_not_migrating (ret)) {
+                /* This dht xlator is not migrating the file. Unwind and
+                 * pass on the original mode bits so the higher DHT layer
+                 * can handle this.
+                 */
+                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
+                                  local->op_errno, local->rebalance.xdata);
+                return 0;
+        }
+
+        if (subvol == NULL)
+                goto err;
+
+        op_errno = local->op_errno;
+
         local->call_cnt = 2; /* This is the second attempt */
 
         if (local->fop == GF_FOP_SETXATTR) {
                 STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
                             subvol->fops->setxattr, &local->loc,
-                            local->rebalance.xdata, local->rebalance.flags,
+                            local->rebalance.xattr, local->rebalance.flags,
                             NULL);
         } else {
                 STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
                             subvol->fops->fsetxattr, local->fd,
-                            local->rebalance.xdata, local->rebalance.flags,
+                            local->rebalance.xattr, local->rebalance.flags,
                             NULL);
         }
 
@@ -3832,7 +3849,7 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
 
         } else {
 
-                local->rebalance.xdata = dict_ref (xattr);
+                local->rebalance.xattr = dict_ref (xattr);
                 local->rebalance.flags = flags;
                 local->call_cnt = 1;
 
@@ -3896,6 +3913,8 @@ dht_file_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->op_ret = 0;
 
         local->rebalance.target_op_fn = dht_removexattr2;
+        if (xdata)
+                local->rebalance.xdata = dict_ref (xdata);
 
         /* Phase 2 of migration */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
@@ -3912,7 +3931,7 @@ dht_file_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                                   &subvol1, &subvol2);
                 if (!dht_mig_info_is_invalid (local->cached_subvol,
                                               subvol1, subvol2)) {
-                        dht_removexattr2 (this, subvol2, frame);
+                        dht_removexattr2 (this, subvol2, frame, 0);
                         return 0;
                 }
 
@@ -3932,7 +3951,8 @@ out:
 }
 
 int
-dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
+                  int ret)
 {
         dht_local_t *local    = NULL;
         int          op_errno = EINVAL;
@@ -3944,6 +3964,17 @@ dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
 
         local->call_cnt = 2; /* This is the second attempt */
 
+        if (we_are_not_migrating (ret)) {
+
+                /* This dht xlator is not migrating the file. Unwind and
+                 * pass on the original mode bits so the higher DHT layer
+                 * can handle this.
+                 */
+                DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
+                                  local->op_errno, local->rebalance.xdata);
+                return 0;
+        }
+
         if (local->fop == GF_FOP_REMOVEXATTR) {
                 STACK_WIND (frame, dht_file_removexattr_cbk, subvol,
                             subvol->fops->removexattr, &local->loc,
@@ -5602,8 +5633,12 @@ dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->inode = inode_ref (inode);
         }
 
+        local->op_ret = op_ret;
         local->op_errno = op_errno;
         local->rebalance.target_op_fn = dht_link2;
+        dht_set_local_rebalance (this, local, stbuf, preparent,
+                                 postparent, xdata);
+
         /* Check if the rebalance phase2 is true */
         if (IS_DHT_MIGRATION_PHASE2 (stbuf)) {
                 ret = dht_inode_ctx_get_mig_info (this, local->loc.inode, NULL,
@@ -5614,7 +5649,7 @@ dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         if (!ret)
                                 return 0;
                 } else {
-                        dht_link2 (this, subvol, frame);
+                        dht_link2 (this, subvol, frame, 0);
                         return 0;
                 }
         }
@@ -5624,7 +5659,7 @@ dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 ret = dht_inode_ctx_get_mig_info (this, local->loc.inode, NULL,
                                                   &subvol);
                 if (subvol) {
-                        dht_link2 (this, subvol, frame);
+                        dht_link2 (this, subvol, frame, 0);
                         return 0;
                 }
                 ret = dht_rebalance_in_progress_check (this, frame);
@@ -5642,7 +5677,7 @@ out:
 
 
 int
-dht_link2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_link2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local  = NULL;
         int          op_errno = EINVAL;
@@ -5652,6 +5687,19 @@ dht_link2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
                 goto err;
 
         op_errno = local->op_errno;
+
+        if (we_are_not_migrating (ret)) {
+                /* This dht xlator is not migrating the file. Unwind and
+                 * pass on the original mode bits so the higher DHT layer
+                 * can handle this.
+                 */
+                DHT_STACK_UNWIND (link, frame, local->op_ret, op_errno,
+                                  local->inode,
+                                  &local->stbuf, &local->preparent,
+                                  &local->postparent, NULL);
+                return 0;
+        }
+
         if (subvol == NULL) {
                 op_errno = EINVAL;
                 goto err;
@@ -7855,3 +7903,43 @@ int32_t dht_migration_needed(xlator_t *this)
 out:
         return ret;
 }
+
+
+
+/*
+This function should not be called more then once during a FOP
+handling path. It is valid only for for ops on files
+*/
+int32_t dht_set_local_rebalance (xlator_t *this, dht_local_t *local,
+                                 struct iatt *stbuf,
+                                 struct iatt *prebuf, struct iatt *postbuf,
+                                 dict_t *xdata)
+{
+
+        if (!local)
+                return -1;
+
+        if (local->rebalance.set) {
+                gf_msg (this->name, GF_LOG_WARNING, 0,
+                        DHT_MSG_REBAL_STRUCT_SET,
+                        "local->rebalance already set");
+        }
+
+
+        if (stbuf)
+                memcpy (&local->rebalance.stbuf, stbuf, sizeof (struct iatt));
+
+        if (prebuf)
+                memcpy (&local->rebalance.prebuf, prebuf, sizeof (struct iatt));
+
+        if (postbuf)
+                memcpy (&local->rebalance.postbuf, postbuf,
+                        sizeof (struct iatt));
+
+        if (xdata)
+                local->rebalance.xdata = dict_ref (xdata);
+
+        local->rebalance.set = 1;
+
+        return 0;
+}
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 6bf8ba1c406..c48bf5800b9 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -37,7 +37,7 @@ typedef int (*dht_selfheal_dir_cbk_t) (call_frame_t *frame, void *cookie,
                                        int32_t       op_ret, int32_t op_errno,
                                        dict_t       *xdata);
 typedef int (*dht_defrag_cbk_fn_t) (xlator_t        *this, xlator_t *dst_node,
-                                    call_frame_t    *frame);
+                                    call_frame_t    *frame, int ret);
 
 typedef int (*dht_refresh_layout_unlock) (call_frame_t *frame, xlator_t *this,
                                          int op_ret);
@@ -116,8 +116,12 @@ struct dht_rebalance_ {
         struct iobref       *iobref;
         struct iovec        *vector;
         struct iatt          stbuf;
+        struct iatt          prebuf;
+        struct iatt          postbuf;
         dht_defrag_cbk_fn_t  target_op_fn;
         dict_t              *xdata;
+        dict_t              *xattr;
+        int32_t              set;
 };
 
 /**
@@ -548,6 +552,8 @@ typedef struct dht_migrate_info {
 
 #define layout_is_sane(layout) ((layout) && (layout->cnt > 0))
 
+#define we_are_not_migrating(x)   ((x) == 1)
+
 #define DHT_STACK_UNWIND(fop, frame, params ...) do {           \
                 dht_local_t *__local = NULL;                    \
                 xlator_t    *__xl    = NULL;                    \
@@ -1078,4 +1084,9 @@ int
 dht_build_parent_loc (xlator_t *this, loc_t *parent, loc_t *child,
                                                  int32_t *op_errno);
 
+int32_t dht_set_local_rebalance (xlator_t *this, dht_local_t *local,
+                                 struct iatt *stbuf,
+                                 struct iatt *prebuf,
+                                 struct iatt *postbuf, dict_t *xdata);
+
 #endif/* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index 88bf7a605da..3a4217029b9 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -132,17 +132,6 @@ dht_frame_return (call_frame_t *frame)
         return this_call_cnt;
 }
 
-/*
- * A slightly "updated" version of the algorithm described in the commit log
- * is used here.
- *
- * The only enhancement is that:
- *
- * - The number of bits used by the backend filesystem for HUGE d_off which
- *   is described as 63, and
- * - The number of bits used by the d_off presented by the transformation
- *   upwards which is described as 64, are both made "configurable."
- */
 
 int
 dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
@@ -455,6 +444,12 @@ dht_local_wipe (xlator_t *this, dht_local_t *local)
 
         GF_FREE (local->key);
 
+        if (local->rebalance.xdata)
+                dict_unref (local->rebalance.xdata);
+
+        if (local->rebalance.xattr)
+                dict_unref (local->rebalance.xattr);
+
         GF_FREE (local->rebalance.vector);
 
         if (local->rebalance.iobref)
@@ -886,7 +881,12 @@ dht_init_subvolumes (xlator_t *this, dht_conf_t *conf)
 }
 
 
-
+/*
+ op_ret values :
+  0 : Success.
+ -1 : Failure.
+  1 : File is being migrated but not by this DHT layer.
+*/
 
 static int
 dht_migration_complete_check_done (int op_ret, call_frame_t *frame, void *data)
@@ -896,7 +896,7 @@ dht_migration_complete_check_done (int op_ret, call_frame_t *frame, void *data)
 
         local = frame->local;
 
-        if (op_ret == -1)
+        if (op_ret != 0)
                 goto out;
 
         if (local->cached_subvol == NULL) {
@@ -907,7 +907,7 @@ dht_migration_complete_check_done (int op_ret, call_frame_t *frame, void *data)
         subvol = local->cached_subvol;
 
 out:
-        local->rebalance.target_op_fn (THIS, subvol, frame);
+        local->rebalance.target_op_fn (THIS, subvol, frame, op_ret);
 
         return 0;
 }
@@ -960,15 +960,35 @@ dht_migration_complete_check_task (void *data)
                 SYNCTASK_SETID (frame->root->uid, frame->root->gid);
         }
 
+
         /*
-         * temporary check related to tier promoting/demoting the file;
-         * the lower level DHT detects the migration (due to sticky
-         * bits) when it is the responsibility of the tier translator
-         * to complete the rebalance transaction. It will be corrected
-         * when rebalance and tier migration are fixed to work together.
+         * Each DHT xlator layer has its own name for the linkto xattr.
+         * If the file mode bits indicate the the file is being migrated but
+         * this layer's linkto xattr is not set, it means that another
+         * DHT layer is migrating the file. In this case, return 1 so
+         * the mode bits can be passed on to the higher layer for appropriate
+         * action.
          */
-        if (strcmp(this->parents->xlator->type, "cluster/tier") == 0) {
-                ret = 0;
+        if (-ret == ENODATA) {
+                /* This DHT translator is not migrating this file */
+
+                ret = inode_ctx_reset1 (inode, this, &tmp_miginfo);
+                if (tmp_miginfo) {
+
+                        /* This can be a problem if the file was
+                         * migrated by two different layers. Raise
+                         * a warning here.
+                         */
+                        gf_msg (this->name, GF_LOG_WARNING, 0,
+                                DHT_MSG_HAS_MIGINFO,
+                                "%s: Found miginfo in the inode ctx",
+                                tmp_loc.path ? tmp_loc.path :
+                                uuid_utoa (tmp_loc.gfid));
+
+                        miginfo = (void *)tmp_miginfo;
+                        GF_REF_PUT (miginfo);
+                }
+                ret = 1;
                 goto out;
         }
 
@@ -1094,7 +1114,14 @@ dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame)
         return ret;
 
 }
+
 /* During 'in-progress' state, both nodes should have the file */
+/*
+ op_ret values :
+  0 : Success
+ -1 : Failure.
+  1 : File is being migrated but not by this DHT layer.
+*/
 static int
 dht_inprogress_check_done (int op_ret, call_frame_t *frame, void *data)
 {
@@ -1104,7 +1131,7 @@ dht_inprogress_check_done (int op_ret, call_frame_t *frame, void *data)
 
         local = frame->local;
 
-        if (op_ret == -1)
+        if (op_ret != 0)
                 goto out;
 
         inode = local->loc.inode ? local->loc.inode : local->fd->inode;
@@ -1120,7 +1147,7 @@ dht_inprogress_check_done (int op_ret, call_frame_t *frame, void *data)
         }
 
 out:
-        local->rebalance.target_op_fn (THIS, dst_subvol, frame);
+        local->rebalance.target_op_fn (THIS, dst_subvol, frame, op_ret);
 
         return 0;
 }
@@ -1142,6 +1169,9 @@ dht_rebalance_inprogress_task (void *data)
         inode_t      *inode    = NULL;
         fd_t         *iter_fd  = NULL;
         int           open_failed = 0;
+        uint64_t      tmp_miginfo  = 0;
+        dht_migrate_info_t *miginfo     = NULL;
+
 
         this  = THIS;
         frame = data;
@@ -1167,6 +1197,35 @@ dht_rebalance_inprogress_task (void *data)
                                         conf->link_xattr_name, NULL, NULL);
         }
 
+        /*
+         * Each DHT xlator layer has its own name for the linkto xattr.
+         * If the file mode bits indicate the the file is being migrated but
+         * this layer's linkto xattr is not present, it means that another
+         * DHT layer is migrating the file. In this case, return 1 so
+         * the mode bits can be passed on to the higher layer for appropriate
+         * action.
+         */
+
+        if (-ret == ENODATA) {
+                /* This DHT layer is not migrating this file */
+                ret = inode_ctx_reset1 (inode, this, &tmp_miginfo);
+                if (tmp_miginfo) {
+                        /* This can be a problem if the file was
+                         * migrated by two different layers. Raise
+                         * a warning here.
+                         */
+                        gf_msg (this->name, GF_LOG_WARNING, 0,
+                                DHT_MSG_HAS_MIGINFO,
+                                "%s: Found miginfo in the inode ctx",
+                                tmp_loc.path ? tmp_loc.path :
+                                uuid_utoa (tmp_loc.gfid));
+                        miginfo = (void *)tmp_miginfo;
+                        GF_REF_PUT (miginfo);
+                }
+                ret = 1;
+                goto out;
+        }
+
         if (ret < 0) {
                 gf_msg (this->name, GF_LOG_ERROR, -ret,
                         DHT_MSG_GET_XATTR_FAILED,
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index a5eda4e1270..02265fd2e85 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -10,13 +10,22 @@
 
 #include "dht-common.h"
 
-int dht_access2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame);
-int dht_readv2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame);
-int dht_attr2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame);
-int dht_open2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame);
-int dht_flush2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame);
-int dht_lk2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame);
-int dht_fsync2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame);
+int dht_access2 (xlator_t *this, xlator_t *dst_node,
+                 call_frame_t *frame, int ret);
+int dht_readv2 (xlator_t *this, xlator_t *dst_node,
+                call_frame_t *frame, int ret);
+int dht_attr2 (xlator_t *this, xlator_t *dst_node,
+               call_frame_t *frame, int ret);
+int dht_open2 (xlator_t *this, xlator_t *dst_node,
+               call_frame_t *frame, int ret);
+int dht_flush2 (xlator_t *this, xlator_t *dst_node,
+                call_frame_t *frame, int ret);
+int dht_lk2 (xlator_t *this, xlator_t *dst_node,
+             call_frame_t *frame, int ret);
+int dht_fsync2 (xlator_t *this, xlator_t *dst_node,
+                call_frame_t *frame, int ret);
+
+
 
 int
 dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
@@ -53,7 +62,7 @@ out:
 }
 
 int
-dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local    = NULL;
         int          op_errno = EINVAL;
@@ -64,6 +73,14 @@ dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
         local = frame->local;
         op_errno = ENOENT;
 
+        if (we_are_not_migrating (ret)) {
+                /* This DHT layer is not migrating the file */
+                DHT_STACK_UNWIND (open, frame, -1, local->op_errno,
+                                  NULL, NULL);
+                return 0;
+
+        }
+
         if (subvol == NULL)
                 goto out;
 
@@ -75,10 +92,11 @@ dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
         return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
+
 int
 dht_open (call_frame_t *frame, xlator_t *this,
           loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
@@ -150,6 +168,8 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
 
         local->op_errno = op_errno;
+        local->op_ret = op_ret;
+
         /* Check if the rebalance phase2 is true */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
                 inode = (local->fd) ? local->fd->inode : local->loc.inode;
@@ -157,13 +177,15 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (!subvol) {
                         /* Phase 2 of migration */
                         local->rebalance.target_op_fn = dht_attr2;
+                        dht_set_local_rebalance (this, local, NULL, NULL,
+                                                 stbuf, xdata);
                         ret = dht_rebalance_complete_check (this, frame);
                         if (!ret)
                                 return 0;
                 } else {
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
-                        dht_attr2 (this, subvol, frame);
+                        dht_attr2 (this, subvol, frame, 0);
                         return 0;
                 }
         }
@@ -176,7 +198,7 @@ err:
 }
 
 int
-dht_attr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_attr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local    = NULL;
         int          op_errno = EINVAL;
@@ -186,6 +208,19 @@ dht_attr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
                 goto out;
 
         op_errno = local->op_errno;
+
+        if (we_are_not_migrating (ret)) {
+                /* This dht xlator is not migrating the file. Unwind and
+                 * pass on the original mode bits so the higher DHT layer
+                 * can handle this.
+                 */
+                DHT_STACK_UNWIND (stat, frame, local->op_ret, op_errno,
+                                  &local->rebalance.postbuf,
+                                  local->rebalance.xdata);
+                return 0;
+        }
+
+
         if (subvol == NULL)
                 goto out;
 
@@ -377,7 +412,8 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         dht_local_t *local      = NULL;
         int          ret        = 0;
-        xlator_t    *subvol = 0;
+        xlator_t    *src_subvol = 0;
+        xlator_t    *dst_subvol = 0;
 
         local = frame->local;
         if (!local) {
@@ -396,23 +432,31 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->op_errno = op_errno;
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
                 /* File would be migrated to other node */
-                ret = dht_inode_ctx_get_mig_info (this, local->fd->inode, NULL,
-                                                  &subvol);
-                if (!subvol) {
+                ret = dht_inode_ctx_get_mig_info (this, local->fd->inode,
+                                                  &src_subvol,
+                                                  &dst_subvol);
+
+                if (dht_mig_info_is_invalid (local->cached_subvol,
+                                             src_subvol, dst_subvol)) {
+                        local->op_ret = op_ret;
                         local->rebalance.target_op_fn = dht_readv2;
+                        dht_set_local_rebalance (this, local, NULL, NULL,
+                                                 stbuf, xdata);
+
                         ret = dht_rebalance_complete_check (this, frame);
                         if (!ret)
                                 return 0;
                 } else {
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
-                        dht_readv2 (this, subvol, frame);
+                        dht_readv2 (this, dst_subvol, frame, 0);
                         return 0;
                 }
         }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
+
         DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
                           iobref, xdata);
 
@@ -420,7 +464,7 @@ out:
 }
 
 int
-dht_readv2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_readv2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local  = NULL;
         int          op_errno = EINVAL;
@@ -430,6 +474,18 @@ dht_readv2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
                 goto out;
 
         op_errno = local->op_errno;
+
+        if (we_are_not_migrating (ret)) {
+                /* This dht xlator is not migrating the file. Unwind and
+                 * pass on the original mode bits so the higher DHT layer
+                 * can handle this.
+                 */
+                DHT_STACK_UNWIND (readv, frame, local->op_ret, op_errno,
+                                  NULL, 0, &local->rebalance.postbuf,
+                                  NULL, local->rebalance.xdata);
+                return 0;
+        }
+
         if (subvol == NULL)
                 goto out;
 
@@ -446,6 +502,7 @@ out:
         return 0;
 }
 
+
 int
 dht_readv (call_frame_t *frame, xlator_t *this,
            fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
@@ -538,7 +595,7 @@ out:
 }
 
 int
-dht_access2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_access2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local    = NULL;
         int          op_errno = EINVAL;
@@ -548,6 +605,17 @@ dht_access2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
                 goto out;
 
         op_errno = local->op_errno;
+
+        if (we_are_not_migrating (ret)) {
+                /* This dht xlator is not migrating the file. Unwind and
+                 * pass on the original mode bits so the higher DHT layer
+                 * can handle this.
+                 */
+
+                DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+                return 0;
+        }
+
         if (subvol == NULL)
                 goto out;
 
@@ -624,7 +692,7 @@ dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         /* If context is set, then send flush() it to the destination */
         dht_inode_ctx_get_mig_info (this, local->fd->inode, NULL, &subvol);
         if (subvol) {
-                dht_flush2 (this, subvol, frame);
+                dht_flush2 (this, subvol, frame, 0);
                 return 0;
         }
 
@@ -635,7 +703,7 @@ out:
 }
 
 int
-dht_flush2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_flush2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local    = NULL;
         int32_t      op_errno = EINVAL;
@@ -647,7 +715,6 @@ dht_flush2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
 
         op_errno = local->op_errno;
 
-        dht_inode_ctx_get_mig_info (this, local->fd->inode, NULL, &subvol);
         if (subvol == NULL)
                 goto out;
 
@@ -713,7 +780,8 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
         call_frame_t *prev = NULL;
         int           ret = -1;
         inode_t      *inode = NULL;
-        xlator_t     *subvol = 0;
+        xlator_t     *src_subvol = 0;
+        xlator_t     *dst_subvol = 0;
 
         local = frame->local;
         prev = cookie;
@@ -736,9 +804,15 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
 
         local->op_errno = op_errno;
         inode = local->fd->inode;
-        dht_inode_ctx_get_mig_info (this, inode, NULL, &subvol);
-        if (!subvol) {
+
+        dht_inode_ctx_get_mig_info (this, inode, &src_subvol, &dst_subvol);
+
+        if (dht_mig_info_is_invalid (local->cached_subvol,
+                                              src_subvol, dst_subvol)) {
+
                 local->rebalance.target_op_fn = dht_fsync2;
+                dht_set_local_rebalance (this, local, NULL, prebuf,
+                                         postbuf, xdata);
 
                 /* Check if the rebalance phase1 is true */
                 if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
@@ -755,13 +829,14 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 if (!ret)
                         return 0;
         } else {
-                dht_fsync2 (this, subvol, frame);
+                dht_fsync2 (this, dst_subvol, frame, 0);
                 return 0;
         }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
+
         DHT_STACK_UNWIND (fsync, frame, op_ret, op_errno,
                           prebuf, postbuf, xdata);
 
@@ -769,7 +844,7 @@ out:
 }
 
 int
-dht_fsync2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_fsync2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local    = NULL;
         int32_t      op_errno = EINVAL;
@@ -780,6 +855,18 @@ dht_fsync2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
         local = frame->local;
         op_errno = local->op_errno;
 
+        if (we_are_not_migrating (ret)) {
+                /* This dht xlator is not migrating the file. Unwind and
+                 * pass on the original mode bits so the higher DHT layer
+                 * can handle this.
+                 */
+                DHT_STACK_UNWIND (fsync, frame, local->op_ret,
+                                  op_errno, &local->rebalance.prebuf,
+                                  &local->rebalance.postbuf,
+                                  local->rebalance.xdata);
+                return 0;
+        }
+
         if (subvol == NULL)
                 goto out;
 
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index ce5006a80df..a1bf2aa5e8d 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -11,12 +11,18 @@
 
 #include "dht-common.h"
 
-int dht_writev2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame);
-int dht_truncate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame);
-int dht_setattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame);
-int dht_fallocate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame);
-int dht_discard2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame);
-int dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame);
+int dht_writev2 (xlator_t *this, xlator_t *subvol,
+                 call_frame_t *frame, int ret);
+int dht_truncate2 (xlator_t *this, xlator_t *subvol,
+                   call_frame_t *frame, int ret);
+int dht_setattr2 (xlator_t *this, xlator_t *subvol,
+                  call_frame_t *frame, int ret);
+int dht_fallocate2 (xlator_t *this, xlator_t *subvol,
+                    call_frame_t *frame, int ret);
+int dht_discard2 (xlator_t *this, xlator_t *subvol,
+                  call_frame_t *frame, int ret);
+int dht_zerofill2 (xlator_t *this, xlator_t *subvol,
+                   call_frame_t *frame, int ret);
 
 int
 dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
@@ -58,7 +64,15 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local->rebalance.target_op_fn = dht_writev2;
 
+        local->op_ret = op_ret;
         local->op_errno = op_errno;
+
+        /* We might need to pass the stbuf information to the higher DHT
+         * layer for appropriate handling.
+         */
+
+        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
+
         /* Phase 2 of migration */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
@@ -75,7 +89,7 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                                   &subvol1, &subvol2);
                 if (!dht_mig_info_is_invalid (local->cached_subvol,
                                               subvol1, subvol2)) {
-                        dht_writev2 (this, subvol2, frame);
+                        dht_writev2 (this, subvol2, frame, 0);
                         return 0;
                 }
                 ret = dht_rebalance_in_progress_check (this, frame);
@@ -94,7 +108,7 @@ out:
 }
 
 int
-dht_writev2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_writev2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local    = NULL;
         int32_t      op_errno = EINVAL;
@@ -105,6 +119,19 @@ dht_writev2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
         local = frame->local;
         op_errno = local->op_errno;
 
+        if (we_are_not_migrating (ret)) {
+                /* This dht xlator is not migrating the file. Unwind and
+                 * pass on the original mode bits so the higher DHT layer
+                 * can handle this.
+                 */
+                DHT_STACK_UNWIND (writev, frame, local->op_ret,
+                                  local->op_errno, &local->rebalance.prebuf,
+                                  &local->rebalance.postbuf,
+                                  local->rebalance.xdata);
+                return 0;
+        }
+
+
         if (subvol == NULL)
                 goto out;
 
@@ -215,7 +242,15 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local->rebalance.target_op_fn = dht_truncate2;
 
+        local->op_ret = op_ret;
         local->op_errno = op_errno;
+
+        /* We might need to pass the stbuf information to the higher DHT
+         * layer for appropriate handling.
+         */
+
+        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
+
         /* Phase 2 of migration */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
@@ -233,7 +268,7 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                             &dst_subvol);
                 if (!dht_mig_info_is_invalid (local->cached_subvol,
                                               src_subvol, dst_subvol)) {
-                        dht_truncate2 (this, dst_subvol, frame);
+                        dht_truncate2 (this, dst_subvol, frame, 0);
                         return 0;
                 }
                 ret = dht_rebalance_in_progress_check (this, frame);
@@ -244,6 +279,7 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
+
         DHT_STACK_UNWIND (truncate, frame, op_ret, op_errno,
                           prebuf, postbuf, xdata);
 err:
@@ -252,7 +288,7 @@ err:
 
 
 int
-dht_truncate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_truncate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local    = NULL;
         int32_t      op_errno = EINVAL;
@@ -263,6 +299,16 @@ dht_truncate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
         local = frame->local;
         op_errno = local->op_errno;
 
+        /* This dht xlator is not migrating the file  */
+        if (we_are_not_migrating (ret)) {
+
+                DHT_STACK_UNWIND (truncate, frame, local->op_ret,
+                                  local->op_errno, &local->rebalance.prebuf,
+                                  &local->rebalance.postbuf,
+                                  local->rebalance.xdata);
+                return 0;
+        }
+
         if (subvol == NULL)
                 goto out;
 
@@ -406,8 +452,13 @@ dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 }
                 goto out;
         }
+
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
         local->rebalance.target_op_fn = dht_fallocate2;
 
+        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
+
         /* Phase 2 of migration */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
@@ -424,7 +475,7 @@ dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                                             &dst_subvol);
                 if (!dht_mig_info_is_invalid (local->cached_subvol,
                                               src_subvol, dst_subvol)) {
-                        dht_fallocate2 (this, dst_subvol, frame);
+                        dht_fallocate2 (this, dst_subvol, frame, 0);
                         return 0;
                 }
                 ret = dht_rebalance_in_progress_check (this, frame);
@@ -435,6 +486,7 @@ dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
+
         DHT_STACK_UNWIND (fallocate, frame, op_ret, op_errno,
                           prebuf, postbuf, xdata);
 err:
@@ -442,7 +494,7 @@ err:
 }
 
 int
-dht_fallocate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_fallocate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local    = NULL;
         int32_t      op_errno = EINVAL;
@@ -453,6 +505,19 @@ dht_fallocate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
         local = frame->local;
         op_errno = local->op_errno;
 
+        if (we_are_not_migrating (ret)) {
+                /* This dht xlator is not migrating the file. Unwind and
+                 * pass on the original mode bits so the higher DHT layer
+                 * can handle this.
+                 */
+                DHT_STACK_UNWIND (fallocate, frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->rebalance.prebuf,
+                                  &local->rebalance.postbuf,
+                                  local->rebalance.xdata);
+                return 0;
+        }
+
         if (subvol == NULL)
                 goto out;
 
@@ -550,7 +615,12 @@ dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 }
                 goto out;
         }
+
         local->rebalance.target_op_fn = dht_discard2;
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+
+        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
 
         /* Phase 2 of migration */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
@@ -568,7 +638,7 @@ dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                                             &dst_subvol);
                 if (!dht_mig_info_is_invalid(local->cached_subvol,
                                              src_subvol, dst_subvol)) {
-                        dht_discard2 (this, dst_subvol, frame);
+                        dht_discard2 (this, dst_subvol, frame, 0);
                         return 0;
                 }
                 ret = dht_rebalance_in_progress_check (this, frame);
@@ -579,6 +649,7 @@ dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
+
         DHT_STACK_UNWIND (discard, frame, op_ret, op_errno,
                           prebuf, postbuf, xdata);
 err:
@@ -586,7 +657,7 @@ err:
 }
 
 int
-dht_discard2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_discard2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local    = NULL;
         int32_t      op_errno = EINVAL;
@@ -597,6 +668,19 @@ dht_discard2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
         local = frame->local;
         op_errno = local->op_errno;
 
+        if (we_are_not_migrating (ret)) {
+                /* This dht xlator is not migrating the file. Unwind and
+                 * pass on the original mode bits so the higher DHT layer
+                 * can handle this.
+                 */
+                DHT_STACK_UNWIND (discard, frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->rebalance.prebuf,
+                                  &local->rebalance.postbuf,
+                                  local->rebalance.xdata);
+                return 0;
+        }
+
         if (subvol == NULL)
                 goto out;
 
@@ -689,7 +773,13 @@ dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 }
                 goto out;
         }
+
         local->rebalance.target_op_fn = dht_zerofill2;
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+
+        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
+
         /* Phase 2 of migration */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
@@ -706,7 +796,7 @@ dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                                                   &subvol1, &subvol2);
                 if (!dht_mig_info_is_invalid (local->cached_subvol,
                                               subvol1, subvol2)) {
-                        dht_zerofill2 (this, subvol2, frame);
+                        dht_zerofill2 (this, subvol2, frame, 0);
                         return 0;
                 }
 
@@ -718,6 +808,7 @@ dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
+
         DHT_STACK_UNWIND (zerofill, frame, op_ret, op_errno,
                           prebuf, postbuf, xdata);
 err:
@@ -725,7 +816,7 @@ err:
 }
 
 int
-dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local    = NULL;
         int32_t      op_errno = EINVAL;
@@ -737,6 +828,20 @@ dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
 
         op_errno = local->op_errno;
 
+        if (we_are_not_migrating (ret)) {
+                /* This dht xlator is not migrating the file. Unwind and
+                 * pass on the original mode bits so the higher DHT layer
+                 * can handle this.
+                 */
+                DHT_STACK_UNWIND (zerofill, frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->rebalance.prebuf,
+                                  &local->rebalance.postbuf,
+                                  local->rebalance.xdata);
+
+                return 0;
+        }
+
         if (subvol == NULL)
                 goto out;
 
@@ -749,6 +854,7 @@ dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
         return 0;
 
 out:
+
         DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
@@ -821,10 +927,18 @@ dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (local->call_cnt != 1)
                 goto out;
 
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+
         local->rebalance.target_op_fn = dht_setattr2;
 
+
         /* Phase 2 of migration */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+
+                dht_set_local_rebalance (this, local, NULL, prebuf,
+                                         postbuf, xdata);
+
                 ret = dht_rebalance_complete_check (this, frame);
                 if (!ret)
                         return 0;
@@ -837,6 +951,7 @@ dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
+
         DHT_STACK_UNWIND (setattr, frame, op_ret, op_errno,
                           prebuf, postbuf, xdata);
 
@@ -844,7 +959,7 @@ out:
 }
 
 int
-dht_setattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+dht_setattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
         dht_local_t *local    = NULL;
         int32_t      op_errno = EINVAL;
@@ -855,6 +970,19 @@ dht_setattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
         local = frame->local;
         op_errno = local->op_errno;
 
+        if (we_are_not_migrating (ret)) {
+                /* This dht xlator is not migrating the file. Unwind and
+                 * pass on the original mode bits so the higher DHT layer
+                 * can handle this.
+                 */
+                DHT_STACK_UNWIND (setattr, frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->rebalance.prebuf,
+                                  &local->rebalance.postbuf,
+                                  local->rebalance.xdata);
+                return 0;
+        }
+
         if (subvol == NULL)
                 goto out;
 
diff --git a/xlators/cluster/dht/src/dht-messages.h b/xlators/cluster/dht/src/dht-messages.h
index 60030832d9f..80b35557408 100644
--- a/xlators/cluster/dht/src/dht-messages.h
+++ b/xlators/cluster/dht/src/dht-messages.h
@@ -40,7 +40,7 @@
  */
 
 #define GLFS_DHT_BASE                   GLFS_MSGID_COMP_DHT
-#define GLFS_DHT_NUM_MESSAGES           104
+#define GLFS_DHT_NUM_MESSAGES           106
 #define GLFS_MSGID_END          (GLFS_DHT_BASE + GLFS_DHT_NUM_MESSAGES + 1)
 
 /* Messages with message IDs */
@@ -978,5 +978,20 @@
 
 #define DHT_MSG_COMMIT_HASH_INFO       (GLFS_DHT_BASE + 104)
 
+/*
+ * @messageid 109105
+ * @diagnosis
+ * @recommendedaction None
+ */
+
+#define DHT_MSG_REBAL_STRUCT_SET       (GLFS_DHT_BASE + 105)
+
+/*
+ * @messageid 109106
+ * @diagnosis
+ * @recommendedaction None
+ */
+
+#define DHT_MSG_HAS_MIGINFO             (GLFS_DHT_BASE + 106)
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 #endif /* _DHT_MESSAGES_H_ */