cluster/distribute: minor fixes in open file migration

* incorporated Avati's comments on the first patch.
* send proper stat information while unwinding

Change-Id: I36982cec610753c241c372272620ab2bd581fd9f
BUG: 3071
Reviewed-on: http://review.gluster.com/408
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

5 files changed, 91 insertions, 37 deletions

diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index fc251c90b..5c5825556 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -136,6 +136,8 @@ dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
 
         WIPE (&local->postparent);
 
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+
         DHT_STACK_UNWIND (lookup, frame, ret, local->op_errno, local->inode,
                           &local->stbuf, local->xattr, &local->postparent);
 
@@ -248,6 +250,7 @@ unlock:
                         dht_layout_set (this, local->inode, layout);
                 }
 
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
                                   local->inode, &local->stbuf, local->xattr,
                                   &local->postparent);
@@ -413,6 +416,7 @@ out:
 
                 WIPE (&local->postparent);
 
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
                                   local->inode, &local->stbuf, local->xattr,
                                   &local->postparent);
@@ -464,6 +468,7 @@ dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
 unwind:
         WIPE (&local->postparent);
 
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
         DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
                           local->inode, &local->stbuf, local->xattr,
                           &local->postparent);
@@ -536,6 +541,7 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
 
                 WIPE (&local->postparent);
 
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret,
                                   local->op_errno, local->inode,
                                   &local->stbuf, local->xattr,
@@ -565,6 +571,7 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
 
                 WIPE (&local->postparent);
 
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret,
                                   local->op_errno, local->inode,
                                   &local->stbuf, local->xattr,
@@ -830,6 +837,7 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
 unwind:
         WIPE (postparent);
 
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
                           postparent);
 
@@ -1009,6 +1017,7 @@ out:
 
         WIPE (postparent);
 
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
                           postparent);
 err:
@@ -2587,6 +2596,7 @@ out:
          * See dht_iatt_merge for reference.
          */
 
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, stbuf, preparent,
                           postparent);
         return 0;
@@ -2828,6 +2838,7 @@ dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         WIPE (postparent);
 
 out:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
                           postparent);
 
@@ -2856,6 +2867,7 @@ dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 
 err:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
                           postparent);
 
@@ -2966,6 +2978,7 @@ dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 out:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, stbuf, preparent,
                           postparent);
         return 0;
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 3545c0f99..97a2bd016 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -244,6 +244,22 @@ typedef struct dht_disk_layout dht_disk_layout_t;
                 ((st_mode_from_ia (s->ia_prot, s->ia_type) & ~S_IFMT)   \
                  == DHT_LINKFILE_MODE))
 
+#define IS_DHT_MIGRATION_PHASE2(buf)  (                                 \
+                IA_ISREG ((buf)->ia_type) &&                            \
+                ((st_mode_from_ia ((buf)->ia_prot, (buf)->ia_type) &    \
+                  ~S_IFMT) == DHT_LINKFILE_MODE))
+
+#define IS_DHT_MIGRATION_PHASE1(buf)  (                                 \
+                IA_ISREG ((buf)->ia_type) &&                            \
+                ((buf)->ia_prot.sticky == 1) &&                         \
+                ((buf)->ia_prot.sgid == 1))
+
+#define DHT_STRIP_PHASE1_FLAGS(buf)  do {               \
+                if (IS_DHT_MIGRATION_PHASE1(buf)) {     \
+                        (buf)->ia_prot.sticky = 0;      \
+                        (buf)->ia_prot.sgid = 0;        \
+                }                                       \
+        } while (0)
 
 #define check_is_dir(i,s,x) (IA_ISDIR(s->ia_type))
 
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index 1e9f54bda..d6a2cf369 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -165,9 +165,7 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
 
         /* Check if the rebalance phase2 is true */
-        if ((op_ret == -1) || (IA_ISREG (stbuf->ia_type) &&
-            ((st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type) &
-              ~S_IFMT) == DHT_LINKFILE_MODE))) {
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
                 if (local->fd)
                         ret = fd_ctx_get (local->fd, this, &tmp_subvol);
                 if (ret) {
@@ -178,12 +176,13 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
                         dht_attr2 (this, frame, 0);
-                        ret = 0;
                 }
                 if (!ret)
-                        goto err;
+                        return 0;
         }
+
 out:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf);
 err:
         return 0;
@@ -405,8 +404,7 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno != ENOENT))
                 goto out;
 
-        if ((op_ret == -1) || ((st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type) &
-                                ~S_IFMT) == DHT_LINKFILE_MODE)) {
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
                 /* File would be migrated to other node */
                 ret = fd_ctx_get (local->fd, this, NULL);
                 if (ret) {
@@ -422,6 +420,7 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 out:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
                           iobref);
 
@@ -708,34 +707,39 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 goto out;
         }
 
-        if (local->call_cnt != 1)
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
                 goto out;
+        }
 
         ret = fd_ctx_get (local->fd, this, NULL);
         if (ret) {
                 local->rebalance.target_op_fn = dht_fsync2;
 
                 /* Check if the rebalance phase1 is true */
-                if (IA_ISREG (postbuf->ia_type) &&
-                    (postbuf->ia_prot.sticky == 1) &&
-                    (postbuf->ia_prot.sgid == 1)) {
+                if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                        dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                        dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+
                         ret = dht_rebalance_in_progress_check (this, frame);
                 }
 
                 /* Check if the rebalance phase2 is true */
-                if (IA_ISREG (postbuf->ia_type) &&
-                    ((st_mode_from_ia (postbuf->ia_prot, postbuf->ia_type) &
-                      ~S_IFMT) == DHT_LINKFILE_MODE)) {
+                if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                         ret = dht_rebalance_complete_check (this, frame);
                 }
         } else {
                 dht_fsync2 (this, frame, 0);
-                ret = 0;
         }
         if (!ret)
                 return 0;
 
 out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
         DHT_STACK_UNWIND (fsync, frame, op_ret, op_errno,
                           prebuf, postbuf);
 
@@ -853,7 +857,7 @@ err:
 /* Symlinks are currently not migrated, so no need for any check here */
 int
 dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, const char *path, struct iatt *sbuf)
+                  int op_ret, int op_errno, const char *path, struct iatt *stbuf)
 {
         dht_local_t *local = NULL;
 
@@ -867,7 +871,8 @@ dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 err:
-        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, sbuf);
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf);
 
         return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index 21eca6117..f9dea75aa 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -48,23 +48,29 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        if (local->call_cnt != 1)
+        if (local->call_cnt != 1) {
+                /* preserve the modes of source */
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
                 goto out;
+        }
 
         local->rebalance.target_op_fn = dht_writev2;
 
         /* Phase 2 of migration */
-        if (IA_ISREG (postbuf->ia_type) &&
-            ((st_mode_from_ia (postbuf->ia_prot, postbuf->ia_type) &
-              ~S_IFMT) == DHT_LINKFILE_MODE)) {
+        if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
                 if (!ret)
                         return 0;
         }
 
         /* Check if the rebalance phase1 is true */
-        if (IA_ISREG (postbuf->ia_type) && (postbuf->ia_prot.sticky == 1) &&
-            (postbuf->ia_prot.sgid == 1)) {
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+
                 ret = fd_ctx_get (local->fd, this, NULL);
                 if (!ret) {
                         dht_writev2 (this, frame, 0);
@@ -76,6 +82,9 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+
         DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf);
 
         return 0;
@@ -185,34 +194,40 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        if (local->call_cnt != 1)
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
                 goto out;
+        }
 
         local->rebalance.target_op_fn = dht_truncate2;
 
         /* Phase 2 of migration */
-        if ((op_ret == -1) || (IA_ISREG (postbuf->ia_type) &&
-            ((st_mode_from_ia (postbuf->ia_prot, postbuf->ia_type) &
-              ~S_IFMT) == DHT_LINKFILE_MODE))) {
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
                 if (!ret)
-                        goto err;
+                        return 0;
         }
 
         /* Check if the rebalance phase1 is true */
-        if (IA_ISREG (postbuf->ia_type) && (postbuf->ia_prot.sticky == 1) &&
-            (postbuf->ia_prot.sgid == 1)) {
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
                 ret = fd_ctx_get (local->fd, this, NULL);
                 if (!ret) {
                         dht_truncate2 (this, frame, 0);
-                        goto err;
+                        return 0;
                 }
                 ret = dht_rebalance_in_progress_check (this, frame);
                 if (!ret)
-                        goto err;
+                        return 0;
         }
 
 out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
         DHT_STACK_UNWIND (truncate, frame, op_ret, op_errno,
                           prebuf, postbuf);
 err:
@@ -362,19 +377,19 @@ dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->rebalance.target_op_fn = dht_setattr2;
 
         /* Phase 2 of migration */
-        if ((op_ret == -1) || (IA_ISREG (postbuf->ia_type) &&
-            ((st_mode_from_ia (postbuf->ia_prot, postbuf->ia_type) &
-              ~S_IFMT) == DHT_LINKFILE_MODE))) {
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
                 if (!ret)
-                        goto out;
+                        return 0;
         }
 
         /* At the end of the migration process, whatever 'attr' we
            have on source file will be migrated to destination file
            in one shot, hence we don't need to check for in progress
-           state here */
+           state here (ie, PHASE1) */
 out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
         DHT_STACK_UNWIND (setattr, frame, op_ret, op_errno,
                           prebuf, postbuf);
 
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index 226ce280d..02b12887f 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -80,6 +80,7 @@ unwind:
                 WIPE (&local->preparent);
                 WIPE (&local->postparent);
 
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                                   &local->stbuf, &local->preoldparent,
                                   &local->postoldparent,
@@ -159,6 +160,7 @@ unwind:
         WIPE (&local->preparent);
         WIPE (&local->postparent);
 
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
         DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                           &local->stbuf, &local->preoldparent,
                           &local->postoldparent,
@@ -344,6 +346,7 @@ dht_rename_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         WIPE (&local->postparent);
 
         if (is_last_call (this_call_cnt)) {
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                                   &local->stbuf, &local->preoldparent,
                                   &local->postoldparent, &local->preparent,
@@ -415,6 +418,7 @@ nolinks:
         WIPE (&local->preparent);
         WIPE (&local->postparent);
 
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
         DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                           &local->stbuf, &local->preoldparent,
                           &local->postoldparent, &local->preparent,
@@ -573,6 +577,7 @@ unwind:
         WIPE (&local->preparent);
         WIPE (&local->postparent);
 
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
         DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                           &local->stbuf, &local->preoldparent,
                           &local->postoldparent, &local->preparent,