cluster/dht: Rename should not fail post hardlink creation

In the rename path, we wind the creation of newname hardlink and
linkto file in dst hashed a the same time. If the linkto creation
fails, but the link creation succeeds, we enter the failure code
and cleanup the created newname hardlink.

In the interim if another client looks up newname and finds it as
a hardlink from FUSE, it could send an unlink for oldname instead
of a rename. This combined with the above cleanup code could end
up losing all the files copies, and thereby losing data.

This fix separates these steps into 2 parts, creating the linkto
first and then the link file, so that post link file creation no
failures would cleanup the newname file. If linkto fails then link
is not attempted, thereby not polluting the name space with
newname.

Change-Id: I61da8e906060da16a31ea1076eec2f01fd617f44
BUG: 1130888
Signed-off-by: Shyam <srangana@redhat.com>
Reviewed-on: http://review.gluster.org/8570
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

2 files changed, 99 insertions, 41 deletions

diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 62fef87a51c..d1671dcd879 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -544,11 +544,11 @@ int dht_filter_loc_subvol_key (xlator_t  *this, loc_t *loc, loc_t *new_loc,
                                xlator_t **subvol);
 
 int                                     dht_rename_cleanup (call_frame_t *frame);
-int dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t           op_ret, int32_t op_errno,
-                          inode_t          *inode, struct iatt *stbuf,
-                          struct iatt      *preparent, struct iatt *postparent,
-                          dict_t *xdata);
+int dht_rename_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t           op_ret, int32_t op_errno,
+                         inode_t          *inode, struct iatt *stbuf,
+                         struct iatt      *preparent, struct iatt *postparent,
+                         dict_t *xdata);
 
 int dht_fix_directory_layout (call_frame_t *frame,
                               dht_selfheal_dir_cbk_t  dir_cbk,
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index 664864919b1..6587aa89984 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -848,9 +848,8 @@ dht_do_rename (call_frame_t *frame)
         return 0;
 }
 
-
 int
-dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+dht_rename_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
                       inode_t *inode, struct iatt *stbuf,
                       struct iatt *preparent, struct iatt *postparent,
@@ -858,8 +857,6 @@ dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
-        int           this_call_cnt  = 0;
-
 
         local = frame->local;
         prev = cookie;
@@ -869,34 +866,85 @@ dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                               "link/file on %s failed (%s)",
                               prev->this->name, strerror (op_errno));
                 local->op_ret   = -1;
-                if (op_errno != ENOENT) {
-                        local->op_errno = op_errno;
-                        if (prev->this == local->src_cached) {
-                                local->added_link = _gf_false;
-                        }
-                }
-        } else if (local->src_cached == prev->this) {
-                /* merge of attr returned only from linkfile creation */
+                local->op_errno = op_errno;
+                local->added_link = _gf_false;
+        } else
                 dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+
+        if (local->op_ret == -1)
+                goto cleanup;
+
+        dht_do_rename (frame);
+
+        return 0;
+
+cleanup:
+        dht_rename_cleanup (frame);
+
+        return 0;
+}
+
+int
+dht_rename_linkto_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      inode_t *inode, struct iatt *stbuf,
+                      struct iatt *preparent, struct iatt *postparent,
+                      dict_t *xdata)
+{
+        dht_local_t     *local = NULL;
+        call_frame_t    *prev = NULL;
+        xlator_t        *src_cached = NULL;
+        dict_t          *xattr = NULL;
+
+        DHT_MARK_FOP_INTERNAL (xattr);
+
+        local = frame->local;
+        prev = cookie;
+
+        src_cached = local->src_cached;
+
+        if (op_ret == -1) {
+                gf_msg_debug (this->name, 0,
+                              "link/file on %s failed (%s)",
+                              prev->this->name, strerror (op_errno));
+                local->op_ret = -1;
+                local->op_errno = op_errno;
         }
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                if (local->op_ret == -1)
-                        goto cleanup;
+        /* If linkto creation failed move to failure cleanup code,
+        * instead of continuing with creating the link file */
+        if (local->op_ret != 0) {
+                goto cleanup;
+        }
 
-                dht_do_rename (frame);
+        gf_msg_trace (this->name, 0,
+                      "link %s => %s (%s)", local->loc.path,
+                      local->loc2.path, src_cached->name);
+        if (uuid_compare (local->loc.pargfid,
+                                local->loc2.pargfid) == 0) {
+                DHT_MARKER_DONT_ACCOUNT(xattr);
         }
 
+        local->added_link = _gf_true;
+
+        STACK_WIND (frame, dht_rename_link_cbk,
+                    src_cached, src_cached->fops->link,
+                    &local->loc, &local->loc2, xattr);
+
+        if (xattr)
+                dict_unref (xattr);
+
         return 0;
 
 cleanup:
         dht_rename_cleanup (frame);
 
+        if (xattr)
+                dict_unref (xattr);
+
         return 0;
 }
 
-
 int
 dht_rename_unlink_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                              int32_t op_ret, int32_t op_errno,
@@ -978,45 +1026,55 @@ dht_rename_create_links (call_frame_t *frame)
                 return 0;
         }
 
-        if (dst_hashed != src_hashed && dst_hashed != src_cached)
+        if (src_cached != dst_hashed) {
+                /* needed to create the link file */
                 call_cnt++;
+                if (dst_hashed != src_hashed)
+                        /* needed to create the linkto file */
+                        call_cnt ++;
+        }
 
-        if (src_cached != dst_hashed)
-                call_cnt++;
-
-        local->call_cnt = call_cnt;
-
-        if (dst_hashed != src_hashed && dst_hashed != src_cached) {
+        /* We should not have any failures post the link creation, as this
+         * introduces the newname into the namespace. Clients could have cached
+         * the existence of the newname and may start taking actions based on
+         * the same. Hence create the linkto first, and then attempt the link.
+         *
+         * NOTE: If another client is attempting the same oldname -> newname
+         * rename, and finds both file names as existing, and are hard links
+         * to each other, then FUSE would send in an unlink for oldname. In
+         * this time duration if we treat the linkto as a critical error and
+         * unlink the newname we created, we would have effectively lost the
+         * file to rename operations. */
+        if (dst_hashed != src_hashed && src_cached != dst_hashed) {
                 gf_msg_trace (this->name, 0,
                               "linkfile %s @ %s => %s",
                               local->loc.path, dst_hashed->name,
                               src_cached->name);
 
                 memcpy (local->gfid, local->loc.inode->gfid, 16);
-		dht_linkfile_create (frame, dht_rename_links_cbk, this,
-				     src_cached, dst_hashed, &local->loc);
-	}
-
-	if (src_cached != dst_hashed) {
+                dht_linkfile_create (frame, dht_rename_linkto_cbk, this,
+                                     src_cached, dst_hashed, &local->loc);
+        } else if (src_cached != dst_hashed) {
                 dict_t *xattr_new = NULL;
 
                 xattr_new = dict_copy_with_ref (xattr, NULL);
 
-		gf_msg_trace (this->name, 0,
+                gf_msg_trace (this->name, 0,
                               "link %s => %s (%s)", local->loc.path,
-			      local->loc2.path, src_cached->name);
+                              local->loc2.path, src_cached->name);
                 if (uuid_compare (local->loc.pargfid,
                                   local->loc2.pargfid) == 0) {
                         DHT_MARKER_DONT_ACCOUNT(xattr_new);
                 }
 
                 local->added_link = _gf_true;
-		STACK_WIND (frame, dht_rename_links_cbk,
-			    src_cached, src_cached->fops->link,
-			    &local->loc, &local->loc2, xattr_new);
+
+                STACK_WIND (frame, dht_rename_link_cbk,
+                            src_cached, src_cached->fops->link,
+                            &local->loc, &local->loc2, xattr_new);
 
                 dict_unref (xattr_new);
-	}
+        }
 
 nolinks:
         if (!call_cnt) {