dht: fix rename race

If two clients try to rename the same file at the same time, we
sometimes end up with *no file at all* in either the old or new
location.  That's kind of bad.  The culprit seems to be some overly
aggressive cleanup code.  AFAICT, based on today's study of the code,
the intent of the changed section is to remove any linkfile we might
have created before the actual rename.  However, what we're removing
might not be our extra link.  If we're racing with another client that's
also doing a rename, it might be the only remaining link to the user's
data.  The solution, which is good enough to pass this test but almost
certainly still not complete, is to be more selective about when we do
this unlink.  Now, we only do it if we know that, at some point, we did
in fact create the link without error (notably ENOENT on the source or
EEXIST on the destination) ourselves.

Change-Id: I8d8cce150b6f8b372c9fb813c90be58d69f8eb7b
BUG: 1117851
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-on: http://review.gluster.org/8269
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

2 files changed, 10 insertions, 3 deletions

diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 54f885d18b0..b4c804d97b4 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -199,6 +199,7 @@ struct dht_local {
         xlator_t        *first_up_subvol;
 
         gf_boolean_t     quota_deem_statfs;
+        gf_boolean_t     added_link;
 };
 typedef struct dht_local dht_local_t;
 
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index 3120db218eb..037f8731fcf 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -442,8 +442,9 @@ dht_rename_cleanup (call_frame_t *frame)
         if (dst_hashed != src_hashed && dst_hashed != src_cached)
                 call_cnt++;
 
-        if (src_cached != dst_hashed)
+        if (local->added_link && (src_cached != dst_hashed)) {
                 call_cnt++;
+        }
 
         local->call_cnt = call_cnt;
 
@@ -475,7 +476,7 @@ dht_rename_cleanup (call_frame_t *frame)
                 xattr_new = NULL;
         }
 
-        if (src_cached != dst_hashed) {
+        if (local->added_link && (src_cached != dst_hashed)) {
                 dict_t *xattr_new = NULL;
 
                 gf_msg_trace (this->name, 0,
@@ -790,8 +791,12 @@ dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                               "link/file on %s failed (%s)",
                               prev->this->name, strerror (op_errno));
                 local->op_ret   = -1;
-                if (op_errno != ENOENT)
+                if (op_errno != ENOENT) {
                         local->op_errno = op_errno;
+                        if (prev->this == local->src_cached) {
+                                local->added_link = _gf_false;
+                        }
+                }
         } else if (local->src_cached == prev->this) {
                 /* merge of attr returned only from linkfile creation */
                 dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
@@ -927,6 +932,7 @@ dht_rename_create_links (call_frame_t *frame)
                         DHT_MARKER_DONT_ACCOUNT(xattr_new);
                 }
 
+                local->added_link = _gf_true;
 		STACK_WIND (frame, dht_rename_links_cbk,
 			    src_cached, src_cached->fops->link,
 			    &local->loc, &local->loc2, xattr_new);