features/shard: Introducing ".shard/.remove_me" for atomic shard deletion (part 1)

PROBLEM:
Shards are deleted synchronously when a sharded file is unlinked or
when a sharded file participating as the dst in a rename() is going to
be replaced. The problem with this approach is it makes the operation
really slow, sometimes causing the application to time out, especially
with large files.

SOLUTION:
To make this operation atomic, we introduce a ".remove_me" directory.
Now renames and unlinks will simply involve two steps:
1. creating an empty file under .remove_me named after the gfid of the file
participating in unlink/rename
2. carrying out the actual rename/unlink
A synctask is created (more on that in part 2) to scan this directory
after every unlink/rename operation (or upon a volume mount) and clean
up all shards associated with it. All of this happens in the background.
The task takes care to delete the shards associated with the gfid in
.remove_me only if this gfid doesn't exist in backend, ensuring that the
file was successfully renamed/unlinked and its shards can be discarded now
safely.

Change-Id: Ia1d238b721a3e99f951a73abbe199e4245f51a3a
updates: bz#1568521
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>

5 files changed, 188 insertions, 31 deletions

diff --git a/tests/bugs/shard/bug-1245547.t b/tests/bugs/shard/bug-1245547.t
index c19b2a6a042..3c46785d10f 100644
--- a/tests/bugs/shard/bug-1245547.t
+++ b/tests/bugs/shard/bug-1245547.t
@@ -25,11 +25,11 @@ TEST touch $M0/bar
 TEST truncate -s 10G $M0/bar
 #Unlink on such a file should succeed.
 TEST unlink $M0/bar
-#
+
 #Create a file 'baz' with holes.
 TEST touch $M0/baz
 TEST truncate -s 10G $M0/baz
 #Rename with a sharded existing dest that has holes must succeed.
 TEST mv -f $M0/foo $M0/baz
 
-cleanup;
+cleanup
diff --git a/tests/bugs/shard/bug-1568521-EEXIST.t b/tests/bugs/shard/bug-1568521-EEXIST.t
new file mode 100644
index 00000000000..e4c3d41098c
--- /dev/null
+++ b/tests/bugs/shard/bug-1568521-EEXIST.t
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+cleanup
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume set $V0 features.shard on
+TEST $CLI volume set $V0 features.shard-block-size 4MB
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0
+
+TEST mkdir $M0/dir
+# Unlink a temporary file to trigger creation of .remove_me
+TEST touch $M0/tmp
+TEST unlink $M0/tmp
+
+TEST stat $B0/${V0}0/.shard/.remove_me
+TEST stat $B0/${V0}1/.shard/.remove_me
+
+TEST dd if=/dev/zero of=$M0/dir/file bs=1024 count=9216
+gfid_file=$(get_gfid_string $M0/dir/file)
+
+# Create marker file from the backend to simulate ENODATA.
+touch $B0/${V0}0/.shard/.remove_me/$gfid_file
+touch $B0/${V0}1/.shard/.remove_me/$gfid_file
+
+# Set block and file size to incorrect values of 64MB and 5MB to simulate "stale xattrs" case
+# and confirm that the correct values are set when the actual unlink takes place
+
+TEST setfattr -n trusted.glusterfs.shard.block-size -v 0x0000000004000000 $B0/${V0}0/.shard/.remove_me/$gfid_file
+TEST setfattr -n trusted.glusterfs.shard.block-size -v 0x0000000004000000 $B0/${V0}1/.shard/.remove_me/$gfid_file
+
+TEST setfattr -n trusted.glusterfs.shard.file-size -v 0x0000000000500000000000000000000000000000000000000000000000000000 $B0/${V0}0/.shard/.remove_me/$gfid_file
+TEST setfattr -n trusted.glusterfs.shard.file-size -v 0x0000000000500000000000000000000000000000000000000000000000000000 $B0/${V0}1/.shard/.remove_me/$gfid_file
+
+# Sleep for 2 seconds to prevent posix_gfid_heal() from believing marker file is "fresh" and failing lookup with ENOENT
+sleep 2
+
+TEST unlink $M0/dir/file
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/.shard/.remove_me/$gfid_file
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/.shard/.remove_me/$gfid_file
+EXPECT "0000000000900000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/.shard/.remove_me/$gfid_file
+EXPECT "0000000000900000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/.shard/.remove_me/$gfid_file
+
+##############################
+### Repeat test for rename ###
+##############################
+
+TEST touch $M0/src
+TEST dd if=/dev/zero of=$M0/dir/dst bs=1024 count=9216
+gfid_dst=$(get_gfid_string $M0/dir/dst)
+
+# Create marker file from the backend to simulate ENODATA.
+touch $B0/${V0}0/.shard/.remove_me/$gfid_dst
+touch $B0/${V0}1/.shard/.remove_me/$gfid_dst
+
+# Set block and file size to incorrect values of 64MB and 5MB to simulate "stale xattrs" case
+# and confirm that the correct values are set when the actual unlink takes place
+
+TEST setfattr -n trusted.glusterfs.shard.block-size -v 0x0000000004000000 $B0/${V0}0/.shard/.remove_me/$gfid_dst
+TEST setfattr -n trusted.glusterfs.shard.block-size -v 0x0000000004000000 $B0/${V0}1/.shard/.remove_me/$gfid_dst
+
+TEST setfattr -n trusted.glusterfs.shard.file-size -v 0x0000000000500000000000000000000000000000000000000000000000000000 $B0/${V0}0/.shard/.remove_me/$gfid_dst
+TEST setfattr -n trusted.glusterfs.shard.file-size -v 0x0000000000500000000000000000000000000000000000000000000000000000 $B0/${V0}1/.shard/.remove_me/$gfid_dst
+
+# Sleep for 2 seconds to prevent posix_gfid_heal() from believing marker file is "fresh" and failing lookup with ENOENT
+sleep 2
+
+TEST mv -f $M0/src $M0/dir/dst
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000900000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000900000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/.shard/.remove_me/$gfid_dst
+
+cleanup
diff --git a/tests/bugs/shard/bug-shard-discard.t b/tests/bugs/shard/bug-shard-discard.t
index 72d8586a2dc..884d9e75ff5 100644
--- a/tests/bugs/shard/bug-shard-discard.t
+++ b/tests/bugs/shard/bug-shard-discard.t
@@ -42,14 +42,14 @@ EXPECT_NOT "1" file_all_zeroes `find $B0 -name $gfid_foo.1`
 
 # Now unlink the file. And ensure that all shards associated with the file are cleaned up
 TEST unlink $M0/foo
-TEST ! stat $B0/${V0}0/.shard/$gfid_foo.1
-TEST ! stat $B0/${V0}1/.shard/$gfid_foo.1
-TEST ! stat $B0/${V0}2/.shard/$gfid_foo.1
-TEST ! stat $B0/${V0}3/.shard/$gfid_foo.1
-TEST ! stat $B0/${V0}0/.shard/$gfid_foo.2
-TEST ! stat $B0/${V0}1/.shard/$gfid_foo.2
-TEST ! stat $B0/${V0}2/.shard/$gfid_foo.2
-TEST ! stat $B0/${V0}3/.shard/$gfid_foo.2
+#TEST ! stat $B0/${V0}0/.shard/$gfid_foo.1
+#TEST ! stat $B0/${V0}1/.shard/$gfid_foo.1
+#TEST ! stat $B0/${V0}2/.shard/$gfid_foo.1
+#TEST ! stat $B0/${V0}3/.shard/$gfid_foo.1
+#TEST ! stat $B0/${V0}0/.shard/$gfid_foo.2
+#TEST ! stat $B0/${V0}1/.shard/$gfid_foo.2
+#TEST ! stat $B0/${V0}2/.shard/$gfid_foo.2
+#TEST ! stat $B0/${V0}3/.shard/$gfid_foo.2
 TEST ! stat $M0/foo
 
 #clean up everything
diff --git a/tests/bugs/shard/shard-inode-refcount-test.t b/tests/bugs/shard/shard-inode-refcount-test.t
index 03e0cc9619b..c92dc07789d 100644
--- a/tests/bugs/shard/shard-inode-refcount-test.t
+++ b/tests/bugs/shard/shard-inode-refcount-test.t
@@ -18,7 +18,7 @@ TEST dd if=/dev/zero conv=fsync of=$M0/one-plus-five-shards bs=1M count=23
 
 ACTIVE_INODES_BEFORE=$(get_mount_active_size_value $V0)
 TEST rm -f $M0/one-plus-five-shards
-EXPECT `expr $ACTIVE_INODES_BEFORE - 5` get_mount_active_size_value $V0
+#EXPECT `expr $ACTIVE_INODES_BEFORE - 4` get_mount_active_size_value $V0
 
 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST $CLI volume stop $V0
diff --git a/tests/bugs/shard/unlinks-and-renames.t b/tests/bugs/shard/unlinks-and-renames.t
index a8f188b2d3a..997c397dedb 100644
--- a/tests/bugs/shard/unlinks-and-renames.t
+++ b/tests/bugs/shard/unlinks-and-renames.t
@@ -32,7 +32,17 @@ TEST truncate -s 5M $M0/dir/foo
 TEST ! stat $B0/${V0}0/.shard
 TEST ! stat $B0/${V0}1/.shard
 # Test to ensure that unlink doesn't fail due to absence of /.shard
+gfid_foo=$(get_gfid_string $M0/dir/foo)
 TEST unlink $M0/dir/foo
+TEST stat $B0/${V0}0/.shard/.remove_me
+TEST stat $B0/${V0}1/.shard/.remove_me
+TEST stat $B0/${V0}0/.shard/.remove_me/$gfid_foo
+TEST stat $B0/${V0}1/.shard/.remove_me/$gfid_foo
+
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/.shard/.remove_me/$gfid_foo
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/.shard/.remove_me/$gfid_foo
+EXPECT "0000000000500000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/.shard/.remove_me/$gfid_foo
+EXPECT "0000000000500000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/.shard/.remove_me/$gfid_foo
 
 ##################################################
 ##### Unlink of a sharded file without holes #####
@@ -46,14 +56,20 @@ TEST stat $B0/${V0}1/.shard/$gfid_new.1
 TEST stat $B0/${V0}0/.shard/$gfid_new.2
 TEST stat $B0/${V0}1/.shard/$gfid_new.2
 TEST unlink $M0/dir/new
-TEST ! stat $B0/${V0}0/.shard/$gfid_new.1
-TEST ! stat $B0/${V0}1/.shard/$gfid_new.1
-TEST ! stat $B0/${V0}0/.shard/$gfid_new.2
-TEST ! stat $B0/${V0}1/.shard/$gfid_new.2
+#TEST ! stat $B0/${V0}0/.shard/$gfid_new.1
+#TEST ! stat $B0/${V0}1/.shard/$gfid_new.1
+#TEST ! stat $B0/${V0}0/.shard/$gfid_new.2
+#TEST ! stat $B0/${V0}1/.shard/$gfid_new.2
 TEST ! stat $M0/dir/new
 TEST ! stat $B0/${V0}0/dir/new
 TEST ! stat $B0/${V0}1/dir/new
+TEST stat $B0/${V0}0/.shard/.remove_me/$gfid_new
+TEST stat $B0/${V0}1/.shard/.remove_me/$gfid_new
 
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/.shard/.remove_me/$gfid_new
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/.shard/.remove_me/$gfid_new
+EXPECT "0000000000900000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/.shard/.remove_me/$gfid_new
+EXPECT "0000000000900000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/.shard/.remove_me/$gfid_new
 #######################################
 ##### Unlink with /.shard present #####
 #######################################
@@ -67,18 +83,32 @@ TEST unlink $M0/dir/foo
 TEST ! stat $B0/${V0}0/dir/foo
 TEST ! stat $B0/${V0}1/dir/foo
 TEST ! stat $M0/dir/foo
+TEST stat $B0/${V0}0/.shard/.remove_me/$gfid_foo
+TEST stat $B0/${V0}1/.shard/.remove_me/$gfid_foo
+
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/.shard/.remove_me/$gfid_foo
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/.shard/.remove_me/$gfid_foo
+EXPECT "0000000000500000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/.shard/.remove_me/$gfid_foo
+EXPECT "0000000000500000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/.shard/.remove_me/$gfid_foo
 
 #############################################################
 ##### Unlink of a file with only one block (the zeroth) #####
 #############################################################
 TEST touch $M0/dir/foo
+gfid_foo=$(get_gfid_string $M0/dir/foo)
 TEST dd if=/dev/zero of=$M0/dir/foo bs=1024 count=1024
-# Test to ensure that unlink of a sparse file works fine.
+# Test to ensure that unlink of a file with only base shard works fine.
 TEST unlink $M0/dir/foo
 TEST ! stat $B0/${V0}0/dir/foo
 TEST ! stat $B0/${V0}1/dir/foo
 TEST ! stat $M0/dir/foo
+TEST stat $B0/${V0}0/.shard/.remove_me/$gfid_foo
+TEST stat $B0/${V0}1/.shard/.remove_me/$gfid_foo
 
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/.shard/.remove_me/$gfid_foo
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/.shard/.remove_me/$gfid_foo
+EXPECT "0000000000100000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/.shard/.remove_me/$gfid_foo
+EXPECT "0000000000100000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/.shard/.remove_me/$gfid_foo
 ####################################################
 ##### Unlink of a sharded file with hard-links #####
 ####################################################
@@ -94,6 +124,8 @@ TEST stat $B0/${V0}1/.shard/$gfid_original.2
 TEST ln $M0/dir/original $M0/link
 # Now delete the original file.
 TEST unlink $M0/dir/original
+TEST ! stat $B0/${V0}0/.shard/.remove_me/$gfid_original
+TEST ! stat $B0/${V0}1/.shard/.remove_me/$gfid_original
 # Ensure the shards are still intact.
 TEST stat $B0/${V0}0/.shard/$gfid_original.1
 TEST stat $B0/${V0}1/.shard/$gfid_original.1
@@ -105,15 +137,22 @@ TEST stat $B0/${V0}0/link
 TEST stat $B0/${V0}1/link
 # Now delete the last link.
 TEST unlink $M0/link
+TEST stat $B0/${V0}0/.shard/.remove_me/$gfid_original
+TEST stat $B0/${V0}1/.shard/.remove_me/$gfid_original
 # Ensure that the shards are all cleaned up.
-TEST ! stat $B0/${V0}0/.shard/$gfid_original.1
-TEST ! stat $B0/${V0}1/.shard/$gfid_original.1
-TEST ! stat $B0/${V0}0/.shard/$gfid_original.2
-TEST ! stat $B0/${V0}1/.shard/$gfid_original.2
-TEST ! stat $M0/link
+#TEST ! stat $B0/${V0}0/.shard/$gfid_original.1
+#TEST ! stat $B0/${V0}1/.shard/$gfid_original.1
+#TEST ! stat $B0/${V0}0/.shard/$gfid_original.2
+#TEST ! stat $B0/${V0}1/.shard/$gfid_original.2
+#TEST ! stat $M0/link
 TEST ! stat $B0/${V0}0/link
 TEST ! stat $B0/${V0}1/link
 
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/.shard/.remove_me/$gfid_original
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/.shard/.remove_me/$gfid_original
+EXPECT "0000000000900000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/.shard/.remove_me/$gfid_original
+EXPECT "0000000000900000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/.shard/.remove_me/$gfid_original
+
 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST $CLI volume stop $V0
 TEST $CLI volume delete $V0
@@ -140,6 +179,7 @@ TEST touch $M0/dir/dst
 ##### Rename with /.shard absent #####
 ######################################
 TEST truncate -s 5M $M0/dir/dst
+gfid_dst=$(get_gfid_string $M0/dir/dst)
 TEST ! stat $B0/${V0}0/.shard
 TEST ! stat $B0/${V0}1/.shard
 # Test to ensure that rename doesn't fail due to absence of /.shard
@@ -150,6 +190,13 @@ TEST ! stat $B0/${V0}0/dir/src
 TEST ! stat $B0/${V0}1/dir/src
 TEST   stat $B0/${V0}0/dir/dst
 TEST   stat $B0/${V0}1/dir/dst
+TEST stat $B0/${V0}0/.shard/.remove_me/$gfid_dst
+TEST stat $B0/${V0}1/.shard/.remove_me/$gfid_dst
+
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000500000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000500000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/.shard/.remove_me/$gfid_dst
 
 ##################################################
 ##### Rename to a sharded file without holes #####
@@ -165,16 +212,23 @@ TEST stat $B0/${V0}1/.shard/$gfid_dst.1
 TEST stat $B0/${V0}0/.shard/$gfid_dst.2
 TEST stat $B0/${V0}1/.shard/$gfid_dst.2
 TEST mv -f $M0/dir/src $M0/dir/dst
-TEST ! stat $B0/${V0}0/.shard/$gfid_dst.1
-TEST ! stat $B0/${V0}1/.shard/$gfid_dst.1
-TEST ! stat $B0/${V0}0/.shard/$gfid_dst.2
-TEST ! stat $B0/${V0}1/.shard/$gfid_dst.2
+#TEST ! stat $B0/${V0}0/.shard/$gfid_dst.1
+#TEST ! stat $B0/${V0}1/.shard/$gfid_dst.1
+#TEST ! stat $B0/${V0}0/.shard/$gfid_dst.2
+#TEST ! stat $B0/${V0}1/.shard/$gfid_dst.2
 TEST ! stat $M0/dir/src
 TEST   stat $M0/dir/dst
 TEST ! stat $B0/${V0}0/dir/src
 TEST ! stat $B0/${V0}1/dir/src
 TEST   stat $B0/${V0}0/dir/dst
 TEST   stat $B0/${V0}1/dir/dst
+TEST stat $B0/${V0}0/.shard/.remove_me/$gfid_dst
+TEST stat $B0/${V0}1/.shard/.remove_me/$gfid_dst
+
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000900000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000900000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/.shard/.remove_me/$gfid_dst
 
 ###################################################
 ##### Rename of dst file with /.shard present #####
@@ -182,7 +236,8 @@ TEST   stat $B0/${V0}1/dir/dst
 TEST unlink $M0/dir/dst
 TEST touch $M0/dir/src
 TEST truncate -s 5M $M0/dir/dst
-# Test to ensure that unlink of a sparse file works fine.
+gfid_dst=$(get_gfid_string $M0/dir/dst)
+# Test to ensure that rename into a sparse file works fine.
 TEST mv -f $M0/dir/src $M0/dir/dst
 TEST ! stat $M0/dir/src
 TEST   stat $M0/dir/dst
@@ -190,6 +245,13 @@ TEST ! stat $B0/${V0}0/dir/src
 TEST ! stat $B0/${V0}1/dir/src
 TEST   stat $B0/${V0}0/dir/dst
 TEST   stat $B0/${V0}1/dir/dst
+TEST stat $B0/${V0}0/.shard/.remove_me/$gfid_dst
+TEST stat $B0/${V0}1/.shard/.remove_me/$gfid_dst
+
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000500000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000500000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/.shard/.remove_me/$gfid_dst
 
 ###############################################################
 ##### Rename of dst file with only one block (the zeroth) #####
@@ -197,7 +259,8 @@ TEST   stat $B0/${V0}1/dir/dst
 TEST unlink $M0/dir/dst
 TEST touch $M0/dir/src
 TEST dd if=/dev/zero of=$M0/dir/dst bs=1024 count=1024
-# Test to ensure that unlink of a sparse file works fine.
+gfid_dst=$(get_gfid_string $M0/dir/dst)
+# Test to ensure that rename into a file with only base shard works fine.
 TEST mv -f $M0/dir/src $M0/dir/dst
 TEST ! stat $M0/dir/src
 TEST   stat $M0/dir/dst
@@ -205,6 +268,13 @@ TEST ! stat $B0/${V0}0/dir/src
 TEST ! stat $B0/${V0}1/dir/src
 TEST   stat $B0/${V0}0/dir/dst
 TEST   stat $B0/${V0}1/dir/dst
+TEST stat $B0/${V0}0/.shard/.remove_me/$gfid_dst
+TEST stat $B0/${V0}1/.shard/.remove_me/$gfid_dst
+
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000100000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000100000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/.shard/.remove_me/$gfid_dst
 
 ########################################################
 ##### Rename to a dst sharded file with hard-links #####
@@ -231,18 +301,26 @@ TEST stat $B0/${V0}1/.shard/$gfid_dst.2
 TEST ! stat $M0/dir/src
 TEST ! stat $B0/${V0}0/dir/src
 TEST ! stat $B0/${V0}1/dir/src
+TEST ! stat $B0/${V0}0/.shard/.remove_me/$gfid_dst
+TEST ! stat $B0/${V0}1/.shard/.remove_me/$gfid_dst
 # Now rename another file to the last link.
 TEST touch $M0/dir/src2
 TEST mv -f $M0/dir/src2 $M0/link
 # Ensure that the shards are all cleaned up.
-TEST ! stat $B0/${V0}0/.shard/$gfid_dst.1
-TEST ! stat $B0/${V0}1/.shard/$gfid_dst.1
-TEST ! stat $B0/${V0}0/.shard/$gfid_dst.2
-TEST ! stat $B0/${V0}1/.shard/$gfid_dst.2
+#TEST ! stat $B0/${V0}0/.shard/$gfid_dst.1
+#TEST ! stat $B0/${V0}1/.shard/$gfid_dst.1
+#TEST ! stat $B0/${V0}0/.shard/$gfid_dst.2
+#TEST ! stat $B0/${V0}1/.shard/$gfid_dst.2
 TEST ! stat $M0/dir/src2
 TEST ! stat $B0/${V0}0/dir/src2
 TEST ! stat $B0/${V0}1/dir/src2
+TEST stat $B0/${V0}0/.shard/.remove_me/$gfid_dst
+TEST stat $B0/${V0}1/.shard/.remove_me/$gfid_dst
 
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000900000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/.shard/.remove_me/$gfid_dst
+EXPECT "0000000000900000000000000000000000000000000000000000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/.shard/.remove_me/$gfid_dst
 # Rename with non-existent dst and a sharded src
 TEST touch $M0/dir/src
 TEST dd if=/dev/zero of=$M0/dir/src bs=1024 count=9216