cluster/afr: refactor

- Remove client side self-healing completely (opendir, openfd, lookup)
- Re-work readdir-failover to work reliably in case of NFS
- Remove unused/dead lock recovery code
- Consistently use xdata in both calls and callbacks in all FOPs
- Per-inode event generation, used to force inode ctx refresh
- Implement dirty flag support (in place of pending counts)
- Eliminate inode ctx structure, use read subvol bits + event_generation
- Implement inode ctx refreshing based on event generation
- Provide backward compatibility in transactions
- remove unused variables and functions
- make code more consistent in style and pattern
- regularize and clean up inode-write transaction code
- regularize and clean up dir-write transaction code
- regularize and clean up common FOPs
- reorganize transaction framework code
- skip setting xattrs in pending dict if nothing is pending
- re-write self-healing code using syncops
- re-write simpler self-heal-daemon

Change-Id: I1e4080c9796c8a2815c2dab4be3073f389d614a8
BUG: 1021686
Signed-off-by: Anand Avati <avati@redhat.com>
Reviewed-on: http://review.gluster.org/6010
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

19 files changed, 101 insertions, 296 deletions

diff --git a/tests/basic/pump.t b/tests/basic/pump.t
index 3faf06f0502..23bdc187d90 100644
--- a/tests/basic/pump.t
+++ b/tests/basic/pump.t
@@ -22,7 +22,7 @@ done
 cd
 TEST umount $M0
 TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 start
-EXPECT_WITHIN 60 "Y" gd_is_replace_brick_completed $H0 $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1
+EXPECT_WITHIN 600 "Y" gd_is_replace_brick_completed $H0 $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1
 TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 commit
 TEST $CLI volume stop $V0
 TEST diff -r --exclude=.glusterfs $B0/${V0}0 $B0/${V0}1
diff --git a/tests/bugs/859927/repl.t b/tests/bugs/859927/repl.t
index 73c86e7be3c..856b057fbcb 100755
--- a/tests/bugs/859927/repl.t
+++ b/tests/bugs/859927/repl.t
@@ -33,20 +33,20 @@ TEST $CLI volume set $V0 cluster.data-self-heal-algorithm full
 EXPECT full volume_option $V0 cluster.data-self-heal-algorithm
 create_setup_for_self_heal $M0/a
 EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
-ls -l $file 2>&1 > /dev/null
+cat $file 2>&1 > /dev/null
 TEST cmp $B0/${V0}1/a $B0/${V0}2/a
 
 TEST $CLI volume set $V0 cluster.data-self-heal-algorithm diff
 EXPECT diff volume_option $V0 cluster.data-self-heal-algorithm
 create_setup_for_self_heal $M0/a
 EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
-ls -l $file 2>&1 > /dev/null
+cat $file 2>&1 > /dev/null
 TEST cmp $B0/${V0}1/a $B0/${V0}2/a
 
 TEST $CLI volume reset $V0 cluster.data-self-heal-algorithm
 create_setup_for_self_heal $M0/a
 EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
-ls -l $file 2>&1 > /dev/null
+cat $file 2>&1 > /dev/null
 TEST cmp $B0/${V0}1/a $B0/${V0}2/a
 
 TEST ! $CLI volume set $V0 cluster.data-self-heal-algorithm ""
diff --git a/tests/bugs/bug-1015990-rep.t b/tests/bugs/bug-1015990-rep.t
index f59bb2f75ba..bca0d7aff07 100755
--- a/tests/bugs/bug-1015990-rep.t
+++ b/tests/bugs/bug-1015990-rep.t
@@ -35,7 +35,6 @@ for  i in  {1..100}; do echo "STRING" > $M0/File$i; done
 brick_2_sh_entries=$(count_sh_entries $B0/$V0"2")
 brick_4_sh_entries=$(count_sh_entries $B0/$V0"4")
 
-
 command_output=$(gluster volume heal $V0 statistics heal-count replica $H0:$B0/$V0"1")
 
 
diff --git a/tests/bugs/bug-1035576.t b/tests/bugs/bug-1035576.t
index 52d93dd87df..938306a8503 100644
--- a/tests/bugs/bug-1035576.t
+++ b/tests/bugs/bug-1035576.t
@@ -34,7 +34,8 @@ quota_limit_val1=$(get_hex_xattr trusted.glusterfs.quota.limit-set $B0/${V0}1/a)
 quota_size_val1=$(get_hex_xattr trusted.glusterfs.quota.size $B0/${V0}1/a)
 
 #Trigger entry,metadata self-heal
-TEST stat $M0/a
+TEST ls $M0/a
+
 quota_limit_val0=$(get_hex_xattr trusted.glusterfs.quota.limit-set $B0/${V0}0/a)
 quota_size_val0=$(get_hex_xattr trusted.glusterfs.quota.size $B0/${V0}0/a)
 
@@ -43,7 +44,7 @@ TEST [ $quota_limit_val0 == $quota_limit_val1 ]
 
 #Only entry, metadata self-heal is done quota size value should not be same
 TEST [ $quota_size_val0 != $quota_size_val1 ]
-TEST stat $M0/a/f
+TEST cat $M0/a/f
 
 #Now that data self-heal is done quota size value should be same
 quota_size_val0=$(get_hex_xattr trusted.glusterfs.quota.size $B0/${V0}0/a)
diff --git a/tests/bugs/bug-1037501.t b/tests/bugs/bug-1037501.t
index d11c788a093..596122a727e 100755
--- a/tests/bugs/bug-1037501.t
+++ b/tests/bugs/bug-1037501.t
@@ -24,14 +24,6 @@ TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}-{0,1,2}
 EXPECT "$V0" volinfo_field $V0 'Volume Name';
 EXPECT 'Created' volinfo_field $V0 'Status';
 
-## Make sure io-cache and write-behind don't interfere.
-TEST $CLI volume set $V0 data-self-heal off;
-
-## Make sure automatic self-heal doesn't perturb our results.
-TEST $CLI volume set $V0 cluster.self-heal-daemon off
-
-TEST $CLI volume set $V0 background-self-heal-count 0
-
 ## Start volume and verify
 TEST $CLI volume start $V0;
 EXPECT 'Started' volinfo_field $V0 'Status';
@@ -48,206 +40,38 @@ TEST $CLI volume add-brick $V0 replica 4 $H0:$B0/$V0-3 force
 TEST $CLI volume add-brick $V0 replica 5 $H0:$B0/$V0-4 force
 TEST $CLI volume add-brick $V0 replica 6 $H0:$B0/$V0-5 force
 
-sleep 10
-
-TEST ls $M0/
-
-
-function compare()
-{
-	var=-1;
-	if [ $1 == $2 ]; then
-		var=0;
-	else
-		var=-1;
-	fi
-
-	echo $var
-}
-
-var2="000000000000000000000000"
-
-var1=`getfattr -d -m . $B0/$V0-0/File -e hex 2>&1 | grep "client-3"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-0/File -e hex 2>&1 | grep "client-4"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-0/File -e hex 2>&1 | grep "client-5"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-1/File -e hex 2>&1 | grep "client-3"`
-EXPECT "0" echo $?
-var3=`echo $var1| cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-1/File -e hex 2>&1 | grep "client-4"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-1/File -e hex 2>&1 | grep "client-5"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-2/File -e hex 2>&1 | grep "client-3"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-2/File -e hex 2>&1 | grep "client-4"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-2/File -e hex 2>&1 | grep "client-5"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-0/Dir -e hex 2>&1 | grep "client-3"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-0/Dir -e hex 2>&1 | grep "client-4"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-0/Dir -e hex 2>&1 | grep "client-5"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-1/Dir -e hex 2>&1 | grep "client-3"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-1/Dir -e hex 2>&1 | grep "client-4"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-1/Dir -e hex 2>&1 | grep "client-5"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-2/Dir -e hex 2>&1 | grep "client-3"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-2/Dir -e hex 2>&1 | grep "client-4"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-2/Dir -e hex 2>&1 | grep "client-5"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-
-var1=`getfattr -d -m . $B0/$V0-0/Link -e hex 2>&1 | grep "client-3"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-0/Link -e hex 2>&1 | grep "client-4"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-0/Link -e hex 2>&1 | grep "client-5"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-1/Link -e hex 2>&1 | grep "client-3"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-1/Link -e hex 2>&1 | grep "client-4"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-1/Link -e hex 2>&1 | grep "client-5"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-2/Link -e hex 2>&1 | grep "client-3"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-2/Link -e hex 2>&1 | grep "client-4"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-2/Link -e hex 2>&1 | grep "client-5"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-
-
-var1=`getfattr -d -m . $B0/$V0-0/FIFO -e hex 2>&1 | grep "client-3"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-0/FIFO -e hex 2>&1 | grep "client-4"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-0/FIFO -e hex 2>&1 | grep "client-5"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-1/FIFO -e hex 2>&1 | grep "client-3"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-1/FIFO -e hex 2>&1 | grep "client-4"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-1/FIFO -e hex 2>&1 | grep "client-5"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-2/FIFO -e hex 2>&1 | grep "client-3"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-2/FIFO -e hex 2>&1 | grep "client-4"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
-
-var1=`getfattr -d -m . $B0/$V0-2/FIFO -e hex 2>&1 | grep "client-5"`
-EXPECT "0" echo $?
-var3=`echo $var1 | cut -d x -f 2`
-EXPECT_NOT $var2 echo $var3
+sleep 5
+
+TEST gluster volume heal $V0 full
+
+sleep 5
+
+EXPECT 10 stat -c '%s' $B0/$V0-0/File
+EXPECT 10 stat -c '%s' $B0/$V0-1/File
+EXPECT 10 stat -c '%s' $B0/$V0-2/File
+EXPECT 10 stat -c '%s' $B0/$V0-3/File
+EXPECT 10 stat -c '%s' $B0/$V0-4/File
+EXPECT 10 stat -c '%s' $B0/$V0-5/File
+
+EXPECT 3 stat -c '%h' $B0/$V0-0/Link
+EXPECT 3 stat -c '%h' $B0/$V0-1/Link
+EXPECT 3 stat -c '%h' $B0/$V0-2/Link
+EXPECT 3 stat -c '%h' $B0/$V0-3/Link
+EXPECT 3 stat -c '%h' $B0/$V0-4/Link
+EXPECT 3 stat -c '%h' $B0/$V0-5/Link
+
+EXPECT 'directory' stat -c '%F' $B0/$V0-0/Dir
+EXPECT 'directory' stat -c '%F' $B0/$V0-1/Dir
+EXPECT 'directory' stat -c '%F' $B0/$V0-2/Dir
+EXPECT 'directory' stat -c '%F' $B0/$V0-3/Dir
+EXPECT 'directory' stat -c '%F' $B0/$V0-4/Dir
+EXPECT 'directory' stat -c '%F' $B0/$V0-5/Dir
+
+EXPECT 'fifo' stat -c '%F' $B0/$V0-0/FIFO
+EXPECT 'fifo' stat -c '%F' $B0/$V0-1/FIFO
+EXPECT 'fifo' stat -c '%F' $B0/$V0-2/FIFO
+EXPECT 'fifo' stat -c '%F' $B0/$V0-3/FIFO
+EXPECT 'fifo' stat -c '%F' $B0/$V0-4/FIFO
+EXPECT 'fifo' stat -c '%F' $B0/$V0-5/FIFO
 
 cleanup;
diff --git a/tests/bugs/bug-1058797.t b/tests/bugs/bug-1058797.t
index 2b80794cf06..1e9f09af0a8 100644
--- a/tests/bugs/bug-1058797.t
+++ b/tests/bugs/bug-1058797.t
@@ -29,7 +29,7 @@ EXPECT "s" echo $setuid_bit1
 #Restart volume and do lookup from mount to trigger heal
 TEST $CLI volume start $V0 force
 EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
-TEST ls -l $M0/file
+TEST dd if=$M0/file of=/dev/null
 
 #Get file permissions from healed brick1 and verify that S_ISUID is indeed set
 file_permissions2=`ls -l $B0/brick1/file | awk '{print $1}' | cut -d. -f1 | cut -d- -f2,3,4,5,6`
diff --git a/tests/bugs/bug-767585-gfid.t b/tests/bugs/bug-767585-gfid.t
index 49cf7423fa0..41043a0b247 100755
--- a/tests/bugs/bug-767585-gfid.t
+++ b/tests/bugs/bug-767585-gfid.t
@@ -26,10 +26,9 @@ TEST setfattr -n trusted.gfid -v $gfid2 $B0/${V0}1/c
 
 sleep 2
 
-cd $M0
-TEST ls -l a
-TEST ls -l b
-TEST ls -l c
+TEST stat $M0/a
+TEST stat $M0/b
+TEST stat $M0/c
 
 TEST gf_get_gfid_xattr $B0/${V0}0/a
 TEST gf_get_gfid_xattr $B0/${V0}1/a
diff --git a/tests/bugs/bug-802417.t b/tests/bugs/bug-802417.t
index 314141f6b6d..b596df30385 100755
--- a/tests/bugs/bug-802417.t
+++ b/tests/bugs/bug-802417.t
@@ -55,7 +55,7 @@ EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
 EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
 EXPECT_WITHIN 20 "1" afr_child_up_status $V0 2
 TEST kill_brick ${V0} ${H0} ${B0}/${V0}-2
-TEST ls -l ${M0}/a_file
+TEST dd if=${M0}/a_file of=/dev/null
 
 
 obs_path_0=${B0}/${V0}-0/a_file
@@ -67,31 +67,31 @@ tgt_xattr_1="trusted.afr.${V0}-client-1"
 tgt_xattr_2="trusted.afr.${V0}-client-2"
 
 actual=$(afr_get_changelog_xattr $obs_path_0 $tgt_xattr_0)
-EXPECT "0x000000000000000000000000" echo $actual
+EXPECT "0x000000000000000000000000|^\$" echo $actual
 
 actual=$(afr_get_changelog_xattr $obs_path_0 $tgt_xattr_1)
-EXPECT "0x000000000000000000000000" echo $actual
+EXPECT "0x000000000000000000000000|^\$" echo $actual
 
 actual=$(afr_get_changelog_xattr $obs_path_0 $tgt_xattr_2)
-EXPECT "0x000000020000000000000000" echo $actual
+EXPECT "0x000000030000000000000000" echo $actual
 
 actual=$(afr_get_changelog_xattr $obs_path_1 $tgt_xattr_0)
-EXPECT "0x000000000000000000000000" echo $actual
+EXPECT "0x000000000000000000000000|^\$" echo $actual
 
 actual=$(afr_get_changelog_xattr $obs_path_1 $tgt_xattr_1)
-EXPECT "0x000000000000000000000000" echo $actual
+EXPECT "0x000000000000000000000000|^\$" echo $actual
 
 actual=$(afr_get_changelog_xattr $obs_path_1 $tgt_xattr_2)
-EXPECT "0x000000020000000000000000" echo $actual
+EXPECT "0x000000010000000000000000" echo $actual
 
 actual=$(afr_get_changelog_xattr $obs_path_2 $tgt_xattr_0)
-EXPECT "0x000000000000000000000000" echo $actual
+EXPECT "0x000000000000000000000000|^\$" echo $actual
 
 actual=$(afr_get_changelog_xattr $obs_path_2 $tgt_xattr_1)
-EXPECT "0x000000000000000000000000" echo $actual
+EXPECT "0x000000000000000000000000|^\$" echo $actual
 
 actual=$(afr_get_changelog_xattr $obs_path_2 $tgt_xattr_2)
-EXPECT "0x000000000000000000000000" echo $actual
+EXPECT "0x000000000000000000000000|^\$" echo $actual
 
 if [ "$EXIT_EARLY" = "1" ]; then
 	exit 0;
diff --git a/tests/bugs/bug-830665.t b/tests/bugs/bug-830665.t
index 0073ff1d948..dd6f3ce2f12 100755
--- a/tests/bugs/bug-830665.t
+++ b/tests/bugs/bug-830665.t
@@ -81,15 +81,17 @@ ls -l $N0 &> /dev/null;
 sleep 5;
 
 ## Force entry self-heal.
-find $N0 | xargs stat > /dev/null;
+TEST $CLI volume set $V0 cluster.self-heal-daemon on
+sleep 1
+TEST gluster volume heal $V0 full
 #ls -lR $N0 > /dev/null;
 
 ## Do NOT check through the NFS mount here. That will force a new self-heal
 ## check, but we want to test whether self-heal already happened.
 
 ## Make sure everything's in order on the recreated brick.
-EXPECT 'test_data' cat $B0/${V0}-0/a_file;
-EXPECT 'more_test_data' cat $B0/${V0}-0/a_dir/another_file;
+EXPECT_WITHIN 20 'test_data' cat $B0/${V0}-0/a_file;
+EXPECT_WITHIN 20 'more_test_data' cat $B0/${V0}-0/a_dir/another_file;
 
 if [ "$EXIT_EARLY" = "1" ]; then
 	exit 0;
diff --git a/tests/bugs/bug-853690.t b/tests/bugs/bug-853690.t
index 77a581f5444..c2f82d1030a 100755
--- a/tests/bugs/bug-853690.t
+++ b/tests/bugs/bug-853690.t
@@ -66,7 +66,6 @@ TEST glusterfs --volfile=$B0/test.vol --attribute-timeout=0 --entry-timeout=0 $M
 # file sizes and immediate split-brain (EIO).
 TEST dd if=/dev/zero of=$M0/file bs=128k count=1
 TEST dd if=$M0/file of=/dev/null bs=128k count=1
-
 ########
 #
 # Test self-heal with short writes...
@@ -76,14 +75,11 @@ TEST dd if=$M0/file of=/dev/null bs=128k count=1
 # Cause a lookup and wait a few seconds for posterity. This self-heal also fails
 # due to a short write.
 TEST ls $M0/file
-
 # Verify the attributes on the healthy replica do not reflect consistency with
 # the other replica.
-TEST "getfattr -n trusted.afr.test-locks-0 $B0/test2/file --only-values > $B0/out1 2> /dev/null"
-TEST "getfattr -n trusted.afr.test-locks-1 $B0/test2/file --only-values > $B0/out2 2> /dev/null"
-TEST ! cmp $B0/out1 $B0/out2
+xa=`getfattr -n trusted.afr.test-locks-0 -e hex $B0/test2/file 2>&1 | grep = | cut -f2 -d=`
+EXPECT_NOT 0x000000000000000000000000 echo $xa
 
-TEST rm -f $B0/out1 $B0/out2
 TEST rm -f $M0/file
 TEST umount $M0
 
diff --git a/tests/bugs/bug-865825.t b/tests/bugs/bug-865825.t
index 6bb1c23482d..8ee75186484 100755
--- a/tests/bugs/bug-865825.t
+++ b/tests/bugs/bug-865825.t
@@ -2,6 +2,8 @@
 
 . $(dirname $0)/../include.rc
 
+cleanup;
+
 TEST glusterd
 TEST pidof glusterd
 TEST $CLI volume info;
@@ -28,6 +30,7 @@ EXPECT 'Created' volinfo_field $V0 'Status';
 ## Make sure io-cache and write-behind don't interfere.
 TEST $CLI volume set $V0 cluster.background-self-heal-count 0
 TEST $CLI volume set $V0 performance.io-cache off;
+TEST $CLI volume set $V0 performance.quick-read off;
 TEST $CLI volume set $V0 performance.write-behind off;
 TEST $CLI volume set $V0 performance.stat-prefetch off
 
@@ -54,19 +57,18 @@ setfattr -n trusted.afr.${V0}-client-2 -v $value $B0/${V0}-0/a_file
 setfattr -x trusted.afr.${V0}-client-2 $B0/${V0}-1/a_file
 echo "wrong_data" > $B0/${V0}-2/a_file
 
-## Remount and force a self-heal.
-TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
-stat ${M0}/a_file > /dev/null
+gluster volume set $V0 cluster.self-heal-daemon on
+sleep 3
+gluster volume heal $V0 full
 
 ## Make sure brick 2 now has the correct contents.
-EXPECT "test_data" cat $B0/${V0}-2/a_file
+EXPECT_WITHIN 30 "test_data" cat $B0/${V0}-2/a_file
 
 if [ "$EXIT_EARLY" = "1" ]; then
 	exit 0;
 fi
 
 ## Finish up
-TEST umount $M0;
 TEST $CLI volume stop $V0;
 EXPECT 'Stopped' volinfo_field $V0 'Status';
 
diff --git a/tests/bugs/bug-873962.t b/tests/bugs/bug-873962.t
index b245cc3dab5..0281417f07f 100755
--- a/tests/bugs/bug-873962.t
+++ b/tests/bugs/bug-873962.t
@@ -61,11 +61,12 @@ EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
 EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
 
 TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M1 --direct-io-mode=enable
+
 #Files are in split-brain, so open should fail
 TEST ! cat $M0/a;
 TEST ! cat $M1/a;
-TEST ! cat $M0/b;
-TEST ! cat $M1/b;
+TEST cat $M0/b;
+TEST cat $M1/b;
 
 #Reset split-brain status
 TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/a;
@@ -75,6 +76,7 @@ TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0
 EXPECT "2" cat $M0/a;
 # FAIL HERE - see comment about cluster.self-heal-background-count above.
 EXPECT "2" cat $M1/a;
+TEST dd if=$M0/b of=/dev/null bs=1M
 EXPECT "def" getfattr -n trusted.mdata --only-values $M0/b 2>/dev/null
 EXPECT "def" getfattr -n trusted.mdata --only-values $M1/b 2>/dev/null
 
@@ -90,8 +92,8 @@ TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $
 #Files are in split-brain, so open should fail
 TEST ! cat $M0/c
 TEST ! cat $M1/c
-TEST ! cat $M0/d
-TEST ! cat $M1/d
+TEST cat $M0/d
+TEST cat $M1/d
 
 TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/c
 TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/d
@@ -102,7 +104,4 @@ EXPECT "2" cat $M1/c
 EXPECT "1" cat $M0/d
 EXPECT "1" cat $M1/d
 
-#Check that the self-heal is not triggered.
-EXPECT "1" cat $B0/${V0}1/c
-EXPECT "abc" getfattr -n trusted.mdata --only-values $B0/${V0}1/d 2>/dev/null
 cleanup;
diff --git a/tests/bugs/bug-888174.t b/tests/bugs/bug-888174.t
index 4ea34645bd6..ef653f76da2 100644
--- a/tests/bugs/bug-888174.t
+++ b/tests/bugs/bug-888174.t
@@ -38,10 +38,9 @@ TEST [ -z $inodelk_max_latency ]
 
 TEST dd of=$M0/a if=/dev/urandom bs=1M count=10 conv=fsync
 #Check for no trace of pending changelog. Flush should make sure of it.
-EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_0/a trusted.afr.$V0-client-0
-EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_0/a trusted.afr.$V0-client-1
-EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_1/a trusted.afr.$V0-client-0
-EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_1/a trusted.afr.$V0-client-1
+EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_0/a trusted.afr.dirty
+EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_1/a trusted.afr.dirty
+
 
 dd of=$M0/a if=/dev/urandom bs=1M count=1024 2>/dev/null &
 p=$!
@@ -51,15 +50,13 @@ TEST $CLI volume set $V0 performance.io-cache off
 TEST $CLI volume set $V0 performance.stat-prefetch off
 TEST $CLI volume set $V0 performance.read-ahead off
 
-kill -SIGTERM $p
+kill -TERM $p
 #wait for dd to exit
 wait  > /dev/null 2>&1
 
 #Goal is to check if there is permanent FOOL changelog
 sleep 5
-EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_0/a trusted.afr.$V0-client-0
-EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_0/a trusted.afr.$V0-client-1
-EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_1/a trusted.afr.$V0-client-0
-EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_1/a trusted.afr.$V0-client-1
+EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_0/a trusted.afr.dirty
+EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_1/a trusted.afr.dirty
 
 cleanup;
diff --git a/tests/bugs/bug-906646.t b/tests/bugs/bug-906646.t
index 0e6a3bcb695..b2cbf6bc32e 100644
--- a/tests/bugs/bug-906646.t
+++ b/tests/bugs/bug-906646.t
@@ -84,7 +84,7 @@ TEST $CLI volume start $V0 force
 
 EXPECT_WITHIN 20 "1" afr_child_up_status $V0 `expr $brick_id - 1`
 
-stat $pth
+cat $pth >/dev/null
 
 # check backends - xattr should not be present anywhere
 EXPECT 1 xattr_query_check ${backend_paths_array[0]} "trusted.name"
diff --git a/tests/bugs/bug-913051.t b/tests/bugs/bug-913051.t
index 69e90cf66c2..9a59424f412 100644
--- a/tests/bugs/bug-913051.t
+++ b/tests/bugs/bug-913051.t
@@ -48,8 +48,8 @@ EXPECT "N" gf_check_file_opened_in_brick $V0 $H0 $B0/${V0}0 $B0/${V0}0/dir/b
 
 #attempt self-heal so that the files are created on brick-0
 
-TEST ls -l $M0/dir/a
-TEST ls -l $M0/dir/b
+TEST dd if=$M0/dir/a of=/dev/null bs=1M
+TEST dd if=$M0/dir/b of=/dev/null bs=1M
 
 #trigger writev for attempting open-fd-fix in afr
 TEST fd_write $wfd "open sesame"
diff --git a/tests/bugs/bug-913544.t b/tests/bugs/bug-913544.t
index 790bc08980f..db28ca814ce 100644
--- a/tests/bugs/bug-913544.t
+++ b/tests/bugs/bug-913544.t
@@ -17,7 +17,7 @@ TEST touch a
 #simulate no-changelog data split-brain
 echo "abc" > $B0/${V0}1/a
 echo "abcd" > $B0/${V0}0/a
-TEST ! truncate -s 0 a
+TEST truncate -s 0 a
 TEST ls
 cd
 
diff --git a/tests/bugs/bug-918437-sh-mtime.t b/tests/bugs/bug-918437-sh-mtime.t
index 080956f519a..11155ad1629 100644
--- a/tests/bugs/bug-918437-sh-mtime.t
+++ b/tests/bugs/bug-918437-sh-mtime.t
@@ -38,7 +38,12 @@ TEST $CLI volume start $V0 force
 EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
 EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
 
-find $M0 | xargs stat 1>/dev/null
+TEST $CLI volume set $V0 cluster.self-heal-daemon on
+sleep 1
+TEST gluster volume heal $V0 full
+
+size=`stat -c '%s' /etc/passwd`
+EXPECT_WITHIN 60 $size stat -c '%s' $B0/gfs0/brick01/a
 
 TEST modify_atstamp1=$(get_mtime $B0/gfs0/brick01/a)
 TEST modify_atstamp2=$(get_mtime $B0/gfs0/brick02/a)
diff --git a/tests/bugs/bug-977797.t b/tests/bugs/bug-977797.t
index 08cdbe8f119..f2252159a21 100755
--- a/tests/bugs/bug-977797.t
+++ b/tests/bugs/bug-977797.t
@@ -54,7 +54,7 @@ TEST chmod 757 $M0/a/file
 TEST $CLI volume start $V0 force
 EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1;
 
-TEST ls   -l $M0/a/file
+TEST dd if=$M0/a/file of=/dev/null bs=1M
 
 b1c0dir=$(afr_get_specific_changelog_xattr $B0/$V0"1"/a \
           trusted.afr.$V0-client-0 "entry")
@@ -75,34 +75,15 @@ b2c0f=$(afr_get_specific_changelog_xattr $B0/$V0"2"/a/file \
 b2c1f=$(afr_get_specific_changelog_xattr $B0/$V0"2"/a/file \
         trusted.afr.$V0-client-1 "data")
 
-EXPECT "00000000" echo $b1c0f
-EXPECT "00000000" echo $b1c1f
-EXPECT "00000000" echo $b2c0f
-EXPECT "00000000" echo $b2c1f
-
-EXPECT "00000000" echo $b1c0dir
-EXPECT "00000000" echo $b1c1dir
-EXPECT "00000000" echo $b2c0dir
-EXPECT "00000000" echo $b2c1dir
-
-contains() {
-    string="$1"
-    substring="$2"
-    var="-1"
-    if test "${string#*$substring}" != "$string"
-    then
-        var="0"    # $substring is in $string
-    else
-        var="1"    # $substring is not in $string
-    fi
-    echo $var
-}
-
-var1=$(cat $M0/a/file 2>&1)
-var2="Input/output error"
-
-
-EXPECT "0" contains "$var1" "$var2"
+EXPECT "00000000|^$" echo $b1c0f
+EXPECT "00000000|^$" echo $b1c1f
+EXPECT "00000000|^$" echo $b2c0f
+EXPECT "00000000|^$" echo $b2c1f
+
+EXPECT "00000000|^$" echo $b1c0dir
+EXPECT "00000000|^$" echo $b1c1dir
+EXPECT "00000000|^$" echo $b2c0dir
+EXPECT "00000000|^$" echo $b2c1dir
 
 ## Finish up
 TEST $CLI volume stop $V0;
diff --git a/tests/volume.rc b/tests/volume.rc
index 5e2f95e766d..9a06687cd86 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -169,7 +169,7 @@ function check_option_help_presence {
 function afr_get_changelog_xattr {
         local file=$1
         local xkey=$2
-        getfattr -n $xkey -e hex $file 2>/dev/null | grep "client-" | cut -f2 -d'='
+        getfattr -n $xkey -e hex $file 2>/dev/null | grep "$xkey" | cut -f2 -d'='
 }
 
 function afr_get_pending_heal_count {