1 files changed, 57 insertions, 34 deletions
diff --git a/tests/bugs/bug-873962.t b/tests/bugs/bug-873962.t
index c18c71f60..b245cc3da 100755
--- a/tests/bugs/bug-873962.t
+++ b/tests/bugs/bug-873962.t
@@ -2,6 +2,7 @@
 
 #AFR TEST-IDENTIFIER SPLIT-BRAIN
 . $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
 
 cleanup;
 
@@ -12,74 +13,96 @@ TEST $CLI volume info;
 B0_hiphenated=`echo $B0 | tr '/' '-'`
 TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2}
 
+# If we allow self-heal to happen in the background, we'll get spurious
+# failures - especially at the point labeled "FAIL HERE" but
+# occasionally elsewhere.  This behavior is very timing-dependent.  It
+# doesn't show up in Jenkins, but it does on JD's and KP's machines, and
+# it got sharply worse because of an unrelated fsync change (6ae6f3d)
+# which changed timing.  Putting anything at the FAIL HERE marker tends
+# to make it go away most of the time on affected machines, even if the
+# "anything" is unrelated.
+#
+# What's going on is that the I/O on the first mountpoint is allowed to
+# complete even though self-heal is still in progress and the state on
+# disk does not reflect its result.  In fact, the state changes during
+# self-heal create the appearance of split brain when the second I/O
+# comes in, so that fails even though we haven't actually been in split
+# brain since the manual xattr operations.  By disallowing background
+# self-heal, we ensure that the second I/O can't happen before self-heal
+# is complete, because it has to follow the first I/O which now has to
+# follow self-heal.
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+
 #Make sure self-heal is not triggered when the bricks are re-started
 TEST $CLI volume set $V0 cluster.self-heal-daemon off
 TEST $CLI volume set $V0 performance.stat-prefetch off
 TEST $CLI volume start $V0
-TEST glusterfs -s $H0 --volfile-id=$V0 $M0
-TEST glusterfs -s $H0 --volfile-id=$V0 $M1
-TEST cd $M0
-TEST touch a
-TEST touch b
-TEST touch c
-TEST touch d
-echo "1" > b
-echo "1" > d
-kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0$B0_hiphenated-${V0}2.pid`
-echo "1" > a
-echo "1" > c
-TEST setfattr -n trusted.mdata -v abc b
-TEST setfattr -n trusted.mdata -v abc d
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=enable
+TEST touch $M0/a
+TEST touch $M0/b
+TEST touch $M0/c
+TEST touch $M0/d
+echo "1" > $M0/b
+echo "1" > $M0/d
+TEST kill_brick $V0 $H0 $B0/${V0}2
+echo "1" > $M0/a
+echo "1" > $M0/c
+TEST setfattr -n trusted.mdata -v abc $M0/b
+TEST setfattr -n trusted.mdata -v abc $M0/d
 TEST $CLI volume start $V0 force
-sleep 5
-kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0$B0_hiphenated-${V0}1.pid`
-echo "2" > a
-echo "2" > c
-TEST setfattr -n trusted.mdata -v def b
-TEST setfattr -n trusted.mdata -v def d
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
+TEST kill_brick $V0 $H0 $B0/${V0}1
+echo "2" > $M0/a
+echo "2" > $M0/c
+TEST setfattr -n trusted.mdata -v def $M0/b
+TEST setfattr -n trusted.mdata -v def $M0/d
 TEST $CLI volume start $V0 force
-sleep 10
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
 
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M1 --direct-io-mode=enable
 #Files are in split-brain, so open should fail
-TEST ! cat a;
+TEST ! cat $M0/a;
 TEST ! cat $M1/a;
-TEST ! cat b;
+TEST ! cat $M0/b;
 TEST ! cat $M1/b;
 
 #Reset split-brain status
 TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/a;
 TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/b;
 
-sleep 1
 #The operations should do self-heal and give correct output
-EXPECT "2" cat a;
+EXPECT "2" cat $M0/a;
+# FAIL HERE - see comment about cluster.self-heal-background-count above.
 EXPECT "2" cat $M1/a;
-EXPECT "def" getfattr -n trusted.mdata --only-values b 2>/dev/null
+EXPECT "def" getfattr -n trusted.mdata --only-values $M0/b 2>/dev/null
 EXPECT "def" getfattr -n trusted.mdata --only-values $M1/b 2>/dev/null
 
+TEST umount $M0
+TEST umount $M1
+
 TEST $CLI volume set $V0 cluster.data-self-heal off
 TEST $CLI volume set $V0 cluster.metadata-self-heal off
-sleep 5 #Wait for vol file to be updated
 
-sleep 1
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=enable
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M1 --direct-io-mode=enable
+
 #Files are in split-brain, so open should fail
-TEST ! cat c
+TEST ! cat $M0/c
 TEST ! cat $M1/c
-TEST ! cat d
+TEST ! cat $M0/d
 TEST ! cat $M1/d
 
 TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/c
 TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/d
 
-sleep 1
 #The operations should NOT do self-heal but give correct output
-EXPECT "2" cat c
+EXPECT "2" cat $M0/c
 EXPECT "2" cat $M1/c
-EXPECT "1" cat d
+EXPECT "1" cat $M0/d
 EXPECT "1" cat $M1/d
 
 #Check that the self-heal is not triggered.
 EXPECT "1" cat $B0/${V0}1/c
 EXPECT "abc" getfattr -n trusted.mdata --only-values $B0/${V0}1/d 2>/dev/null
-cd
 cleanup;