test/ec: Fix spurious failures caused by self-heal

The sha1sum of a file may update the access time of that file.
If this happens while a brick is down, as it is forced in the
test, that brick doesn't get the update, getting out of sync.

When the brick is restarted, self-heal repairs the file, but
the test shouldn't access brick contents until self-heal finishes.
If this is combined with a kill of another brick before self-heal
has finished repairing the file, the volume could become inaccessible.

Since the purpose of these tests is only to check ec functionality
(there is another test that checks self-heal), the test that corrupts
the file has been removed.

Additional checks to validate the state of the volume have been added
to avoid some timing issues.

BUG: 1144108
Change-Id: Ibd9288de519914663998a1fbc4321ec92ed6082c
Signed-off-by: Xavier Hernandez <xhernandez@datalab.es>
Reviewed-on: http://review.gluster.org/8892
Reviewed-by: Emmanuel Dreyfus <manu@netbsd.org>
Tested-by: Emmanuel Dreyfus <manu@netbsd.org>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Dan Lambright <dlambrig@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 18 insertions, 19 deletions

diff --git a/tests/basic/ec/self-heal.t b/tests/basic/ec/self-heal.t
index 524c11d2220..a40193f7447 100644
--- a/tests/basic/ec/self-heal.t
+++ b/tests/basic/ec/self-heal.t
@@ -17,8 +17,11 @@ TESTS_EXPECTED_IN_LOOP=85
 TEST glusterd
 TEST pidof glusterd
 TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5}
+EXPECT "Created" volinfo_field $V0 'Status'
 TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status'
 TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0;
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0
 
 TEST dd if=/dev/urandom of=$tmp/test bs=1024 count=1024
 
@@ -40,7 +43,7 @@ for idx1 in {0..5}; do
     TEST chmod 666 ${brick[$idx1]}/test
     sleep 1
     EXPECT "-rw-r--r--" stat -c "%A" test
-    EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test
+    EXPECT_WITHIN $HEAL_TIMEOUT "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test
 done
 
 for idx1 in {0..4}; do
@@ -50,8 +53,8 @@ for idx1 in {0..4}; do
             TEST chmod 600 ${brick[$idx2]}/test
             sleep 1
             EXPECT "-rw-r--r--" stat -c "%A" test
-            EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test
-            EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx2]}/test
+            EXPECT_WITHIN $HEAL_TIMEOUT "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test
+            EXPECT_WITHIN $HEAL_TIMEOUT "-rw-r--r--" stat -c "%A" ${brick[$idx2]}/test
         fi
     done
 done
@@ -67,8 +70,8 @@ sleep 1
 EXPECT "1048576" stat -c "%s" test
 TEST ! getfattr -n user.test test
 
-EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[0]}/test
-EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[1]}/test
+EXPECT_WITHIN $HEAL_TIMEOUT "262144" stat -c "%s" ${brick[0]}/test
+EXPECT_WITHIN $HEAL_TIMEOUT "262144" stat -c "%s" ${brick[1]}/test
 TEST ! getfattr -n user.test ${brick[0]}/test
 TEST ! getfattr -n user.test ${brick[1]}/test
 EXPECT "-rw-r--r--" stat -c "%A" ${brick[0]}/test
@@ -92,31 +95,27 @@ TEST [ -f test4 ]
 EXPECT "2" stat -c "%h" test2
 EXPECT "2" stat -c "%h" test4
 
-cd
-TEST umount $M0
-TEST $CLI volume stop $V0 force
-TEST $CLI volume start $V0
-TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0;
-cd $M0
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0
 
 EXPECT "1048576" stat -c "%s" test2
 EXPECT "-rwxrwxrwx" stat -c "%A" test2
-EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[0]}/test2
-EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[1]}/test2
+EXPECT_WITHIN $HEAL_TIMEOUT "262144" stat -c "%s" ${brick[0]}/test2
+EXPECT_WITHIN $HEAL_TIMEOUT "262144" stat -c "%s" ${brick[1]}/test2
 EXPECT "-rwxrwxrwx" stat -c "%A" ${brick[0]}/test2
 EXPECT "-rwxrwxrwx" stat -c "%A" ${brick[1]}/test2
 
 TEST ls -al dir1
-EXPECT_WITHIN 5 "1" eval "if [ -d ${brick[0]}/dir1 ]; then echo 1; fi"
-EXPECT_WITHIN 5 "1" eval "if [ -d ${brick[1]}/dir1 ]; then echo 1; fi"
+EXPECT_WITHIN $HEAL_TIMEOUT "1" eval "if [ -d ${brick[0]}/dir1 ]; then echo 1; fi"
+EXPECT_WITHIN $HEAL_TIMEOUT "1" eval "if [ -d ${brick[1]}/dir1 ]; then echo 1; fi"
 
 TEST [ -h test3 ]
-EXPECT_WITHIN 5 "1" eval "if [ -h ${brick[0]}/test3 ]; then echo 1; fi"
-EXPECT_WITHIN 5 "1" eval "if [ -h ${brick[1]}/test3 ]; then echo 1; fi"
+EXPECT_WITHIN $HEAL_TIMEOUT "1" eval "if [ -h ${brick[0]}/test3 ]; then echo 1; fi"
+EXPECT_WITHIN $HEAL_TIMEOUT "1" eval "if [ -h ${brick[1]}/test3 ]; then echo 1; fi"
 
 EXPECT "2" stat -c "%h" test4
-EXPECT_WITHIN 5 "3" stat -c "%h" ${brick[0]}/test4
-EXPECT_WITHIN 5 "3" stat -c "%h" ${brick[1]}/test4
+EXPECT_WITHIN $HEAL_TIMEOUT "3" stat -c "%h" ${brick[0]}/test4
+EXPECT_WITHIN $HEAL_TIMEOUT "3" stat -c "%h" ${brick[1]}/test4
 
 TEST rm -rf $tmp