test/ec: Fix spurious failures caused by self-heal

The sha1sum of a file may update the access time of that file.
If this happens while a brick is down, as it is forced in the
test, that brick doesn't get the update, getting out of sync.

When the brick is restarted, self-heal repairs the file, but
the test shouldn't access brick contents until self-heal finishes.
If this is combined with a kill of another brick before self-heal
has finished repairing the file, the volume could become inaccessible.

Since the purpose of these tests is only to check ec functionality
(there is another test that checks self-heal), the test that corrupts
the file has been removed.

Additional checks to validate the state of the volume have been added
to avoid some timing issues.

This is a backport of http://review.gluster.org/8892/

BUG: 1149118
Change-Id: I8a40b7f07fc8ecd2c721bad1bcdd351dd8504155
Signed-off-by: Xavier Hernandez <xhernandez@datalab.es>
Reviewed-on: http://review.gluster.org/8902
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Dan Lambright <dlambrig@redhat.com>

12 files changed, 45 insertions, 41 deletions

diff --git a/tests/basic/ec/ec-12-4.t b/tests/basic/ec/ec-12-4.t
index 9ab47018617..76e6f8e77e8 100644
--- a/tests/basic/ec/ec-12-4.t
+++ b/tests/basic/ec/ec-12-4.t
@@ -8,7 +8,7 @@
 DISPERSE=12
 REDUNDANCY=4
 
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=634
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=541
 
 . $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-3-1.t b/tests/basic/ec/ec-3-1.t
index 5769c202289..511ca6420a2 100644
--- a/tests/basic/ec/ec-3-1.t
+++ b/tests/basic/ec/ec-3-1.t
@@ -8,7 +8,7 @@
 DISPERSE=3
 REDUNDANCY=1
 
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=238
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=217
 
 . $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-4-1.t b/tests/basic/ec/ec-4-1.t
index d34e1fb4e95..3f0d0c72e44 100644
--- a/tests/basic/ec/ec-4-1.t
+++ b/tests/basic/ec/ec-4-1.t
@@ -8,7 +8,7 @@
 DISPERSE=4
 REDUNDANCY=1
 
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=282
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=253
 
 . $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-5-1.t b/tests/basic/ec/ec-5-1.t
index 61d1cb6ce48..35c205da4b7 100644
--- a/tests/basic/ec/ec-5-1.t
+++ b/tests/basic/ec/ec-5-1.t
@@ -8,7 +8,7 @@
 DISPERSE=5
 REDUNDANCY=1
 
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=326
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=289
 
 . $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-5-2.t b/tests/basic/ec/ec-5-2.t
index 4dc1c186f02..6d9e91b0f58 100644
--- a/tests/basic/ec/ec-5-2.t
+++ b/tests/basic/ec/ec-5-2.t
@@ -8,7 +8,7 @@
 DISPERSE=5
 REDUNDANCY=2
 
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=326
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=289
 
 . $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-6-2.t b/tests/basic/ec/ec-6-2.t
index 23ec84e60e9..b4451f905a1 100644
--- a/tests/basic/ec/ec-6-2.t
+++ b/tests/basic/ec/ec-6-2.t
@@ -8,7 +8,7 @@
 DISPERSE=6
 REDUNDANCY=2
 
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=370
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=325
 
 . $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-7-3.t b/tests/basic/ec/ec-7-3.t
index 4ebba2a1de3..9d9d5f691bf 100644
--- a/tests/basic/ec/ec-7-3.t
+++ b/tests/basic/ec/ec-7-3.t
@@ -8,7 +8,7 @@
 DISPERSE=7
 REDUNDANCY=3
 
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=414
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=361
 
 . $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-common b/tests/basic/ec/ec-common
index 0fddd35ac14..65422d72095 100644
--- a/tests/basic/ec/ec-common
+++ b/tests/basic/ec/ec-common
@@ -24,8 +24,11 @@ fi
 TEST glusterd
 TEST pidof glusterd
 TEST $CLI volume create $V0 redundancy $REDUNDANCY $H0:$B0/${V0}{0..$LAST_BRICK}
+EXPECT 'Created' volinfo_field $V0 'Status'
 TEST $CLI volume start $V0
-TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status'
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 --volfile-id=/$V0 --volfile-server=$H0 $M0
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "$DISPERSE" ec_child_up_count $V0 0
 
 TEST dd if=/dev/urandom of=$tmp/small bs=1024 count=1
 TEST dd if=/dev/urandom of=$tmp/big bs=1024 count=4096
@@ -83,15 +86,9 @@ for dir in . dir1; do
 
         EXPECT "1024" stat -c "%s" $dir/small
         EXPECT "4194304" stat -c "%s" $dir/big
-        EXPECT "$cs_small" echo $(sha1sum $dir/small | awk '{ print $1 }')
-        EXPECT "$cs_big" echo $(sha1sum $dir/big | awk '{ print $1 }')
-
-        cd
-        TEST umount $M0
-        TEST $CLI volume stop $V0 force
-        TEST $CLI volume start $V0
-        TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
-        cd $M0
+
+        TEST $CLI volume start $V0 force
+        EXPECT_WITHIN $CHILD_UP_TIMEOUT "$DISPERSE" ec_child_up_count $V0 0
     done
 
     for size in $SIZE_LIST; do
diff --git a/tests/basic/ec/ec.t b/tests/basic/ec/ec.t
index e81de0d97bd..4c61f950fa5 100644
--- a/tests/basic/ec/ec.t
+++ b/tests/basic/ec/ec.t
@@ -137,10 +137,11 @@ EXPECT 'Created' volinfo_field $V0 'Status'
 EXPECT '10' brick_count $V0
 
 TEST $CLI volume start $V0
-EXPECT 'Started' volinfo_field $V0 'Status'
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status'
 
 # Mount FUSE with caching disabled
 TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "10" ec_child_up_count $V0 0
 
 # Create local files for comparisons etc.
 tmpdir=$(mktemp -d)
@@ -186,6 +187,7 @@ TEST setup_perm_file $M0
 # Unmount/remount so that create/write and truncate don't see cached data.
 TEST umount $M0
 TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "8" ec_child_up_count $V0 0
 
 # Test create/write and truncate *before* the bricks are brought back.
 TEST check_create_write $M0
@@ -193,11 +195,13 @@ TEST check_truncate $M0
 
 # Restart the bricks and allow repair to occur.
 TEST $CLI volume start $V0 force
-sleep 10
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status'
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "10" ec_child_up_count $V0 0
 
 # Unmount/remount again, same reason as before.
 TEST umount $M0
 TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "10" ec_child_up_count $V0 0
 
 # Make sure everything is as it should be.  Most tests check for consistency
 # between the bricks and the front end.  This is not valid for disperse, so we
@@ -217,14 +221,14 @@ TEST stat $M0/removexattr
 TEST stat $M0/perm_dir
 TEST stat $M0/perm_dir/perm_file
 
-EXPECT_WITHIN 5 "Y" check_hard_link $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_soft_link $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_unlink $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_rmdir $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_mkdir $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_setxattr $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_removexattr $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_perm_file $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_hard_link $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_soft_link $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_unlink $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_rmdir $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_mkdir $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_setxattr $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_removexattr $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_perm_file $B0/${V0}{0..9}
 
 rm -rf $tmpdir
 userdel --force ${TEST_USER}
diff --git a/tests/basic/ec/nfs.t b/tests/basic/ec/nfs.t
index 53ec30c92b0..f68568bc82f 100644
--- a/tests/basic/ec/nfs.t
+++ b/tests/basic/ec/nfs.t
@@ -8,7 +8,9 @@ cleanup
 TEST glusterd
 TEST pidof glusterd
 TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5}
+EXPECT "Created" volinfo_field $V0 'Status'
 TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status'
 
 EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available;
 TEST mount_nfs $H0:/$V0 $N0
diff --git a/tests/basic/ec/self-heal.t b/tests/basic/ec/self-heal.t
index b7f09a56cd6..9d5e5844062 100644
--- a/tests/basic/ec/self-heal.t
+++ b/tests/basic/ec/self-heal.t
@@ -17,8 +17,11 @@ TESTS_EXPECTED_IN_LOOP=250
 TEST glusterd
 TEST pidof glusterd
 TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5}
+EXPECT "Created" volinfo_field $V0 'Status'
 TEST $CLI volume start $V0
-TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status'
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 --volfile-id=/$V0 --volfile-server=$H0 $M0;
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0
 
 TEST dd if=/dev/urandom of=$tmp/test bs=1024 count=1024
 
@@ -91,12 +94,8 @@ TEST [ -f test4 ]
 EXPECT "2" stat -c "%h" test2
 EXPECT "2" stat -c "%h" test4
 
-cd
-TEST umount $M0
-TEST $CLI volume stop $V0 force
-TEST $CLI volume start $V0
-TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
-cd $M0
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0
 
 EXPECT "1048576" stat -c "%s" test2
 EXPECT "-rwxrwxrwx" stat -c "%A" test2
diff --git a/tests/basic/ec/statedump.t b/tests/basic/ec/statedump.t
index bff41ab333e..5f607c5d2b6 100644
--- a/tests/basic/ec/statedump.t
+++ b/tests/basic/ec/statedump.t
@@ -8,7 +8,9 @@ cleanup
 TEST glusterd
 TEST pidof glusterd
 TEST $CLI volume create $V0 redundancy 1 $H0:$B0/${V0}{0..2}
+EXPECT "Created" volinfo_field $V0 'Status'
 TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status'
 TEST glusterfs --entry-timeout=0 --attribute-timeout=0 --volfile-id=/$V0 --volfile-server=$H0 $M0
 
 EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0