summaryrefslogtreecommitdiffstats
path: root/tests/basic/ec
diff options
context:
space:
mode:
authorXavier Hernandez <xhernandez@datalab.es>2014-09-18 18:42:34 +0200
committerVijay Bellur <vbellur@redhat.com>2014-10-03 02:01:26 -0700
commita97ad9b69bb17f2351c59512fa9c6cb25d82b4da (patch)
treeb13239ea132df6efb4185a5dfd0cfea18dc0b28e /tests/basic/ec
parent7e76bd24305ee29506dd11be5a0691cfa65b6d0c (diff)
test/ec: Fix spurious failures caused by self-heal
The sha1sum of a file may update the access time of that file. If this happens while a brick is down, as it is forced in the test, that brick doesn't get the update, getting out of sync. When the brick is restarted, self-heal repairs the file, but the test shouldn't access brick contents until self-heal finishes. If this is combined with a kill of another brick before self-heal has finished repairing the file, the volume could become inaccessible. Since the purpose of these tests is only to check ec functionality (there is another test that checks self-heal), the test that corrupts the file has been removed. Additional checks to validate the state of the volume have been added to avoid some timing issues. BUG: 1144108 Change-Id: Ibd9288de519914663998a1fbc4321ec92ed6082c Signed-off-by: Xavier Hernandez <xhernandez@datalab.es> Reviewed-on: http://review.gluster.org/8892 Reviewed-by: Emmanuel Dreyfus <manu@netbsd.org> Tested-by: Emmanuel Dreyfus <manu@netbsd.org> Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Dan Lambright <dlambrig@redhat.com> Reviewed-by: Vijay Bellur <vbellur@redhat.com>
Diffstat (limited to 'tests/basic/ec')
-rw-r--r--tests/basic/ec/ec-12-4.t4
-rw-r--r--tests/basic/ec/ec-3-1.t4
-rw-r--r--tests/basic/ec/ec-4-1.t4
-rw-r--r--tests/basic/ec/ec-5-1.t4
-rw-r--r--tests/basic/ec/ec-5-2.t4
-rw-r--r--tests/basic/ec/ec-6-2.t4
-rw-r--r--tests/basic/ec/ec-7-3.t4
-rw-r--r--tests/basic/ec/ec-common17
-rw-r--r--tests/basic/ec/ec.t24
-rw-r--r--tests/basic/ec/nfs.t2
-rw-r--r--tests/basic/ec/self-heal.t37
-rw-r--r--tests/basic/ec/statedump.t2
12 files changed, 57 insertions, 53 deletions
diff --git a/tests/basic/ec/ec-12-4.t b/tests/basic/ec/ec-12-4.t
index 9ab47018617..76e6f8e77e8 100644
--- a/tests/basic/ec/ec-12-4.t
+++ b/tests/basic/ec/ec-12-4.t
@@ -8,7 +8,7 @@
DISPERSE=12
REDUNDANCY=4
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=634
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=541
. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-3-1.t b/tests/basic/ec/ec-3-1.t
index 5769c202289..511ca6420a2 100644
--- a/tests/basic/ec/ec-3-1.t
+++ b/tests/basic/ec/ec-3-1.t
@@ -8,7 +8,7 @@
DISPERSE=3
REDUNDANCY=1
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=238
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=217
. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-4-1.t b/tests/basic/ec/ec-4-1.t
index d34e1fb4e95..3f0d0c72e44 100644
--- a/tests/basic/ec/ec-4-1.t
+++ b/tests/basic/ec/ec-4-1.t
@@ -8,7 +8,7 @@
DISPERSE=4
REDUNDANCY=1
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=282
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=253
. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-5-1.t b/tests/basic/ec/ec-5-1.t
index 61d1cb6ce48..35c205da4b7 100644
--- a/tests/basic/ec/ec-5-1.t
+++ b/tests/basic/ec/ec-5-1.t
@@ -8,7 +8,7 @@
DISPERSE=5
REDUNDANCY=1
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=326
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=289
. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-5-2.t b/tests/basic/ec/ec-5-2.t
index 4dc1c186f02..6d9e91b0f58 100644
--- a/tests/basic/ec/ec-5-2.t
+++ b/tests/basic/ec/ec-5-2.t
@@ -8,7 +8,7 @@
DISPERSE=5
REDUNDANCY=2
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=326
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=289
. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-6-2.t b/tests/basic/ec/ec-6-2.t
index 23ec84e60e9..b4451f905a1 100644
--- a/tests/basic/ec/ec-6-2.t
+++ b/tests/basic/ec/ec-6-2.t
@@ -8,7 +8,7 @@
DISPERSE=6
REDUNDANCY=2
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=370
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=325
. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-7-3.t b/tests/basic/ec/ec-7-3.t
index 4ebba2a1de3..9d9d5f691bf 100644
--- a/tests/basic/ec/ec-7-3.t
+++ b/tests/basic/ec/ec-7-3.t
@@ -8,7 +8,7 @@
DISPERSE=7
REDUNDANCY=3
-# This must be equal to 44 * $DISPERSE + 106
-TESTS_EXPECTED_IN_LOOP=414
+# This must be equal to 36 * $DISPERSE + 109
+TESTS_EXPECTED_IN_LOOP=361
. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-common b/tests/basic/ec/ec-common
index 92e6499fa87..7abb4f2176d 100644
--- a/tests/basic/ec/ec-common
+++ b/tests/basic/ec/ec-common
@@ -23,8 +23,11 @@ fi
TEST glusterd
TEST pidof glusterd
TEST $CLI volume create $V0 redundancy $REDUNDANCY $H0:$B0/${V0}{0..$LAST_BRICK}
+EXPECT 'Created' volinfo_field $V0 'Status'
TEST $CLI volume start $V0
-TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0;
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status'
+TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "$DISPERSE" ec_child_up_count $V0 0
TEST dd if=/dev/urandom of=$tmp/small bs=1024 count=1
TEST dd if=/dev/urandom of=$tmp/big bs=1024 count=4096
@@ -82,15 +85,9 @@ for dir in . dir1; do
EXPECT "1024" stat -c "%s" $dir/small
EXPECT "4194304" stat -c "%s" $dir/big
- EXPECT "$cs_small" echo $(sha1sum $dir/small | awk '{ print $1 }')
- EXPECT "$cs_big" echo $(sha1sum $dir/big | awk '{ print $1 }')
-
- cd
- EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
- TEST $CLI volume stop $V0 force
- TEST $CLI volume start $V0
- TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0;
- cd $M0
+
+ TEST $CLI volume start $V0 force
+ EXPECT_WITHIN $CHILD_UP_TIMEOUT "$DISPERSE" ec_child_up_count $V0 0
done
for size in $SIZE_LIST; do
diff --git a/tests/basic/ec/ec.t b/tests/basic/ec/ec.t
index 864f9f72bad..c12aba3afe6 100644
--- a/tests/basic/ec/ec.t
+++ b/tests/basic/ec/ec.t
@@ -137,10 +137,11 @@ EXPECT 'Created' volinfo_field $V0 'Status'
EXPECT '10' brick_count $V0
TEST $CLI volume start $V0
-EXPECT 'Started' volinfo_field $V0 'Status'
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status'
# Mount FUSE with caching disabled
TEST $GFS -s $H0 --volfile-id $V0 $M0
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "10" ec_child_up_count $V0 0
# Create local files for comparisons etc.
tmpdir=$(mktemp -d -t ${0##*/}.XXXXXX)
@@ -186,6 +187,7 @@ TEST setup_perm_file $M0
# Unmount/remount so that create/write and truncate don't see cached data.
TEST umount $M0
TEST $GFS -s $H0 --volfile-id $V0 $M0
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "8" ec_child_up_count $V0 0
# Test create/write and truncate *before* the bricks are brought back.
TEST check_create_write $M0
@@ -193,11 +195,13 @@ TEST check_truncate $M0
# Restart the bricks and allow repair to occur.
TEST $CLI volume start $V0 force
-sleep 10
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status'
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "10" ec_child_up_count $V0 0
# Unmount/remount again, same reason as before.
TEST umount $M0
TEST $GFS -s $H0 --volfile-id $V0 $M0
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "10" ec_child_up_count $V0 0
# Make sure everything is as it should be. Most tests check for consistency
# between the bricks and the front end. This is not valid for disperse, so we
@@ -217,14 +221,14 @@ TEST stat $M0/removexattr
TEST stat $M0/perm_dir
TEST stat $M0/perm_dir/perm_file
-EXPECT_WITHIN 5 "Y" check_hard_link $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_soft_link $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_unlink $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_rmdir $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_mkdir $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_setxattr $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_removexattr $B0/${V0}{0..9}
-EXPECT_WITHIN 5 "Y" check_perm_file $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_hard_link $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_soft_link $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_unlink $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_rmdir $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_mkdir $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_setxattr $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_removexattr $B0/${V0}{0..9}
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_perm_file $B0/${V0}{0..9}
TEST rm -rf $tmpdir
TEST userdel --force ${TEST_USER}
diff --git a/tests/basic/ec/nfs.t b/tests/basic/ec/nfs.t
index ebc8f9fcbce..b826296d58b 100644
--- a/tests/basic/ec/nfs.t
+++ b/tests/basic/ec/nfs.t
@@ -8,7 +8,9 @@ cleanup
TEST glusterd
TEST pidof glusterd
TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5}
+EXPECT "Created" volinfo_field $V0 'Status'
TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status'
EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available;
TEST mount_nfs $H0:/$V0 $N0 nolock
diff --git a/tests/basic/ec/self-heal.t b/tests/basic/ec/self-heal.t
index 524c11d2220..a40193f7447 100644
--- a/tests/basic/ec/self-heal.t
+++ b/tests/basic/ec/self-heal.t
@@ -17,8 +17,11 @@ TESTS_EXPECTED_IN_LOOP=85
TEST glusterd
TEST pidof glusterd
TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5}
+EXPECT "Created" volinfo_field $V0 'Status'
TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status'
TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0;
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0
TEST dd if=/dev/urandom of=$tmp/test bs=1024 count=1024
@@ -40,7 +43,7 @@ for idx1 in {0..5}; do
TEST chmod 666 ${brick[$idx1]}/test
sleep 1
EXPECT "-rw-r--r--" stat -c "%A" test
- EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test
+ EXPECT_WITHIN $HEAL_TIMEOUT "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test
done
for idx1 in {0..4}; do
@@ -50,8 +53,8 @@ for idx1 in {0..4}; do
TEST chmod 600 ${brick[$idx2]}/test
sleep 1
EXPECT "-rw-r--r--" stat -c "%A" test
- EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test
- EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx2]}/test
+ EXPECT_WITHIN $HEAL_TIMEOUT "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test
+ EXPECT_WITHIN $HEAL_TIMEOUT "-rw-r--r--" stat -c "%A" ${brick[$idx2]}/test
fi
done
done
@@ -67,8 +70,8 @@ sleep 1
EXPECT "1048576" stat -c "%s" test
TEST ! getfattr -n user.test test
-EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[0]}/test
-EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[1]}/test
+EXPECT_WITHIN $HEAL_TIMEOUT "262144" stat -c "%s" ${brick[0]}/test
+EXPECT_WITHIN $HEAL_TIMEOUT "262144" stat -c "%s" ${brick[1]}/test
TEST ! getfattr -n user.test ${brick[0]}/test
TEST ! getfattr -n user.test ${brick[1]}/test
EXPECT "-rw-r--r--" stat -c "%A" ${brick[0]}/test
@@ -92,31 +95,27 @@ TEST [ -f test4 ]
EXPECT "2" stat -c "%h" test2
EXPECT "2" stat -c "%h" test4
-cd
-TEST umount $M0
-TEST $CLI volume stop $V0 force
-TEST $CLI volume start $V0
-TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0;
-cd $M0
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0
EXPECT "1048576" stat -c "%s" test2
EXPECT "-rwxrwxrwx" stat -c "%A" test2
-EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[0]}/test2
-EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[1]}/test2
+EXPECT_WITHIN $HEAL_TIMEOUT "262144" stat -c "%s" ${brick[0]}/test2
+EXPECT_WITHIN $HEAL_TIMEOUT "262144" stat -c "%s" ${brick[1]}/test2
EXPECT "-rwxrwxrwx" stat -c "%A" ${brick[0]}/test2
EXPECT "-rwxrwxrwx" stat -c "%A" ${brick[1]}/test2
TEST ls -al dir1
-EXPECT_WITHIN 5 "1" eval "if [ -d ${brick[0]}/dir1 ]; then echo 1; fi"
-EXPECT_WITHIN 5 "1" eval "if [ -d ${brick[1]}/dir1 ]; then echo 1; fi"
+EXPECT_WITHIN $HEAL_TIMEOUT "1" eval "if [ -d ${brick[0]}/dir1 ]; then echo 1; fi"
+EXPECT_WITHIN $HEAL_TIMEOUT "1" eval "if [ -d ${brick[1]}/dir1 ]; then echo 1; fi"
TEST [ -h test3 ]
-EXPECT_WITHIN 5 "1" eval "if [ -h ${brick[0]}/test3 ]; then echo 1; fi"
-EXPECT_WITHIN 5 "1" eval "if [ -h ${brick[1]}/test3 ]; then echo 1; fi"
+EXPECT_WITHIN $HEAL_TIMEOUT "1" eval "if [ -h ${brick[0]}/test3 ]; then echo 1; fi"
+EXPECT_WITHIN $HEAL_TIMEOUT "1" eval "if [ -h ${brick[1]}/test3 ]; then echo 1; fi"
EXPECT "2" stat -c "%h" test4
-EXPECT_WITHIN 5 "3" stat -c "%h" ${brick[0]}/test4
-EXPECT_WITHIN 5 "3" stat -c "%h" ${brick[1]}/test4
+EXPECT_WITHIN $HEAL_TIMEOUT "3" stat -c "%h" ${brick[0]}/test4
+EXPECT_WITHIN $HEAL_TIMEOUT "3" stat -c "%h" ${brick[1]}/test4
TEST rm -rf $tmp
diff --git a/tests/basic/ec/statedump.t b/tests/basic/ec/statedump.t
index 6881c7f8c27..8d311ec3d6d 100644
--- a/tests/basic/ec/statedump.t
+++ b/tests/basic/ec/statedump.t
@@ -8,7 +8,9 @@ cleanup
TEST glusterd
TEST pidof glusterd
TEST $CLI volume create $V0 redundancy 1 $H0:$B0/${V0}{0..2}
+EXPECT "Created" volinfo_field $V0 'Status'
TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status'
TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0
EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0