Tests: fix spurious failures in trash.t

Fix timing problems that cause rare spurious failures in trash.t

BUG: 1129939
Change-Id: I673e033b53b6b4bb993c22fadbdcee725b2c1e96
Signed-off-by: Emmanuel Dreyfus <manu@netbsd.org>
Reviewed-on: http://review.gluster.org/10360
Tested-by: NetBSD Build System
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 69 insertions, 46 deletions

diff --git a/tests/features/trash.t b/tests/features/trash.t
index f1dc1ab6086..268baac570f 100755
--- a/tests/features/trash.t
+++ b/tests/features/trash.t
@@ -6,17 +6,34 @@
 cleanup
 
 test_mount() {
-        glusterfs -s $H0 --volfile-id $V0 $M0 --attribute-timeout=0
+        volume=$1
+        mount=$2
+        test_dir=$3
+        RETVAL=0
+        glusterfs -s $H0 --volfile-id $volume $mount --attribute-timeout=0
+
+        if [ "x$test_dir" = "x" ] ; then return $RETVAL; fi
+
         timeout=0
         while [ $timeout -lt $PROCESS_UP_TIMEOUT ] ; do
                 timeout=$(( $timeout + 1 ))
-                test -d $M0/.trashcan && break
+                test -d $test_dir
+                RETVAL=$?
+                if [ $RETVAL -eq 0 ] ; then break ; fi
+                sleep 1
         done
+
+        return $RETVAL
 }
 
 start_vol() {
-        $CLI volume start $V0
-        test_mount
+        volume=$1
+        mount=$2
+        test_dir=$3
+        $CLI volume start $volume
+        test_mount $volume $mount $test_dir
+        RETVAL=$?
+        return $RETVAL
 }
 
 create_files() {
@@ -76,6 +93,11 @@ wildcard_not_exists() {
         if [ $? -eq 0 ]; then echo "Y"; else echo "N"; fi
 }
 
+heal_ready() {
+        $CLI volume heal $1 info | grep -q '^Brick'
+        if [ $? -eq 0 ]; then echo "Y"; else echo "N"; fi
+}
+
 # testing glusterd [1-3]
 TEST glusterd
 TEST pidof glusterd
@@ -89,141 +111,142 @@ EXPECT "$V0" volinfo_field $V0 'Volume Name'
 EXPECT 'Created' volinfo_field $V0 'Status'
 EXPECT '2' brick_count $V0
 
-# test without enabling trash translator [8-10]
-TEST $CLI volume start $V0
-TEST glusterfs -s $H0 --volfile-id $V0 $M0 --attribute-timeout=0
-TEST [ -d $M0/.trashcan ]
+# test without enabling trash translator [8]
+TEST start_vol $V0 $M0
 
-# test on enabling trash translator [11-12]
+# test on enabling trash translator [9-10]
 TEST $CLI volume set $V0 features.trash on
 EXPECT 'on' volinfo_field $V0 'features.trash'
 
-# files directly under mount point [13]
+# files directly under mount point [11]
 create_files $M0/file1 $M0/file2
 TEST file_exists $V0 file1 file2
 
-# perform unlink [14]
+# perform unlink [12]
 TEST unlink_op file1
 
-# perform truncate [15]
+# perform truncate [13]
 TEST truncate_op file2 4
 
-# create files directory hierarchy and check [16]
+# create files directory hierarchy and check [14]
 mkdir -p $M0/1/2/3
 create_files $M0/1/2/3/foo1 $M0/1/2/3/foo2
 TEST file_exists $V0 1/2/3/foo1 1/2/3/foo2
 
-# perform unlink [17]
+# perform unlink [15]
 TEST unlink_op 1/2/3/foo1
 
-# perform truncate [18]
+# perform truncate [16]
 TEST truncate_op 1/2/3/foo2 4
 
 # create a directory for eliminate pattern
 mkdir $M0/a
 
-# set the eliminate pattern [19-20]
+# set the eliminate pattern [17-18]
 TEST $CLI volume set $V0 features.trash-eliminate-path /a
 EXPECT '/a' volinfo_field $V0 'features.trash-eliminate-path'
 
-# create two files and check [21]
+# create two files and check [19]
 create_files $M0/a/test1 $M0/a/test2
 TEST file_exists $V0 a/test1 a/test2
 
-# remove from eliminate pattern [22]
+# remove from eliminate pattern [20]
 rm -f $M0/a/test1
 EXPECT "Y" wildcard_not_exists $M0/.trashcan/a/test1*
 
-# truncate from eliminate path [23-25]
+# truncate from eliminate path [21-23]
 truncate -s 2 $M0/a/test2
 TEST [ -e $M0/a/test2 ]
 TEST [ `ls -l $M0/a/test2 | awk '{print $5}'` -eq 2 ]
 EXPECT "Y" wildcard_not_exists $M0/.trashcan/a/test2*
 
-# set internal op on [26-27]
+# set internal op on [24-25]
 TEST $CLI volume set $V0 features.trash-internal-op on
 EXPECT 'on' volinfo_field $V0 'features.trash-internal-op'
 
-# again create two files and check [28]
+# again create two files and check [26]
 create_files $M0/inop1 $M0/inop2
 TEST file_exists $V0 inop1 inop2
 
-# perform unlink [29]
+# perform unlink [27]
 TEST unlink_op inop1
 
-# perform truncate [30]
+# perform truncate [28]
 TEST truncate_op inop2 4
 
-# remove one brick and restart the volume [31-34]
+# remove one brick and restart the volume [28-31]
 TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}2 force
 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST $CLI volume stop $V0
-TEST start_vol
+TEST start_vol $V0 $M0 $M0/.trashcan
 
-# again create two files and check [35]
+# again create two files and check [33]
 create_files $M0/rebal1 $M0/rebal2
 TEST file_exists $V0 rebal1 rebal2
 
-# add one brick [36-37]
+# add one brick [34-35]
 TEST $CLI volume add-brick $V0 $H0:$B0/${V0}3
 TEST [ -d $B0/${V0}3 ]
 
-# perform rebalance [38]
+# perform rebalance [36]
 TEST $CLI volume rebalance $V0 start force
-sleep 3
 
-# check whether rebalance was succesful [39-42]
-TEST [ -e $B0/${V0}3/rebal2 ]
-EXPECT "Y" wildcard_exists $B0/${V0}1/.trashcan/internal_op/rebal2*
+# check whether rebalance was succesful [37-40]
+EXPECT_WITHIN $REBALANCE_TIMEOUT "Y" wildcard_exists $B0/${V0}3/rebal2
+EXPECT_WITHIN $REBALANCE_TIMEOUT "Y" wildcard_exists $B0/${V0}1/.trashcan/internal_op/rebal2*
 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
-TEST $CLI volume stop $V0
+# force required in case rebalance is not over
+TEST $CLI volume stop $V0 force
 
-# create a replicated volume [43]
+# create a replicated volume [41]
 TEST $CLI volume create $V1 replica 2 $H0:$B0/${V1}{1,2}
 
-# checking volume status [44-47]
+# checking volume status [42-45]
 EXPECT "$V1" volinfo_field $V1 'Volume Name'
 EXPECT 'Replicate' volinfo_field $V1 'Type'
 EXPECT 'Created' volinfo_field $V1 'Status'
 EXPECT '2' brick_count $V1
 
-# enable trash with options and start the replicate volume by disabling automatic self-heal [48-52]
+# enable trash with options and start the replicate volume by disabling automatic self-heal [46-50]
 TEST $CLI volume set $V1 features.trash on
 TEST $CLI volume set $V1 features.trash-internal-op on
 EXPECT 'on' volinfo_field $V1 'features.trash'
 EXPECT 'on' volinfo_field $V1 'features.trash-internal-op'
-TEST $CLI volume start $V1
+TEST start_vol $V1 $M1 $M1/.trashcan
 
-# mount and check for trash directory [53]
-glusterfs -s $H0 --volfile-id $V1 $M1 --attribute-timeout=0
+# mount and check for trash directory [51]
 TEST [ -d $M1/.trashcan/internal_op ]
 
-# create a file and check [54]
+# create a file and check [52]
 touch $M1/self
 TEST [ -e $B0/${V1}1/self -a -e $B0/${V1}2/self ]
 
-# kill one brick and delete the file from mount point [55-56]
+# kill one brick and delete the file from mount point [53-54]
 kill_brick $V1 $H0 $B0/${V1}1
 EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "1" online_brick_count
 rm -f $M1/self
 EXPECT "Y" wildcard_exists $B0/${V1}2/.trashcan/self*
 
-# force start the volume and trigger the self-heal manually [57-59]
+# force start the volume and trigger the self-heal manually [55-59]
 TEST $CLI volume start $V1 force
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" online_brick_count
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status
+# volume heal one sometimes fail with "Launching heal operation to
+# perform index self heal on volume patchy1 has been unsuccessful"
+# Hence firt check heal is really functionnal.
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" heal_ready $V1
 TEST $CLI volume heal $V1
 
 # check for the removed file in trashcan [60]
 EXPECT_WITHIN $HEAL_TIMEOUT "Y" wildcard_exists $B0/${V1}1/.trashcan/internal_op/self*
 
-# check renaming of trash directory through cli [61-65]
+# check renaming of trash directory through cli [61-64]
 TEST $CLI volume set $V0 trash-dir abc
-TEST $CLI volume start $V0
-TEST glusterfs -s $H0 --volfile-id $V0 $M0 --attribute-timeout=0
+TEST start_vol $V0 $M0 $M0/abc
 TEST [ -e $M0/abc -a ! -e $M0/.trashcan ]
 EXPECT "Y" wildcard_exists $B0/${V0}1/abc/internal_op/rebal2*
 
-# ensure that rename and delete operation on trash directory fails [66-68]
+# ensure that rename and delete operation on trash directory fails [65-67]
 rm -rf $M0/abc/internal_op
 TEST [ -e $M0/abc/internal_op ]
 rm -rf $M0/abc/