tests: Change umount with force_umount with 5 retries

Change-Id: I0e2dbdfd34080328dfa6b4eebef0366f2b0fcb04
BUG: 1092850
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/8117
Tested-by: Justin Clift <justin@gluster.org>
Reviewed-by: Harshavardhana <harsha@harshavardhana.net>
Tested-by: Harshavardhana <harsha@harshavardhana.net>
Reviewed-by: Sachin Pandit <spandit@redhat.com>
Reviewed-by: Prashanth Pai <ppai@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

11 files changed, 47 insertions, 46 deletions

diff --git a/tests/basic/afr/gfid-mismatch.t b/tests/basic/afr/gfid-mismatch.t
index 05f48d43a2c..29951354a04 100644
--- a/tests/basic/afr/gfid-mismatch.t
+++ b/tests/basic/afr/gfid-mismatch.t
@@ -2,6 +2,7 @@
 #Test that GFID mismatches result in EIO
 
 . $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
 cleanup;
 
 #Init
@@ -20,7 +21,7 @@ TEST setfattr -n trusted.gfid -v 0sBfz5vAdHTEK1GZ99qjqTIg== $B0/brick0/file
 TEST ! "find $M0/file | xargs stat"
 
 #Cleanup
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST $CLI volume stop $V0
 TEST $CLI volume delete $V0
 TEST rm -rf $B0/*
diff --git a/tests/basic/afr/read-subvol-data.t b/tests/basic/afr/read-subvol-data.t
index 25ae3e63c64..2fa20a2c181 100644
--- a/tests/basic/afr/read-subvol-data.t
+++ b/tests/basic/afr/read-subvol-data.t
@@ -27,7 +27,7 @@ TEST $CLI volume start $V0 force
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "10485760" echo `ls -l $M0/afr_success_5.txt | awk '{ print $5}'`
 
 #Cleanup
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST $CLI volume stop $V0
 TEST $CLI volume delete $V0
 TEST rm -rf $B0/*
diff --git a/tests/basic/afr/read-subvol-entry.t b/tests/basic/afr/read-subvol-entry.t
index 3e7ee3f35dd..f64276fabd4 100644
--- a/tests/basic/afr/read-subvol-entry.t
+++ b/tests/basic/afr/read-subvol-entry.t
@@ -29,7 +29,7 @@ TEST $CLI volume start $V0 force
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "ghi" echo `ls $M0/abc/def/`
 
 #Cleanup
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST $CLI volume stop $V0
 TEST $CLI volume delete $V0
 TEST rm -rf $B0/*
diff --git a/tests/basic/bd.t b/tests/basic/bd.t
index eb6305414f5..717daa5a54c 100755
--- a/tests/basic/bd.t
+++ b/tests/basic/bd.t
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 . $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
 
 function execute()
 {
@@ -106,7 +107,7 @@ TEST stat /dev/$V0/${gfid}
 sleep 1
 ## Check mounting
 TEST mount -o loop $M0/lv $M1
-umount $M1
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M1
 
 # Snapshot
 TEST touch $M0/lv_sn
@@ -123,7 +124,7 @@ TEST ! stat /dev/$V0/${gfid}
 
 rm $M0/* -f
 
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST $CLI volume stop ${V0}
 EXPECT 'Stopped' volinfo_field $V0 'Status';
 TEST $CLI volume delete ${V0}
diff --git a/tests/basic/cdc.t b/tests/basic/cdc.t
index 70d2171a84d..f7c117063f8 100755
--- a/tests/basic/cdc.t
+++ b/tests/basic/cdc.t
@@ -89,7 +89,7 @@ TEST test ${checksum[client-file]} = ${checksum[dump-file-readv]}
 
 ## Cleanup files and unmount
 TEST rm -f /tmp/cdc* $M0/cdc*
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 ## Stop the volume
 TEST $CLI volume stop $V0;
@@ -115,7 +115,7 @@ TEST ! test -e /tmp/cdcdump.gz
 
 ## Cleanup files and unmount
 TEST rm -f /tmp/cdc* $M0/cdc*
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 ## Reset the network.compression options
 TEST $CLI volume reset $V0 network.compression.debug
diff --git a/tests/basic/mount-options.disabled b/tests/basic/mount-options.disabled
index 86d945ac5b3..67f893ca488 100644
--- a/tests/basic/mount-options.disabled
+++ b/tests/basic/mount-options.disabled
@@ -16,97 +16,97 @@ TEST $CLI volume start $V0
 #being removed in future breaking backward-compatibility.
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --entry-timeout=0
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile=/var/lib/glusterd/vols/$V0/${V0}-fuse.vol $M0
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 --log-file=/tmp/a.txt --log-level=DEBUG $M0
 EXPECT_NOT "0" wc -l /tmp/a.txt
 TEST grep " D " /tmp/a.txt
 TEST rm -f /tmp/a.txt
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --acl
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --aux-gfid-mount
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --enable-ino32
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --fopen-keep-cache=yes
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --fopen-keep-cache=no
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --fopen-keep-cache=fail
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --mac-compat=yes
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --mac-compat=no
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --mac-compat=fail
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --use-readdirp=yes
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --use-readdirp=no
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --use-readdirp=fail
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --direct-io-mode=yes
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --direct-io-mode=no
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --direct-io-mode=fail
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --read-only
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --selinux
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --worm
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volfile-check
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --dump-fuse=/tmp/a.txt
 EXPECT "0" stat /tmp/a.txt
 TEST rm -f /tmp/a.txt
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --gid-timeout=0
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --gid-timeout=-1
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --gid-timeout=abc
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --background-qlen=16
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --background-qlen=abc
 
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --background-qlen=-1
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --congestion-threshold=12
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --congestion-threshold=abc
 
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --congestion=threshold=-1
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --negative-timeout=10
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --negative-timeout=abc
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --negative-timeout=-1
@@ -114,23 +114,23 @@ TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --negative-timeout=-
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --pid-file=/tmp/a.txt
 EXPECT_NOT "0" wc -l /tmp/a.txt
 TEST rm -f /tmp/a.txt
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volfile-server-port=24007
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volfile-server-port=2400
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volfile-server-transport=tcp
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volfile-server-transport=ib-verbs
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volfile-server-port=socket
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volume-name=$V0
-TEST umount -l $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volume-name=abcd
 
diff --git a/tests/basic/posixonly.t b/tests/basic/posixonly.t
index b9de317a4d0..4844818fcc3 100755
--- a/tests/basic/posixonly.t
+++ b/tests/basic/posixonly.t
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 . $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
 
 cleanup;
 
@@ -25,6 +26,6 @@ TEST chown 100:100 $M0/dirname/filename;
 TEST chown 100:100 $M0/dirname;
 TEST rm -rf $M0/filename $M0/dirname;
 
-TEST umount $M0;
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 cleanup;
diff --git a/tests/basic/pump.t b/tests/basic/pump.t
index 23bdc187d90..a2e4fad7470 100644
--- a/tests/basic/pump.t
+++ b/tests/basic/pump.t
@@ -20,7 +20,7 @@ do
         mkdir dir$i && cd dir$i
 done
 cd
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 start
 EXPECT_WITHIN 600 "Y" gd_is_replace_brick_completed $H0 $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1
 TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 commit
diff --git a/tests/basic/quota.t b/tests/basic/quota.t
index 60dd5549627..1f6febd73a5 100755
--- a/tests/basic/quota.t
+++ b/tests/basic/quota.t
@@ -183,7 +183,7 @@ TEST getfattr -d -m "trusted.glusterfs.quota.limit-set" -e hex \
 ## </Test quota functionality in add-brick senarios>
 ## -------------------------------------------------
 
-umount -l $N0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0
 
 TEST $CLI volume quota $V0 disable
 TEST $CLI volume stop $V0;
diff --git a/tests/basic/uss.t b/tests/basic/uss.t
index 23a6bf33125..004395660b5 100644
--- a/tests/basic/uss.t
+++ b/tests/basic/uss.t
@@ -44,7 +44,7 @@ TEST $CLI snapshot create snap4 $V0;
 
 TEST $CLI volume set $V0 features.uss enable;
 
-TEST umount $M0;
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0;
 
diff --git a/tests/basic/volume-snapshot.t b/tests/basic/volume-snapshot.t
index 21d6ea3b939..bb483850eeb 100755
--- a/tests/basic/volume-snapshot.t
+++ b/tests/basic/volume-snapshot.t
@@ -108,11 +108,9 @@ TEST $CLI_1 snapshot config $V0 snap-max-hard-limit 100
 TEST $CLI_1 snapshot config $V1 snap-max-hard-limit 100
 
 TEST glusterfs -s $H1 --volfile-id=/snaps/${V0}_snap/${V0} $M0
-sleep 2
-TEST umount -f $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST glusterfs -s $H2 --volfile-id=/snaps/${V1}_snap/${V1} $M0
-sleep 2
-TEST umount -f $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 #Clean up
 stop_force_volumes 2