Fixes quota aux mount failure

The aux mount is created on the first limit/remove_limit/list command
and it remains until volume is stopped / deleted / (quota is disabled)
, where we do a lazy unmount. If the process is uncleanly terminated,
then the mount entry remains and we get (Transport disconnected) error
on subsequent attempts to run quota list/limit-usage/remove commands.

Second issue, There is also a risk of inadvertent rm -rf on the
/var/run/gluster causing data loss for the user. Ideally, /var/run is
a temp path for application use and should not cause any data loss to
persistent storage.

Solution:
1) unmount the aux mount after each use.
2) clean stale mount before mounting, if any.

One caveat with doing mount/unmount on each command is that we cannot
use same mount point for both list and limit commands.
The reason for this is that list command needs mount to be accessible
in cli after response from glusterd, So it could be unmounted by a
limit command if executed in parallel (had we used same mount point)
Hence we use separate mount points for list and limit commands.

> Reviewed-on: https://review.gluster.org/16938
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Manikandan Selvaganesh <manikandancs333@gmail.com>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
> Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
> (cherry picked from commit 2ae4b4058691b324535d802f4e6d24cce89a10e5)

Change-Id: I4f9e39da2ac2b65941399bffb6440db8a6ba59d0
BUG: 1449779
Signed-off-by: Sanoj Unnikrishnan <sunnikri@redhat.com>
Reviewed-on: https://review.gluster.org/17241
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra Talur <rtalur@redhat.com>

15 files changed, 71 insertions, 19 deletions

diff --git a/tests/basic/ec/quota.t b/tests/basic/ec/quota.t
index b023240b87e..c9612c8b76a 100755
--- a/tests/basic/ec/quota.t
+++ b/tests/basic/ec/quota.t
@@ -40,7 +40,6 @@ EXPECT_WITHIN $MARKER_UPDATE_TIMEOUT "8.0MB" quotausage "/test"
 TEST rm $M0/test/file2.txt
 EXPECT_WITHIN $MARKER_UPDATE_TIMEOUT "0Bytes" quotausage "/test"
 TEST $CLI volume stop $V0
-EXPECT "1" get_aux
 
 rm -f $QDD
 cleanup;
diff --git a/tests/basic/quota-ancestry-building.t b/tests/basic/quota-ancestry-building.t
index f8c508261f9..fcb39ee31f5 100755
--- a/tests/basic/quota-ancestry-building.t
+++ b/tests/basic/quota-ancestry-building.t
@@ -65,7 +65,6 @@ exec 5>&-
 exec 6>&-
 
 TEST $CLI volume stop $V0
-EXPECT "1" get_aux
 
 rm -f $QDD
 cleanup;
diff --git a/tests/basic/quota-anon-fd-nfs.t b/tests/basic/quota-anon-fd-nfs.t
index ea07b529c5a..279700dea42 100755
--- a/tests/basic/quota-anon-fd-nfs.t
+++ b/tests/basic/quota-anon-fd-nfs.t
@@ -107,7 +107,6 @@ EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0
 sleep 3
 
 TEST $CLI volume stop $V0
-EXPECT "1" get_aux
 
 rm -f $QDD
 
diff --git a/tests/basic/quota-nfs.t b/tests/basic/quota-nfs.t
index 74fde400bd1..663a8da90ad 100755
--- a/tests/basic/quota-nfs.t
+++ b/tests/basic/quota-nfs.t
@@ -58,7 +58,6 @@ TEST rm -f $N0/$deep/newfile_2
 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" umount_nfs $N0
 
 TEST $CLI volume stop $V0
-EXPECT "1" get_aux
 
 rm -f $QDD
 cleanup;
diff --git a/tests/basic/quota.t b/tests/basic/quota.t
index 17d571060e1..7f8b21de6f8 100755
--- a/tests/basic/quota.t
+++ b/tests/basic/quota.t
@@ -40,12 +40,8 @@ EXPECT 'on' volinfo_field $V0 'features.quota'
 EXPECT 'on' volinfo_field $V0 'features.inode-quota'
 EXPECT 'on' volinfo_field $V0 'features.quota-deem-statfs'
 
-#Wait for the auxiliarymount to come up
-sleep 3
 
 TEST $CLI volume quota $V0 limit-usage /test_dir 100MB
-# Checking for auxiliary mount
-EXPECT "0"  get_aux
 
 TEST $CLI volume quota $V0 limit-usage /test_dir/in_test_dir 150MB
 
@@ -231,9 +227,7 @@ EXPECT 'off' volinfo_field $V0 'features.quota'
 EXPECT 'off' volinfo_field $V0 'features.inode-quota'
 EXPECT '' volinfo_field $V0 'features.quota-deem-statfs'
 
-# aux mount should be removed
 TEST $CLI volume stop $V0;
-EXPECT "1" get_aux
 
 rm -f $QDD
 cleanup;
diff --git a/tests/basic/quota_aux_mount.t b/tests/basic/quota_aux_mount.t
new file mode 100755
index 00000000000..78d7f47e373
--- /dev/null
+++ b/tests/basic/quota_aux_mount.t
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+##-------------------------------------------------------------
+## Tests to verify that aux mount is unmounted after each quota
+## command executes.
+##-------------------------------------------------------------
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2  $H0:$B0/${V0}{1,2,3,4};
+
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT '4' brick_count $V0
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+TEST $GFS -s $H0 --volfile-id $V0 $M0;
+
+TEST mkdir -p $M0/test_dir/
+
+TEST $CLI volume quota $V0 enable
+EXPECT 'on' volinfo_field $V0 'features.quota'
+EXPECT 'on' volinfo_field $V0 'features.inode-quota'
+
+TEST $CLI volume quota $V0 limit-usage /test_dir 150MB
+EXPECT "1"  get_limit_aux
+TEST $CLI volume quota $V0 limit-objects /test_dir 10
+EXPECT "1"  get_limit_aux
+EXPECT "150.0MB" quota_hard_limit "/test_dir";
+EXPECT "1"  get_list_aux
+EXPECT "10" quota_object_hard_limit "/test_dir";
+EXPECT "1"  get_list_aux
+
+TEST $CLI volume quota $V0 remove /test_dir/
+EXPECT "1"  get_limit_aux
+TEST $CLI volume quota $V0 remove-objects /test_dir
+EXPECT "1"  get_limit_aux
+
+TEST $CLI volume quota $V0 disable
+
+TEST $CLI volume stop $V0;
+
+cleanup;
+#G_TESTDEF_TEST_STATUS_NETBSD7=BAD_TEST,BUG=1447344
diff --git a/tests/bugs/cli/bug-1022905.t b/tests/bugs/cli/bug-1022905.t
index 1d8981e0e9c..ee629e970d9 100644
--- a/tests/bugs/cli/bug-1022905.t
+++ b/tests/bugs/cli/bug-1022905.t
@@ -32,7 +32,6 @@ TEST $CLI volume set $V0 diagnostics.client-log-level DEBUG
 TEST $CLI volume reset $V0 force;
 
 TEST $CLI volume stop $V0
-EXPECT "1" get_aux
 TEST $CLI volume delete $V0
 
 cleanup;
diff --git a/tests/bugs/distribute/bug-1099890.t b/tests/bugs/distribute/bug-1099890.t
index 40f70d4938b..1a19ba880c0 100644
--- a/tests/bugs/distribute/bug-1099890.t
+++ b/tests/bugs/distribute/bug-1099890.t
@@ -123,7 +123,6 @@ EXPECT "1" is_dht_linkfile "$B0/${V0}1/zz"
 
 force_umount $M0
 TEST $CLI volume stop $V0
-EXPECT "1" get_aux
 UMOUNT_LOOP ${B0}/${V0}{1,2}
 rm -f ${B0}/brick{1,2}
 
diff --git a/tests/bugs/distribute/bug-1161156.t b/tests/bugs/distribute/bug-1161156.t
index 44a234c60dc..fed90e7f478 100755
--- a/tests/bugs/distribute/bug-1161156.t
+++ b/tests/bugs/distribute/bug-1161156.t
@@ -50,7 +50,6 @@ TEST ! mv $N0/dir/newfile_3 $N0/newdir/
 
 umount_nfs $N0
 TEST $CLI volume stop $V0
-EXPECT "1" get_aux
 
 rm -f $QDD
 
diff --git a/tests/bugs/glusterd/bug-765230-remove-quota-related-option-after-disabling-quota.t b/tests/bugs/glusterd/bug-765230-remove-quota-related-option-after-disabling-quota.t
index 9fe55a3d9df..de48c091c7e 100755
--- a/tests/bugs/glusterd/bug-765230-remove-quota-related-option-after-disabling-quota.t
+++ b/tests/bugs/glusterd/bug-765230-remove-quota-related-option-after-disabling-quota.t
@@ -54,7 +54,6 @@ EXPECT '' volinfo_field $V0 'features.quota-deem-statfs'
 
 ## Finish up
 TEST $CLI volume stop $V0
-EXPECT "1" get_aux
 EXPECT 'Stopped' volinfo_field $V0 'Status';
 
 TEST $CLI volume delete $V0;
diff --git a/tests/bugs/glusterfs/bug-848251.t b/tests/bugs/glusterfs/bug-848251.t
index ed3caa34b01..69ffe680f7f 100644
--- a/tests/bugs/glusterfs/bug-848251.t
+++ b/tests/bugs/glusterfs/bug-848251.t
@@ -48,6 +48,5 @@ EXPECT "80%" quota_list
 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $MOUNTDIR
 TEST   rm -rf $MOUNTDIR
 TEST $CLI volume stop $V0
-EXPECT "1" get_aux
 
 cleanup;
diff --git a/tests/bugs/posix/bug-990028.t b/tests/bugs/posix/bug-990028.t
index d04bb2b4af1..c86421492cd 100755
--- a/tests/bugs/posix/bug-990028.t
+++ b/tests/bugs/posix/bug-990028.t
@@ -153,6 +153,5 @@ __init;
 links_in_same_directory;
 links_across_directories;
 TEST $CLI volume stop $V0
-EXPECT "1" get_aux
 
 cleanup
diff --git a/tests/bugs/quota/bug-1087198.t b/tests/bugs/quota/bug-1087198.t
index 0694b251d9f..95133085f13 100644
--- a/tests/bugs/quota/bug-1087198.t
+++ b/tests/bugs/quota/bug-1087198.t
@@ -78,7 +78,6 @@ TEST grep -e "\"Usage is above soft limit:.*used by /\"" -- $BRICK_LOG_DIR/*
 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0
 
 TEST $CLI volume stop $V0
-EXPECT "1" get_aux
 
 rm -f $QDD
 
diff --git a/tests/bugs/snapshot/bug-1202436-calculate-quota-cksum-during-snap-restore.t b/tests/bugs/snapshot/bug-1202436-calculate-quota-cksum-during-snap-restore.t
index 366937245f2..addc05917d8 100644
--- a/tests/bugs/snapshot/bug-1202436-calculate-quota-cksum-during-snap-restore.t
+++ b/tests/bugs/snapshot/bug-1202436-calculate-quota-cksum-during-snap-restore.t
@@ -27,7 +27,6 @@ EXPECT '1' get_snap_count CLI_1 $V0
 
 TEST $CLI_1 volume stop $V0
 EXPECT 'Stopped' volinfo_field $V0 'Status'
-EXPECT "1" get_aux
 
 TEST $CLI_1 snapshot restore $($CLI_1 snapshot list)
 EXPECT '0' get_snap_count CLI_1 $V0
diff --git a/tests/volume.rc b/tests/volume.rc
index 197002d82a4..ecc4602a960 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -643,8 +643,9 @@ function num_graphs
 function get_aux()
 {
 ##Check if a auxiliary mount is there
+local aux_suffix=$1
 local rundir=$(gluster --print-statedumpdir)
-local pidfile="${rundir}/${V0}.pid"
+local pidfile="${rundir}/${V0}$aux_suffix.pid"
 if [ -f $pidfile ];
 then
         local pid=$(cat ${rundir}/${V0}.pid)
@@ -661,6 +662,18 @@ else
 fi
 }
 
+function get_list_aux()
+{
+# check for quota list aux mount
+	get_aux "_quota_list"
+}
+
+function get_limit_aux()
+{
+# check for quota list aux mount
+	get_aux "_quota_limit"
+}
+
 function check_for_xattr {
         local xattr=$1
         local filepath=$2
@@ -745,6 +758,10 @@ function quota_hl_exceeded()
 
 }
 
+function quota_object_hard_limit()
+{
+        quota_object_list_field $1 2
+}
 
 function scrub_status()
 {