Fixes quota aux mount failure

The aux mount is created on the first limit/remove_limit/list command
and it remains until volume is stopped / deleted / (quota is disabled)
, where we do a lazy unmount. If the process is uncleanly terminated,
then the mount entry remains and we get (Transport disconnected) error
on subsequent attempts to run quota list/limit-usage/remove commands.

Second issue, There is also a risk of inadvertent rm -rf on the
/var/run/gluster causing data loss for the user. Ideally, /var/run is
a temp path for application use and should not cause any data loss to
persistent storage.

Solution:
1) unmount the aux mount after each use.
2) clean stale mount before mounting, if any.

One caveat with doing mount/unmount on each command is that we cannot
use same mount point for both list and limit commands.
The reason for this is that list command needs mount to be accessible
in cli after response from glusterd, So it could be unmounted by a
limit command if executed in parallel (had we used same mount point)
Hence we use separate mount points for list and limit commands.

> Reviewed-on: https://review.gluster.org/16938
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Manikandan Selvaganesh <manikandancs333@gmail.com>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
> Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
> (cherry picked from commit 2ae4b4058691b324535d802f4e6d24cce89a10e5)

Change-Id: I4f9e39da2ac2b65941399bffb6440db8a6ba59d0
BUG: 1449779
Signed-off-by: Sanoj Unnikrishnan <sunnikri@redhat.com>
Reviewed-on: https://review.gluster.org/17241
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra Talur <rtalur@redhat.com>

4 files changed, 48 insertions, 60 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index c1c95ae5170..0e6629cf0dd 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -703,11 +703,6 @@ glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr,
                 }
         }
 
-        //Remove aux mount of the volume on every node in the cluster
-        ret = glusterd_remove_auxiliary_mount (volinfo->volname);
-        if (ret)
-                goto out;
-
         *crawl = _gf_true;
 
         (void) glusterd_clean_up_quota_store (volinfo);
@@ -737,7 +732,7 @@ glusterd_set_quota_limit (char *volname, char *path, char *hard_limit,
         priv = this->private;
         GF_ASSERT (priv);
 
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (abspath, volname, path);
+        GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH (abspath, volname, path);
         ret = gf_lstat_dir (abspath, NULL);
         if (ret) {
                 gf_asprintf (op_errstr, "Failed to find the directory %s. "
@@ -1362,7 +1357,7 @@ glusterd_remove_quota_limit (char *volname, char *path, char **op_errstr,
         priv = this->private;
         GF_ASSERT (priv);
 
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (abspath, volname, path);
+        GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH (abspath, volname, path);
         ret = gf_lstat_dir (abspath, NULL);
         if (ret) {
                 gf_asprintf (op_errstr, "Failed to find the directory %s. "
@@ -1693,6 +1688,16 @@ glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 
         ret = 0;
 out:
+        if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
+            type == GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS ||
+            type == GF_QUOTA_OPTION_TYPE_REMOVE ||
+            type == GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS) {
+                /* During a list operation we need the aux mount to be
+                 * accessible until the listing is done at the cli
+                 */
+                glusterd_remove_auxiliary_mount (volinfo->volname);
+        }
+
         return ret;
 }
 
@@ -1851,7 +1856,7 @@ out:
 }
 
 static int
-glusterd_create_quota_auxiliary_mount (xlator_t *this, char *volname)
+glusterd_create_quota_auxiliary_mount (xlator_t *this, char *volname, int type)
 {
         int                ret                     = -1;
         char               mountdir[PATH_MAX]      = {0,};
@@ -1861,28 +1866,30 @@ glusterd_create_quota_auxiliary_mount (xlator_t *this, char *volname)
         char              *volfileserver           = NULL;
         glusterd_conf_t   *priv                    = NULL;
         struct stat        buf                     = {0,};
+        FILE              *file                    = NULL;
 
         GF_VALIDATE_OR_GOTO ("glusterd", this, out);
         priv = this->private;
         GF_VALIDATE_OR_GOTO (this->name, priv, out);
 
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile_path, volname);
 
-        if (gf_is_service_running (pidfile_path, NULL)) {
-                gf_msg_debug (this->name, 0, "Aux mount of volume %s is running"
-                              " already", volname);
-                ret = 0;
-                goto out;
+        if (type == GF_QUOTA_OPTION_TYPE_LIST ||
+            type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS) {
+                GLUSTERFS_GET_QUOTA_LIST_MOUNT_PIDFILE (pidfile_path, volname);
+                GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH (mountdir, volname, "/");
+        } else {
+                GLUSTERFS_GET_QUOTA_LIMIT_MOUNT_PIDFILE (pidfile_path, volname);
+                GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH (mountdir, volname, "/");
         }
 
-        if (glusterd_is_fuse_available () == _gf_false) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MOUNT_REQ_FAIL, "Fuse unavailable");
-                ret = -1;
-                goto out;
+        file = fopen (pidfile_path, "r");
+        if (file) {
+                /* Previous command did not clean up pid file.
+                 * remove aux mount if it exists*/
+                gf_umount_lazy (this->name, mountdir, 1);
+                fclose(file);
         }
 
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
         ret = sys_mkdir (mountdir, 0777);
         if (ret && errno != EEXIST) {
                 gf_msg (this->name, GF_LOG_ERROR, errno,
@@ -2036,7 +2043,7 @@ glusterd_op_stage_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                  */
                 if (is_origin_glusterd (dict)) {
                         ret = glusterd_create_quota_auxiliary_mount (this,
-                                                                     volname);
+                                                            volname, type);
                         if (ret) {
                                 *op_errstr = gf_strdup ("Failed to start aux "
                                                         "mount");
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 01c4f28c7a2..51db13df0f6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -11628,21 +11628,12 @@ glusterd_remove_auxiliary_mount (char *volname)
 {
         int       ret                = -1;
         char      mountdir[PATH_MAX] = {0,};
-        char      pidfile[PATH_MAX]  = {0,};
         xlator_t *this               = NULL;
 
         this = THIS;
         GF_ASSERT (this);
 
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile, volname);
-
-        if (!gf_is_service_running (pidfile, NULL)) {
-                gf_msg_debug (this->name, 0, "Aux mount of volume %s "
-                        "absent, hence returning", volname);
-                return 0;
-        }
-
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
+        GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH (mountdir, volname, "/");
         ret = gf_umount_lazy (this->name, mountdir, 1);
         if (ret) {
                 gf_msg (this->name, GF_LOG_ERROR, errno,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 7bd728b7a31..2fbd545304c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2680,8 +2680,6 @@ glusterd_stop_volume (glusterd_volinfo_t *volinfo)
 {
         int                     ret                     = -1;
         glusterd_brickinfo_t    *brickinfo              = NULL;
-        char                    mountdir[PATH_MAX]      = {0,};
-        char                    pidfile[PATH_MAX]       = {0,};
         xlator_t                *this                   = NULL;
         glusterd_svc_t          *svc                    = NULL;
 
@@ -2710,24 +2708,6 @@ glusterd_stop_volume (glusterd_volinfo_t *volinfo)
                 goto out;
         }
 
-        /* If quota auxiliary mount is present, unmount it */
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile, volinfo->volname);
-
-        if (!gf_is_service_running (pidfile, NULL)) {
-                gf_msg_debug (this->name, 0, "Aux mount of volume %s "
-                        "absent", volinfo->volname);
-        } else {
-                GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volinfo->volname,
-                                                   "/");
-
-                ret = gf_umount_lazy (this->name, mountdir, 0);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_UNOUNT_FAILED,
-                                "umount on %s failed",
-                                mountdir);
-        }
-
         if (!volinfo->is_snap_volume) {
                 svc = &(volinfo->snapd.svc);
                 ret = svc->manager (svc, volinfo, PROC_START_NO_WAIT);
@@ -2816,10 +2796,6 @@ glusterd_op_delete_volume (dict_t *dict)
                 goto out;
         }
 
-        ret = glusterd_remove_auxiliary_mount (volname);
-        if (ret)
-                goto out;
-
         if (glusterd_check_ganesha_export (volinfo)) {
                 ret = manage_export_config (volname, "off", NULL);
                 if (ret)
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 54758d3b660..f05fbbb0de5 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -623,9 +623,15 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
 #define GLUSTERD_GET_QUOTAD_DIR(path, priv) \
         snprintf (path, PATH_MAX, "%s/quotad", priv->workdir);
 
-#define GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH(abspath, volname, path)      \
-        snprintf (abspath, sizeof (abspath)-1,                          \
-                  DEFAULT_VAR_RUN_DIRECTORY"/%s%s", volname, path);
+#define GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH(abspath, volname, path) do {      \
+        snprintf (abspath, sizeof (abspath)-1,                                \
+                 DEFAULT_VAR_RUN_DIRECTORY"/%s_quota_limit%s", volname, path);\
+        } while (0)
+
+#define GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(abspath, volname, path) do {       \
+        snprintf (abspath, sizeof (abspath)-1,                                \
+                  DEFAULT_VAR_RUN_DIRECTORY"/%s_quota_list%s", volname, path);\
+        } while (0)
 
 #define GLUSTERD_GET_TMP_PATH(abspath, path) do {                       \
         snprintf (abspath, sizeof (abspath)-1,                          \
@@ -714,11 +720,19 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
                            uuid_utoa(MY_UUID));                         \
         } while (0)
 
-#define GLUSTERFS_GET_AUX_MOUNT_PIDFILE(pidfile, volname) {               \
+#define GLUSTERFS_GET_QUOTA_LIMIT_MOUNT_PIDFILE(pidfile, volname) {       \
                 snprintf (pidfile, PATH_MAX-1,                            \
-                          DEFAULT_VAR_RUN_DIRECTORY"/%s.pid", volname);   \
+                          DEFAULT_VAR_RUN_DIRECTORY"/%s_quota_limit.pid", \
+                           volname);                                      \
         }
 
+#define GLUSTERFS_GET_QUOTA_LIST_MOUNT_PIDFILE(pidfile, volname) {    \
+                snprintf (pidfile, PATH_MAX-1,                            \
+                          DEFAULT_VAR_RUN_DIRECTORY"/%s_quota_list.pid",  \
+                          volname);                                       \
+        }
+
+
 #define GLUSTERD_GET_UUID_NOHYPHEN(ret_string, uuid) do {               \
                 char *snap_volname_ptr = ret_string;                    \
                 char  tmp_uuid[64];                                     \