Fixes quota aux mount failure

The aux mount is created on the first limit/remove_limit/list command
and it remains until volume is stopped / deleted / (quota is disabled)
, where we do a lazy unmount. If the process is uncleanly terminated,
then the mount entry remains and we get (Transport disconnected) error
on subsequent attempts to run quota list/limit-usage/remove commands.

Second issue, There is also a risk of inadvertent rm -rf on the
/var/run/gluster causing data loss for the user. Ideally, /var/run is
a temp path for application use and should not cause any data loss to
persistent storage.

Solution:
1) unmount the aux mount after each use.
2) clean stale mount before mounting, if any.

One caveat with doing mount/unmount on each command is that we cannot
use same mount point for both list and limit commands.
The reason for this is that list command needs mount to be accessible
in cli after response from glusterd, So it could be unmounted by a
limit command if executed in parallel (had we used same mount point)
Hence we use separate mount points for list and limit commands.

>Reviewed-on: https://review.gluster.org/16938
>NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
>Smoke: Gluster Build System <jenkins@build.gluster.org>
>Reviewed-by: Manikandan Selvaganesh <manikandancs333@gmail.com>
>CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
>Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
>Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
>(cherry picked from commit 2ae4b4058691b324535d802f4e6d24cce89a10e5)

Change-Id: I4f9e39da2ac2b65941399bffb6440db8a6ba59d0
BUG: 1449775
Signed-off-by: Sanoj Unnikrishnan <sunnikri@redhat.com>
Reviewed-on: https://review.gluster.org/17240
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

4 files changed, 48 insertions, 60 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index 1e62c9dfdeb..fc34042a8a7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -709,11 +709,6 @@ glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr,
                 }
         }
 
-        //Remove aux mount of the volume on every node in the cluster
-        ret = glusterd_remove_auxiliary_mount (volinfo->volname);
-        if (ret)
-                goto out;
-
         *crawl = _gf_true;
 
         (void) glusterd_clean_up_quota_store (volinfo);
@@ -743,7 +738,7 @@ glusterd_set_quota_limit (char *volname, char *path, char *hard_limit,
         priv = this->private;
         GF_ASSERT (priv);
 
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (abspath, volname, path);
+        GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH (abspath, volname, path);
         ret = gf_lstat_dir (abspath, NULL);
         if (ret) {
                 gf_asprintf (op_errstr, "Failed to find the directory %s. "
@@ -1373,7 +1368,7 @@ glusterd_remove_quota_limit (char *volname, char *path, char **op_errstr,
         priv = this->private;
         GF_ASSERT (priv);
 
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (abspath, volname, path);
+        GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH (abspath, volname, path);
         ret = gf_lstat_dir (abspath, NULL);
         if (ret) {
                 gf_asprintf (op_errstr, "Failed to find the directory %s. "
@@ -1704,6 +1699,16 @@ glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 
         ret = 0;
 out:
+        if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
+            type == GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS ||
+            type == GF_QUOTA_OPTION_TYPE_REMOVE ||
+            type == GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS) {
+                /* During a list operation we need the aux mount to be
+                 * accessible until the listing is done at the cli
+                 */
+                glusterd_remove_auxiliary_mount (volinfo->volname);
+        }
+
         return ret;
 }
 
@@ -1862,7 +1867,7 @@ out:
 }
 
 static int
-glusterd_create_quota_auxiliary_mount (xlator_t *this, char *volname)
+glusterd_create_quota_auxiliary_mount (xlator_t *this, char *volname, int type)
 {
         int                ret                     = -1;
         char               mountdir[PATH_MAX]      = {0,};
@@ -1872,28 +1877,30 @@ glusterd_create_quota_auxiliary_mount (xlator_t *this, char *volname)
         char              *volfileserver           = NULL;
         glusterd_conf_t   *priv                    = NULL;
         struct stat        buf                     = {0,};
+        FILE              *file                    = NULL;
 
         GF_VALIDATE_OR_GOTO ("glusterd", this, out);
         priv = this->private;
         GF_VALIDATE_OR_GOTO (this->name, priv, out);
 
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile_path, volname);
 
-        if (gf_is_service_running (pidfile_path, NULL)) {
-                gf_msg_debug (this->name, 0, "Aux mount of volume %s is running"
-                              " already", volname);
-                ret = 0;
-                goto out;
+        if (type == GF_QUOTA_OPTION_TYPE_LIST ||
+            type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS) {
+                GLUSTERFS_GET_QUOTA_LIST_MOUNT_PIDFILE (pidfile_path, volname);
+                GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH (mountdir, volname, "/");
+        } else {
+                GLUSTERFS_GET_QUOTA_LIMIT_MOUNT_PIDFILE (pidfile_path, volname);
+                GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH (mountdir, volname, "/");
         }
 
-        if (glusterd_is_fuse_available () == _gf_false) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MOUNT_REQ_FAIL, "Fuse unavailable");
-                ret = -1;
-                goto out;
+        file = fopen (pidfile_path, "r");
+        if (file) {
+                /* Previous command did not clean up pid file.
+                 * remove aux mount if it exists*/
+                gf_umount_lazy (this->name, mountdir, 1);
+                fclose(file);
         }
 
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
         ret = sys_mkdir (mountdir, 0777);
         if (ret && errno != EEXIST) {
                 gf_msg (this->name, GF_LOG_ERROR, errno,
@@ -2047,7 +2054,7 @@ glusterd_op_stage_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                  */
                 if (is_origin_glusterd (dict)) {
                         ret = glusterd_create_quota_auxiliary_mount (this,
-                                                                     volname);
+                                                            volname, type);
                         if (ret) {
                                 *op_errstr = gf_strdup ("Failed to start aux "
                                                         "mount");
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index b512e6673dd..a66e04934db 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -11657,21 +11657,12 @@ glusterd_remove_auxiliary_mount (char *volname)
 {
         int       ret                = -1;
         char      mountdir[PATH_MAX] = {0,};
-        char      pidfile[PATH_MAX]  = {0,};
         xlator_t *this               = NULL;
 
         this = THIS;
         GF_ASSERT (this);
 
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile, volname);
-
-        if (!gf_is_service_running (pidfile, NULL)) {
-                gf_msg_debug (this->name, 0, "Aux mount of volume %s "
-                        "absent, hence returning", volname);
-                return 0;
-        }
-
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
+        GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH (mountdir, volname, "/");
         ret = gf_umount_lazy (this->name, mountdir, 1);
         if (ret) {
                 gf_msg (this->name, GF_LOG_ERROR, errno,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 732374c11f5..95056d501a3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2648,8 +2648,6 @@ glusterd_stop_volume (glusterd_volinfo_t *volinfo)
 {
         int                     ret                     = -1;
         glusterd_brickinfo_t    *brickinfo              = NULL;
-        char                    mountdir[PATH_MAX]      = {0,};
-        char                    pidfile[PATH_MAX]       = {0,};
         xlator_t                *this                   = NULL;
         glusterd_svc_t          *svc                    = NULL;
 
@@ -2678,24 +2676,6 @@ glusterd_stop_volume (glusterd_volinfo_t *volinfo)
                 goto out;
         }
 
-        /* If quota auxiliary mount is present, unmount it */
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile, volinfo->volname);
-
-        if (!gf_is_service_running (pidfile, NULL)) {
-                gf_msg_debug (this->name, 0, "Aux mount of volume %s "
-                        "absent", volinfo->volname);
-        } else {
-                GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volinfo->volname,
-                                                   "/");
-
-                ret = gf_umount_lazy (this->name, mountdir, 0);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_UNOUNT_FAILED,
-                                "umount on %s failed",
-                                mountdir);
-        }
-
         if (!volinfo->is_snap_volume) {
                 svc = &(volinfo->snapd.svc);
                 ret = svc->manager (svc, volinfo, PROC_START_NO_WAIT);
@@ -2784,10 +2764,6 @@ glusterd_op_delete_volume (dict_t *dict)
                 goto out;
         }
 
-        ret = glusterd_remove_auxiliary_mount (volname);
-        if (ret)
-                goto out;
-
         ret = glusterd_delete_volume (volinfo);
 out:
         gf_msg_debug (this->name, 0, "returning %d", ret);
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index dff5ba15261..17e29bbbad3 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -620,9 +620,15 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
 #define GLUSTERD_GET_QUOTAD_DIR(path, priv) \
         snprintf (path, PATH_MAX, "%s/quotad", priv->workdir);
 
-#define GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH(abspath, volname, path)      \
-        snprintf (abspath, sizeof (abspath)-1,                          \
-                  DEFAULT_VAR_RUN_DIRECTORY"/%s%s", volname, path);
+#define GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH(abspath, volname, path) do {      \
+        snprintf (abspath, sizeof (abspath)-1,                                \
+                 DEFAULT_VAR_RUN_DIRECTORY"/%s_quota_limit%s", volname, path);\
+        } while (0)
+
+#define GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(abspath, volname, path) do {       \
+        snprintf (abspath, sizeof (abspath)-1,                                \
+                  DEFAULT_VAR_RUN_DIRECTORY"/%s_quota_list%s", volname, path);\
+        } while (0)
 
 #define GLUSTERD_GET_TMP_PATH(abspath, path) do {                       \
         snprintf (abspath, sizeof (abspath)-1,                          \
@@ -711,11 +717,19 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
                            uuid_utoa(MY_UUID));                         \
         } while (0)
 
-#define GLUSTERFS_GET_AUX_MOUNT_PIDFILE(pidfile, volname) {               \
+#define GLUSTERFS_GET_QUOTA_LIMIT_MOUNT_PIDFILE(pidfile, volname) {       \
                 snprintf (pidfile, PATH_MAX-1,                            \
-                          DEFAULT_VAR_RUN_DIRECTORY"/%s.pid", volname);   \
+                          DEFAULT_VAR_RUN_DIRECTORY"/%s_quota_limit.pid", \
+                           volname);                                      \
         }
 
+#define GLUSTERFS_GET_QUOTA_LIST_MOUNT_PIDFILE(pidfile, volname) {    \
+                snprintf (pidfile, PATH_MAX-1,                            \
+                          DEFAULT_VAR_RUN_DIRECTORY"/%s_quota_list.pid",  \
+                          volname);                                       \
+        }
+
+
 #define GLUSTERD_GET_UUID_NOHYPHEN(ret_string, uuid) do {               \
                 char *snap_volname_ptr = ret_string;                    \
                 char  tmp_uuid[64];                                     \