quota: start aux mount from glusterd with inet address

With below patches, quota aux mount now
uses unix domain socket to connect to
glusterd
http://review.gluster.org/#/c/12645/
http://review.gluster.org/#/c/12819/

When USS is enabled, snapd protocol client tries to
connect to glusterd with inet and fails,
because remote-host option by client process
is set to UDS file

This patch starts the aux client process from glusterd
with inet address

Change-Id: I6967043bfd8824658ea39bfd2842591fcc3280fd
BUG: 1299497
Signed-off-by: vmallika <vmallika@redhat.com>
Reviewed-on: http://review.gluster.org/13255
Smoke: Gluster Build System <jenkins@build.gluster.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

5 files changed, 106 insertions, 161 deletions

diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 19ae35a90b3..71221055d58 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -1104,98 +1104,6 @@ out:
         return ret;
 }
 
-static int
-gf_cli_create_auxiliary_mount (char *volname)
-{
-        int      ret                     = -1;
-        char     mountdir[PATH_MAX]      = {0,};
-        char     pidfile_path[PATH_MAX]  = {0,};
-        char     logfile[PATH_MAX]       = {0,};
-        char     qpid [16]               = {0,};
-        char     *sockpath               = NULL;
-
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile_path, volname);
-
-        if (gf_is_service_running (pidfile_path, NULL)) {
-                gf_log ("cli", GF_LOG_DEBUG, "Aux mount of volume %s is running"
-                        " already", volname);
-                ret = 0;
-                goto out;
-        }
-
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
-        ret = sys_mkdir (mountdir, 0777);
-        if (ret && errno != EEXIST) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to create auxiliary mount "
-                        "directory %s. Reason : %s", mountdir,
-                        strerror (errno));
-                goto out;
-        }
-
-        snprintf (logfile, PATH_MAX-1, "%s/quota-mount-%s.log",
-                  DEFAULT_LOG_FILE_DIRECTORY, volname);
-        snprintf(qpid, 15, "%d", GF_CLIENT_PID_QUOTA_MOUNT);
-
-        if (global_state->glusterd_sock) {
-                sockpath = global_state->glusterd_sock;
-        } else {
-                sockpath = DEFAULT_GLUSTERD_SOCKFILE;
-        }
-
-        ret = runcmd (SBIN_DIR"/glusterfs",
-                      "--volfile-server", sockpath,
-                      "--volfile-server-transport", "unix",
-                      "--volfile-id", volname,
-                      "-l", logfile,
-                      "-p", pidfile_path,
-                      "--client-pid", qpid,
-                      mountdir,
-                      NULL);
-
-        if (ret) {
-                gf_log ("cli", GF_LOG_WARNING, "failed to mount glusterfs "
-                        "client. Please check the log file %s for more details",
-                        logfile);
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-static int
-cli_stage_quota_op (char *volname, int op_code)
-{
-        int ret = -1;
-
-        switch (op_code) {
-        case GF_QUOTA_OPTION_TYPE_ENABLE:
-        case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
-        case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
-        case GF_QUOTA_OPTION_TYPE_REMOVE:
-        case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
-        case GF_QUOTA_OPTION_TYPE_LIST:
-                ret = gf_cli_create_auxiliary_mount (volname);
-                if (ret) {
-                        cli_err ("quota: Could not start quota "
-                                 "auxiliary mount");
-                        goto out;
-                }
-                ret = 0;
-                break;
-
-        default:
-                ret = 0;
-                break;
-        }
-
-out:
-        return ret;
-}
-
 int
 cli_get_soft_limit (dict_t *options, const char **words, dict_t *xdata)
 {
@@ -1297,42 +1205,6 @@ out:
         return limits_set;
 }
 
-/* Checks if the mount is connected to the bricks
- *
- * Returns true if connected and false if not
- */
-gf_boolean_t
-_quota_aux_mount_online (char *volname)
-{
-        int         ret = 0;
-        char        mount_path[PATH_MAX + 1] = {0,};
-        struct stat buf = {0,};
-
-        GF_ASSERT (volname);
-
-        /* Try to create the aux mount before checking if bricks are online */
-        ret = gf_cli_create_auxiliary_mount (volname);
-        if (ret) {
-                cli_err ("quota: Could not start quota auxiliary mount");
-                return _gf_false;
-        }
-
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mount_path, volname, "/");
-
-        ret = sys_stat (mount_path, &buf);
-        if (ret) {
-                if (ENOTCONN == errno) {
-                        cli_err ("quota: Cannot connect to bricks. Check if "
-                                 "bricks are online.");
-                } else {
-                        cli_err ("quota: Error on quota auxiliary mount (%s).",
-                                 strerror (errno));
-                }
-                return _gf_false;
-        }
-        return _gf_true;
-}
-
 int
 cli_cmd_quota_handle_list_all (const char **words, dict_t *options)
 {
@@ -1403,12 +1275,6 @@ cli_cmd_quota_handle_list_all (const char **words, dict_t *options)
                 goto out;
         }
 
-        /* Check if the mount is online before doing any listing */
-        if (!_quota_aux_mount_online (volname)) {
-                ret = -1;
-                goto out;
-        }
-
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame) {
                 ret = -1;
@@ -1642,11 +1508,6 @@ cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
                 goto out;
         }
 
-        //create auxiliary mount need for quota commands that operate on path
-        ret = cli_stage_quota_op (volname, type);
-        if (ret)
-                goto out;
-
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame) {
                 ret = -1;
diff --git a/cli/src/cli-cmd.h b/cli/src/cli-cmd.h
index 2f96bdda2ff..54e3686c9e1 100644
--- a/cli/src/cli-cmd.h
+++ b/cli/src/cli-cmd.h
@@ -116,7 +116,4 @@ int cli_cmd_sent_status_get (int *status);
 gf_boolean_t
 _limits_set_on_volume (char *volname, int type);
 
-gf_boolean_t
-_quota_aux_mount_online (char *volname);
-
 #endif /* __CLI_CMD_H__ */
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 62a113a4dd8..4684679e557 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -3290,12 +3290,6 @@ gf_cli_print_limit_list_from_dict (cli_local_t *local, char *volname,
                 goto out;
         }
 
-        /* Check if the mount is online before doing any listing */
-        if (!_quota_aux_mount_online (volname)) {
-                ret = -1;
-                goto out;
-        }
-
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_vol_quota_limit_list_begin
                                 (local, op_ret, op_errno, op_errstr);
diff --git a/cli/src/cli.h b/cli/src/cli.h
index 95593e6313f..999772d4bc4 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -66,11 +66,6 @@ typedef enum {
         snprintf (abspath, sizeof (abspath)-1,                          \
                   DEFAULT_VAR_RUN_DIRECTORY"/%s%s", volname, path);
 
-#define GLUSTERFS_GET_AUX_MOUNT_PIDFILE(pidfile,volname) {               \
-                snprintf (pidfile, PATH_MAX-1,                             \
-                          DEFAULT_VAR_RUN_DIRECTORY"/%s.pid", volname);  \
-        }
-
 struct cli_state;
 struct cli_cmd_word;
 struct cli_cmd_tree;
diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index 73018ad44dc..f6277a382d8 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -1700,6 +1700,94 @@ out:
         return ret;
 }
 
+static int
+glusterd_create_quota_auxiliary_mount (xlator_t *this, char *volname)
+{
+        int                ret                     = -1;
+        int                retry                   = 0;
+        char               mountdir[PATH_MAX]      = {0,};
+        char               pidfile_path[PATH_MAX]  = {0,};
+        char               logfile[PATH_MAX]       = {0,};
+        char               qpid[16]                = {0,};
+        char              *volfileserver           = NULL;
+        glusterd_conf_t   *priv                    = NULL;
+        struct stat        buf                     = {0,};
+
+        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+        priv = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, priv, out);
+
+        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile_path, volname);
+
+        if (gf_is_service_running (pidfile_path, NULL)) {
+                gf_msg_debug (this->name, 0, "Aux mount of volume %s is running"
+                              " already", volname);
+                ret = 0;
+                goto out;
+        }
+
+        if (glusterd_is_fuse_available () == _gf_false) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_MOUNT_REQ_FAIL, "Fuse unavailable");
+                ret = -1;
+                goto out;
+        }
+
+        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
+        ret = sys_mkdir (mountdir, 0777);
+        if (ret && errno != EEXIST) {
+                gf_msg (this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_MOUNT_REQ_FAIL, "Failed to create auxiliary "
+                        "mount directory %s", mountdir);
+                goto out;
+        }
+        snprintf (logfile, PATH_MAX-1, "%s/quota-mount-%s.log",
+                  DEFAULT_LOG_FILE_DIRECTORY, volname);
+        snprintf(qpid, 15, "%d", GF_CLIENT_PID_QUOTA_MOUNT);
+
+        if (dict_get_str (this->options, "transport.socket.bind-address",
+                          &volfileserver) != 0)
+                volfileserver = "localhost";
+
+        synclock_unlock (&priv->big_lock);
+        ret = runcmd (SBIN_DIR"/glusterfs",
+                      "--volfile-server", volfileserver,
+                      "--volfile-id", volname,
+                      "-l", logfile,
+                      "-p", pidfile_path,
+                      "--client-pid", qpid,
+                      mountdir,
+                      NULL);
+        if (ret == 0) {
+                /* Block here till mount process is ready to accept FOPs.
+                 * Else, if glusterd acquires biglock below before
+                 * mount process is ready, then glusterd and mount process
+                 * can get into a deadlock situation.
+                 */
+                ret = sys_stat (mountdir, &buf);
+                if (ret < 0)
+                        ret = -errno;
+        } else {
+                ret = -errno;
+        }
+
+        synclock_lock (&priv->big_lock);
+
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, -ret,
+                        GD_MSG_MOUNT_REQ_FAIL, "Failed to mount glusterfs "
+                        "client. Please check the log file %s for more details",
+                        logfile);
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
 int
 glusterd_op_stage_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
@@ -1787,19 +1875,29 @@ glusterd_op_stage_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         }
 
         switch (type) {
-        case GF_QUOTA_OPTION_TYPE_ENABLE:
-        case GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS:
         case GF_QUOTA_OPTION_TYPE_LIST:
         case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
-                /* Fuse mount req. only for enable & list-usage options*/
-                if (is_origin_glusterd (dict) &&
-                    !glusterd_is_fuse_available ()) {
-                        *op_errstr = gf_strdup ("Fuse unavailable");
-                        ret = -1;
-                        goto out;
+        case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+        case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
+        case GF_QUOTA_OPTION_TYPE_REMOVE:
+        case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
+                /* Quota auxiliary mount is needed by CLI
+                 * for list command and need by glusterd for
+                 * setting/removing limit
+                 */
+                if (is_origin_glusterd (dict)) {
+                        ret = glusterd_create_quota_auxiliary_mount (this,
+                                                                     volname);
+                        if (ret) {
+                                *op_errstr = gf_strdup ("Failed to start aux "
+                                                        "mount");
+                                goto out;
+                        }
                 }
                 break;
+        }
 
+        switch (type) {
         case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
                 ret = dict_get_str (dict, "hard-limit", &hard_limit_str);
                 if (ret) {