fuse: allow requests during mount (needed for SELinux labels)

Resurrecting Jeff's commit:

  commit 7d0397c2144810c8a396e00187a6617873c94002
  Author: Jeff Darcy <jdarcy@redhat.com>

      fuse: allow requests during mount (needed for SELinux labels)

that was reverted as of:

  commit 4ab1c326f3862714b960302f06c6323d6291b695
  Author: Vijay Bellur <vijay@gluster.com>

      Revert "fuse: allow requests during mount (needed for SELinux labels)"

BUG: 811217
Change-Id: Ia1af402897e6a7290acf79617c34fdc804751729
Signed-off-by: Csaba Henk <csaba@redhat.com>
Reviewed-on: http://review.gluster.com/3452
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

5 files changed, 187 insertions, 40 deletions

diff --git a/contrib/fuse-include/fuse-mount.h b/contrib/fuse-include/fuse-mount.h
index 9f83faf02a0..7a3756d92b8 100644
--- a/contrib/fuse-include/fuse-mount.h
+++ b/contrib/fuse-include/fuse-mount.h
@@ -9,4 +9,4 @@
 
 void gf_fuse_unmount (const char *mountpoint, int fd);
 int gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
-                   pid_t *mtab_pid);
+                   pid_t *mtab_pid, int status_fd);
diff --git a/contrib/fuse-lib/mount.c b/contrib/fuse-lib/mount.c
index 800fd193e6c..8c5da3d619f 100644
--- a/contrib/fuse-lib/mount.c
+++ b/contrib/fuse-lib/mount.c
@@ -544,19 +544,25 @@ gf_fuse_unmount (const char *mountpoint, int fd)
 
 #ifndef FUSE_UTIL
 static int
-fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mtab_pid)
+fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mtab_pid, int in_fd, int status_fd)
 {
         int fd = -1, ret = -1;
         unsigned mounted = 0;
         char *mnt_param_mnt = NULL;
         char *fstype = "fuse.glusterfs";
         char *source = fsname;
+        pid_t mypid = -1;
 
-        fd = open ("/dev/fuse", O_RDWR);
-        if (fd == -1) {
-                GFFUSE_LOGERR ("cannot open /dev/fuse (%s)", strerror (errno));
-
-                return -1;
+        if (in_fd >= 0) {
+                fd = in_fd;
+        }
+        else {
+                fd = open ("/dev/fuse", O_RDWR);
+                if (fd == -1) {
+                        GFFUSE_LOGERR ("cannot open /dev/fuse (%s)",
+                                        strerror (errno));
+                        return -1;
+                }
         }
 
         ret = asprintf (&mnt_param_mnt,
@@ -567,8 +573,14 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
 
                 goto out;
         }
+        ret = fork();
+        if (ret != 0) {
+                goto parent_out;
+        }
+        GFFUSE_LOGERR("calling mount");
         ret = mount (source, mountpoint, fstype, 0,
                      mnt_param_mnt);
+        GFFUSE_LOGERR("mount returned %d",ret);
         if (ret == -1 && errno == ENODEV) {
                 /* fs subtype support was added by 79c0b2df aka
                    v2.6.21-3159-g79c0b2d. Probably we have an
@@ -607,11 +619,31 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
                 }
         }
 
- out:
+        ret = 0;
+out:
+        if (status_fd >= 0) {
+                GFFUSE_LOGERR("writing status");
+                (void)write(status_fd,&ret,sizeof(ret));
+                mypid = getpid();
+                /*
+                 * This seems awkward, but the alternative would be to add
+                 * or change return values for functions in multiple layers,
+                 * just so they can store the value for later retrieval by
+                 * the code already running in the right context at the other
+                 * end of this pipe.  That's a lot of disruption for nothing.
+                 */
+                (void)write(status_fd,&mypid,sizeof(mypid));
+        }
+        GFFUSE_LOGERR("Mount child exiting");
+        exit(0);
+
+parent_out:
         if (ret == -1) {
                 if (mounted)
                         umount2 (mountpoint, 2); /* lazy umount */
-                close (fd);
+                if (fd != in_fd) {
+                        close (fd);
+                }
                 fd = -1;
         }
         FREE (mnt_param_mnt);
@@ -651,13 +683,21 @@ escape (char *s)
 
 int
 gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
-               pid_t *mtab_pid)
+               pid_t *mtab_pid, int status_fd)
 {
         int fd = -1, rv = -1;
         char *fm_mnt_params = NULL, *p = NULL;
         char *efsname = NULL;
 
-        fd = fuse_mount_sys (mountpoint, fsname, mnt_param, mtab_pid);
+        fd = open ("/dev/fuse", O_RDWR);
+        if (fd == -1) {
+                GFFUSE_LOGERR ("cannot open /dev/fuse (%s)",
+                                strerror (errno));
+                return -1;
+        }
+
+        fd = fuse_mount_sys (mountpoint, fsname, mnt_param, mtab_pid, fd,
+                             status_fd);
         if (fd == -1) {
                 gf_log ("glusterfs-fuse", GF_LOG_INFO,
                         "direct mount failed (%s), "
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index 9041131ee9b..f14ccf5c178 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -17,6 +17,7 @@
    <http://www.gnu.org/licenses/>.
 */
 
+#include <sys/wait.h>
 #include "fuse-bridge.h"
 
 static int gf_fuse_conn_err_log;
@@ -3874,20 +3875,50 @@ unlock:
         return 0;
 }
 
+int
+fuse_get_mount_status (xlator_t *this)
+{
+        int             kid_status = -1;
+        pid_t           kid_pid = -1;
+        fuse_private_t *priv = this->private;
+        int             our_status = -1;
+
+        if (read(priv->status_pipe[0],&kid_status, sizeof(kid_status)) < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "could not get mount status");
+                goto out;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "mount status is %d", kid_status);
+
+        if (read(priv->status_pipe[0],&kid_pid, sizeof(kid_pid)) < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "could not get mount PID");
+                goto out;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "mount PID is %d", kid_pid);
+
+        (void)waitpid(kid_pid,NULL,0);
+        our_status = kid_status;
+
+out:
+        close(priv->status_pipe[0]);
+        close(priv->status_pipe[1]);
+        return our_status;
+}
 
 static void *
 fuse_thread_proc (void *data)
 {
-        char           *mount_point = NULL;
-        xlator_t       *this = NULL;
-        fuse_private_t *priv = NULL;
-        ssize_t         res = 0;
-        struct iobuf   *iobuf = NULL;
-        fuse_in_header_t *finh;
-        struct iovec iov_in[2];
-        void *msg = NULL;
-        const size_t msg0_size = sizeof (*finh) + 128;
-        fuse_handler_t **fuse_ops = NULL;
+        char                     *mount_point = NULL;
+        xlator_t                 *this = NULL;
+        fuse_private_t           *priv = NULL;
+        ssize_t                   res = 0;
+        struct iobuf             *iobuf = NULL;
+        fuse_in_header_t         *finh;
+        struct iovec              iov_in[2];
+        void                     *msg = NULL;
+        const size_t              msg0_size = sizeof (*finh) + 128;
+        fuse_handler_t          **fuse_ops = NULL;
+        struct pollfd             pfd[2] = {{0,}};
+        gf_boolean_t              mount_finished = _gf_false;
 
         this = data;
         priv = this->private;
@@ -3904,6 +3935,41 @@ fuse_thread_proc (void *data)
                 /* THIS has to be reset here */
                 THIS = this;
 
+                if (!mount_finished) {
+                        memset(pfd,0,sizeof(pfd));
+                        pfd[0].fd = priv->status_pipe[0];
+                        pfd[0].events = POLLIN | POLLHUP | POLLERR;
+                        pfd[1].fd = priv->fd;
+                        pfd[1].events = POLLIN | POLLHUP | POLLERR;
+                        if (poll(pfd,2,-1) < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "poll error %s", strerror(errno));
+                                break;
+                        }
+                        if (pfd[0].revents & POLLIN) {
+                                if (fuse_get_mount_status(this) != 0) {
+                                        break;
+                                }
+                                mount_finished = _gf_true;
+                        }
+                        else if (pfd[0].revents) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "mount pipe closed without status");
+                                break;
+                        }
+                        if (!pfd[1].revents) {
+                                continue;
+                        }
+                }
+
+                /*
+                 * We don't want to block on readv while we're still waiting
+                 * for mount status.  That means we only want to get here if
+                 * mount_status is true (meaning that our wait completed
+                 * already) or if we already called poll(2) on priv->fd to
+                 * make sure it's ready.
+                 */
+
                 if (priv->init_recvd)
                         fuse_graph_sync (this);
 
@@ -4025,8 +4091,11 @@ fuse_thread_proc (void *data)
                 GF_FREE (iov_in[0].iov_base);
         }
 
-        iobuf_unref (iobuf);
-        GF_FREE (iov_in[0].iov_base);
+        /*
+         * We could be in all sorts of states with respect to iobuf and iov_in
+         * by the time we get here, and it's just not worth untangling them if
+         * we're about to kill ourselves anyway.
+         */
 
         if (dict_get (this->options, ZR_MOUNTPOINT_OPT))
                 mount_point = data_to_str (dict_get (this->options,
@@ -4034,11 +4103,10 @@ fuse_thread_proc (void *data)
         if (mount_point) {
                 gf_log (this->name, GF_LOG_INFO,
                         "unmounting %s", mount_point);
-                dict_del (this->options, ZR_MOUNTPOINT_OPT);
         }
 
+        /* Kill the whole process, not just this thread. */
         kill (getpid(), SIGTERM);
-
         return NULL;
 }
 
@@ -4507,8 +4575,15 @@ init (xlator_t *this_xl)
         if (!mnt_args)
                 goto cleanup_exit;
 
+        if (pipe(priv->status_pipe) < 0) {
+                gf_log (this_xl->name, GF_LOG_ERROR,
+                        "could not create pipe to separate mount process");
+                goto cleanup_exit;
+        }
+
         priv->fd = gf_fuse_mount (priv->mount_point, fsname, mnt_args,
-                                  sync_mtab ? &ctx->mtab_pid : NULL);
+                                  sync_mtab ? &ctx->mtab_pid : NULL,
+                                  priv->status_pipe[1]);
         if (priv->fd == -1)
                 goto cleanup_exit;
 
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index cdb2ad1f74c..3711ca54da0 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -66,7 +66,7 @@
 
 #define MAX_FUSE_PROC_DELAY 1
 
-#define DISABLE_SELINUX 1
+//#define DISABLE_SELINUX 1
 
 typedef struct fuse_in_header fuse_in_header_t;
 typedef void (fuse_handler_t) (xlator_t *this, fuse_in_header_t *finh,
@@ -116,6 +116,9 @@ struct fuse_private {
         int                  revchan_in;
         int                  revchan_out;
         gf_boolean_t         reverse_fuse_thread_started;
+
+        /* For communicating with separate mount thread. */
+        int                  status_pipe[2];
 };
 typedef struct fuse_private fuse_private_t;
 
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index 3ca09b03b1d..37cc2f9f60a 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -34,6 +34,36 @@ _init ()
     UPDATEDBCONF=/etc/updatedb.conf
 }
 
+# Mount happens asynchronously, so the command status alone will never be
+# sufficient.  Instead, we have to wait for multiple events representing
+# different possible outcomes.
+wait_for ()
+{
+    local daemon_pid=$1
+    local mount_point=$2
+
+    waited=0
+    while true; do
+        kill -s 0 $daemon_pid
+        if [ $? != 0 ]; then
+                echo "Gluster client daemon exited unexpectedly."
+                umount $mount_point &> /dev/null
+                exit 1
+        fi
+        inode=$(stat -c %i $mount_point 2>/dev/null);
+        if [ "$inode" = "1" ]; then
+            break
+        fi
+        if [ $waited -ge 10 ]; then
+            break
+        fi
+        sleep 1
+        waited=$((waited+1))
+    done
+
+    echo "$inode"
+}
+
 start_glusterfs ()
 {
     if [ -n "$log_level_str" ]; then
@@ -121,34 +151,33 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --volfile=$volfile_loc");
     fi
 
-    cmd_line=$(echo "$cmd_line $mount_point");
-    err=0;
-    $cmd_line;
-
-
-    inode=$(stat -c %i $mount_point 2>/dev/null);
+    cmd_line=$(echo "$cmd_line $mount_point")
+    err=0
+    $cmd_line
+    daemon_pid=$$
 
+    # Wait for the inode to change *or* for the daemon to exit (indicating a
+    # problem with the mount).
+    inode=$(wait_for $daemon_pid $mount_point)
     # this is required if the stat returns error
     if [ -z "$inode" ]; then
-        inode="0";
+        inode="0"
     fi
 
     # retry the failover
-    # if [ $? != "0" ]; then # <--- TODO: Once glusterfs returns proper error code, change it.
     if [ $inode -ne 1 ]; then
-        err=1;
         if [ -n "$cmd_line1" ]; then
             cmd_line1=$(echo "$cmd_line1 $mount_point");
-            $cmd_line1;
-            err=0;
+            $cmd_line1
+            daemon_pid=$$
 
-            inode=$(stat -c %i $mount_point 2>/dev/null);
+            inode=$(wait_for $daemon_pid $mount_point)
             # this is required if the stat returns error
             if [ -z "$inode" ]; then
                 inode="0";
             fi
             if [ $inode -ne 1 ]; then
-                err=1;
+                err=1
             fi
         fi
     fi