fuse, glusterfsd: mount logic fixes

Commit 7d0397c2 introduced two issues:

 i) broke the libfuse derived mount logic (details below)
ii) in case of a daemonized glusterfs client is ran as daemon, parent
    process can return earlier than the mount is in place, which breaks
    agents that programmatically do a gluster mount via a direct call to
    glusterfs (ie. not via mount(8)).

This patch fixes these issues by a refactor that merges the approaches
sported by commits

  7d0397c2 fuse: allow requests during mount (needed for SELinux labels)
  c5d781e0 upon daemonizing, wait on mtab update to terminate in parent

Original daemonized libfuse event flow is as follows:

  try:
    fd = open("/dev/fuse")
    mount("-oopts,fd=%s" % fd ...)
    mount(8) -f    # manipulate mtab
  except:
    sp = socketpair()
    env _FUSE_COMMFD=sp fusermount -oopts
    fd = receive_fd(sp)

  where fusermount(1) does:
    fd = open("/dev/fuse")
    mount("-oopts,fd=%d" % fd ...)
    sp = atoi(getenv("_FUSE_COMMFD"))
    send_fd(sp, fd)

  daemonize(
    # in child
    fuse_loop(fd)
  )
  # in parent
  exit()

As of 013850c9 (instead of adopting FUSE's 47e61004¹), we went for async
mtab manipulation, and as of c5d781e0, still wanted keep that in sync
with termination of daemon parent, so we changed it to:

  try:
    fd = open("/dev/fuse")
    mount("-oopts,fd=%s" % fd ...)
    pid = fork(
      # in child
      mount(8) -f
    )
  except:
    sp = socketpair()
    env _FUSE_COMMFD=sp fusermount -oopts
    fd = receive_fd(sp)

  daemonize(
    fuse_loop(fd)
  )
  waitpid(pid)
  exit()

(Note the new approch came only to direct [privileged] mount, so fusermount
based mounting was already partially broken.)

As of 7d0397c2, with the purpose of facilitating async mount, the event flow
was practically reduced to:

  fd = open("/dev/fuse")
  fork(
    mount("-oopts,fd=%s" % fd ...)
    fork(
      mount(8) -n
    )
  )

  daemonize(
    fuse_loop(fd)
  )
  exit()

Thus fusermount based mounting become defunct; however, the dead
code was still kept around. So, we should either drop it or fix
it. Also, the mtab manipulator is forked into yet another child
with no purpose, while syncing with it in daemon parent is broken.
mount(2) is neither synced with parent.

Now we are coming to the following scheme:

  fd = open("/dev/fuse")
  pid = fork(
    try:
      mount("-oopts,fd=%s" % fd ...)
      mount(8) -n
    except:
      env _FUSE_DEVFD=fd fusermount -oopts
  )

  where fusermount(1) does:
    fd = getenv("_FUSE_DEVFD")
    mount("-oopts,fd=%s" % fd ...)

  daemonize(
    fuse_loop(fd)
  )
  waitpid(pid)
  exit()

Nb.:

- We can't help losing compatibility with upstream fusermount,
  as it sends back the fd only when mount(2) is completed,
  thus defeating the async mount approach. The
  'getenv("_FUSE_DEVFD")' mechanism is specfic to glusterfs'
  fusermount (at the moment -- sure we can talk about it with
  upstream)

- fusermount opens /dev/fuse at same privilege level as of
  original process², so we can bravely go on with doing the open
  unconditionally in original process

- Original mounting code actually tries to mount through
  fusermount _twice_: if first attempt fails, then, assuming
  subtype support is missing in kernel, it tries again subtype
  stripped. However, this is redundant, as fusermount internally
  also performs the subtype check³. Therefore we simplified the
  logic to have just a single fusermount call.

- we revert the changes to mount.glusterfs as of 7d0397c2, as
  now there is no issue with glusterfs to work around in that scope

¹ http://fuse.git.sourceforge.net/git/gitweb.cgi?p=fuse/fuse;a=blobdiff;f=ChangeLog;h=47e61004;hb=4c3d9b19;hpb=e61b775a
² http://fuse.git.sourceforge.net/git/gitweb.cgi?p=fuse/fuse;a=blob;f=util/fusermount.c;h=b2e87d95#l1023
³ http://fuse.git.sourceforge.net/git/gitweb.cgi?p=fuse/fuse;a=blob;f=util/fusermount.c;h=b2e87d95#l839

Change-Id: I0c4ab70e0c5ad7b27337228749b266bcd0ba941d
BUG: 811217
Signed-off-by: Csaba Henk <csaba@redhat.com>
Reviewed-on: http://review.gluster.com/3341
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-by: Anand Avati <avati@redhat.com>

1 files changed, 107 insertions, 205 deletions

diff --git a/contrib/fuse-lib/mount.c b/contrib/fuse-lib/mount.c
index fde9f316a4d..759014fe2af 100644
--- a/contrib/fuse-lib/mount.c
+++ b/contrib/fuse-lib/mount.c
@@ -26,7 +26,6 @@
 #endif /* __NetBSD__ */
 #include <sys/stat.h>
 #include <sys/poll.h>
-#include <sys/socket.h>
 #include <sys/un.h>
 #include <sys/wait.h>
 #include <sys/mount.h>
@@ -56,7 +55,7 @@
 #else
 #define FUSERMOUNT_PROG "fusermount"
 #endif
-#define FUSE_COMMFD_ENV "_FUSE_COMMFD"
+#define FUSE_DEVFD_ENV "_FUSE_DEVFD"
 
 #define GFFUSE_LOGERR(...) \
         gf_log ("glusterfs-fuse", GF_LOG_ERROR, ## __VA_ARGS__)
@@ -115,8 +114,7 @@ static
 #endif
 int
 fuse_mnt_add_mount (const char *progname, const char *fsname,
-                    const char *mnt, const char *type, const char *opts,
-                    pid_t *mtab_pid)
+                    const char *mnt, const char *type, const char *opts)
 {
         int res;
         int status;
@@ -144,24 +142,6 @@ fuse_mnt_add_mount (const char *progname, const char *fsname,
                 char templ[] = "/tmp/fusermountXXXXXX";
                 char *tmp;
 
-                if (!mtab_pid) {
-                        /* mtab update done async, just log if fails */
-                        res = fork ();
-                        if (res)
-                                exit (res == -1 ? 1 : 0);
-                        res = fork ();
-                        if (res) {
-                                if (res != -1) {
-                                        if (!(res == waitpid (res, &status, 0)
-                                              && status == 0))
-                                                GFFUSE_LOGERR ("%s: /etc/mtab "
-                                                               "update failed",
-                                                               progname);
-                                }
-                                exit (0);
-                        }
-                }
-
                 sigprocmask (SIG_SETMASK, &oldmask, NULL);
                 setuid (geteuid ());
 
@@ -187,15 +167,11 @@ fuse_mnt_add_mount (const char *progname, const char *fsname,
                                progname, _PATH_MOUNT, strerror (errno));
                 exit (1);
         }
-        if (mtab_pid) {
-                *mtab_pid = res;
-                res = 0;
-        } else {
-                if (!(res == waitpid (res, &status, 0) && status == 0))
-                        res = -1;
-        }
+
+        res = waitpid (res, &status, 0);
         if (res == -1)
                 GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
+        res = (res != -1 && status == 0) ? 0 : -1;
 
  out_restore:
         sigprocmask (SIG_SETMASK, &oldmask, NULL);
@@ -273,76 +249,75 @@ char
 }
 
 #ifndef FUSE_UTIL
-/* return value:
- * >= 0         => fd
- * -1         => error
- */
-static int
-receive_fd (int fd)
+static char *
+escape (char *s)
 {
-        struct msghdr msg;
-        struct iovec iov;
-        char buf[1];
-        int rv;
-        size_t ccmsg[CMSG_SPACE (sizeof (int)) / sizeof (size_t)];
-        struct cmsghdr *cmsg;
-        int *recv_fd;
-
-        iov.iov_base = buf;
-        iov.iov_len = 1;
-
-        msg.msg_name = 0;
-        msg.msg_namelen = 0;
-        msg.msg_iov = &iov;
-        msg.msg_iovlen = 1;
-        /* old BSD implementations should use msg_accrights instead of
-         * msg_control; the interface is different. */
-        msg.msg_control = ccmsg;
-        msg.msg_controllen = sizeof (ccmsg);
-
-        while (((rv = recvmsg (fd, &msg, 0)) == -1) && errno == EINTR);
-        if (rv == -1) {
-                GFFUSE_LOGERR ("recvmsg failed: %s", strerror (errno));
-                return -1;
-        }
-        if (!rv) {
-                /* EOF */
-                return -1;
+        size_t len = 0;
+        char *p = NULL;
+        char *q = NULL;
+        char *e = NULL;
+
+        for (p = s; *p; p++) {
+                if (*p == ',')
+                       len++;
+                len++;
         }
 
-        cmsg = CMSG_FIRSTHDR (&msg);
-        /*
-         * simplify condition expression
-         */
-        if (cmsg->cmsg_type != SCM_RIGHTS) {
-                GFFUSE_LOGERR ("got control message of unknown type %d",
-                               cmsg->cmsg_type);
-                return -1;
+        e = CALLOC (1, len + 1);
+        if (!e)
+                return NULL;
+
+        for (p = s, q = e; *p; p++, q++) {
+                if (*p == ',') {
+                        *q = '\\';
+                        q++;
+                }
+                *q = *p;
         }
 
-        recv_fd = (int *) CMSG_DATA (cmsg);
-        return (*recv_fd);
+        return e;
 }
 
 static int
-fuse_mount_fusermount (const char *mountpoint, const char *opts)
+fuse_mount_fusermount (const char *mountpoint, char *fsname, char *mnt_param,
+                       int fd)
 {
-        int fds[2], pid;
-        int res;
-        int rv;
+        int  pid = -1;
+        int  res = 0;
+        int  ret = -1;
+        char *fm_mnt_params = NULL;
+        char *efsname = NULL;
+
+#ifndef GF_FUSERMOUNT
+        GFFUSE_LOGERR ("Mounting via helper utility "
+                       "(unprivileged mounting) is supported "
+                       "only if glusterfs is compiled with "
+                       "--enable-fusermount");
+        return -1;
+#endif
+
+        efsname = escape (fsname);
+        if (!efsname) {
+                GFFUSE_LOGERR ("Out of memory");
 
-        res = socketpair (PF_UNIX, SOCK_STREAM, 0, fds);
-        if (res == -1) {
-                GFFUSE_LOGERR ("socketpair() failed: %s", strerror (errno));
                 return -1;
         }
+        ret = asprintf (&fm_mnt_params,
+                        "%s,fsname=%s,nonempty,subtype=glusterfs",
+                        mnt_param, efsname);
+        FREE (efsname);
+        if (ret == -1) {
+                GFFUSE_LOGERR ("Out of memory");
+
+                goto out;
+        }
 
+        /* fork to exec fusermount */
         pid = fork ();
         if (pid == -1) {
                 GFFUSE_LOGERR ("fork() failed: %s", strerror (errno));
-                close (fds[0]);
-                close (fds[1]);
-                return -1;
+                ret = -1;
+                goto out;
         }
 
         if (pid == 0) {
@@ -351,30 +326,25 @@ fuse_mount_fusermount (const char *mountpoint, const char *opts)
                 int a = 0;
 
                 argv[a++] = FUSERMOUNT_PROG;
-                if (opts) {
-                        argv[a++] = "-o";
-                        argv[a++] = opts;
-                }
+                argv[a++] = "-o";
+                argv[a++] = fm_mnt_params;
                 argv[a++] = "--";
                 argv[a++] = mountpoint;
                 argv[a++] = NULL;
 
-                close (fds[1]);
-                fcntl (fds[0], F_SETFD, 0);
-                snprintf (env, sizeof (env), "%i", fds[0]);
-                setenv (FUSE_COMMFD_ENV, env, 1);
+                snprintf (env, sizeof (env), "%i", fd);
+                setenv (FUSE_DEVFD_ENV, env, 1);
                 execvp (FUSERMOUNT_PROG, (char **)argv);
                 GFFUSE_LOGERR ("failed to exec fusermount: %s",
                                strerror (errno));
                 _exit (1);
         }
 
-        close (fds[0]);
-        rv = receive_fd (fds[1]);
-        close (fds[1]);
-        waitpid (pid, NULL, 0); /* bury zombie */
-
-        return rv;
+        ret = waitpid (pid, &res, 0);
+        ret = (ret == pid && res == 0) ? 0 : -1;
+ out:
+        FREE (fm_mnt_params);
+        return ret;
 }
 #endif
 
@@ -544,26 +514,13 @@ gf_fuse_unmount (const char *mountpoint, int fd)
 
 #ifndef FUSE_UTIL
 static int
-fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mtab_pid, int in_fd, int status_fd)
+fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, int fd)
 {
-        int fd = -1, ret = -1;
+        int ret = -1;
         unsigned mounted = 0;
         char *mnt_param_mnt = NULL;
         char *fstype = "fuse.glusterfs";
         char *source = fsname;
-        pid_t mypid = -1;
-
-        if (in_fd >= 0) {
-                fd = in_fd;
-        }
-        else {
-                fd = open ("/dev/fuse", O_RDWR);
-                if (fd == -1) {
-                        GFFUSE_LOGERR ("cannot open /dev/fuse (%s)",
-                                        strerror (errno));
-                        return -1;
-                }
-        }
 
         ret = asprintf (&mnt_param_mnt,
                         "%s,fd=%i,rootmode=%o,user_id=%i,group_id=%i",
@@ -573,14 +530,8 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
 
                 goto out;
         }
-        ret = fork();
-        if (ret != 0) {
-                goto parent_out;
-        }
-        GFFUSE_LOGERR("calling mount");
         ret = mount (source, mountpoint, fstype, 0,
                      mnt_param_mnt);
-        GFFUSE_LOGERR("mount returned %d",ret);
         if (ret == -1 && errno == ENODEV) {
                 /* fs subtype support was added by 79c0b2df aka
                    v2.6.21-3159-g79c0b2d. Probably we have an
@@ -611,7 +562,7 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
                 }
 
                 ret = fuse_mnt_add_mount ("fuse", source, newmnt, fstype,
-                                          mnt_param, mtab_pid);
+                                          mnt_param);
                 FREE (newmnt);
                 if (ret == -1) {
                         GFFUSE_LOGERR ("failed to add mtab entry");
@@ -621,75 +572,25 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
         }
 #endif /* __NetBSD__ */
 
-        ret = 0;
 out:
-        if (status_fd >= 0) {
-                GFFUSE_LOGERR("writing status");
-                (void)write(status_fd,&ret,sizeof(ret));
-                mypid = getpid();
-                /*
-                 * This seems awkward, but the alternative would be to add
-                 * or change return values for functions in multiple layers,
-                 * just so they can store the value for later retrieval by
-                 * the code already running in the right context at the other
-                 * end of this pipe.  That's a lot of disruption for nothing.
-                 */
-                (void)write(status_fd,&mypid,sizeof(mypid));
-        }
-        GFFUSE_LOGERR("Mount child exiting");
-        exit(0);
-
-parent_out:
         if (ret == -1) {
                 if (mounted)
                         umount2 (mountpoint, 2); /* lazy umount */
-                if (fd != in_fd) {
-                        close (fd);
-                }
-                fd = -1;
         }
         FREE (mnt_param_mnt);
         if (source != fsname)
                 FREE (source);
-        return fd;
-}
-
-static char *
-escape (char *s)
-{
-        size_t len = 0;
-        char *p = NULL;
-        char *q = NULL;
-        char *e = NULL;
 
-        for (p = s; *p; p++) {
-                if (*p == ',')
-                       len++;
-                len++;
-        }
-
-        e = CALLOC (1, len + 1);
-        if (!e)
-                return NULL;
-
-        for (p = s, q = e; *p; p++, q++) {
-                if (*p == ',') {
-                        *q = '\\';
-                        q++;
-                }
-                *q = *p;
-        }
-
-        return e;
+        return ret;
 }
 
 int
 gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
-               pid_t *mtab_pid, int status_fd)
+               pid_t *mnt_pid, int status_fd)
 {
-        int fd = -1, rv = -1;
-        char *fm_mnt_params = NULL, *p = NULL;
-        char *efsname = NULL;
+        int   fd  = -1;
+        pid_t pid = -1;
+        int   ret = -1;
 
         fd = open ("/dev/fuse", O_RDWR);
         if (fd == -1) {
@@ -698,45 +599,46 @@ gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
                 return -1;
         }
 
-        fd = fuse_mount_sys (mountpoint, fsname, mnt_param, mtab_pid, fd,
-                             status_fd);
-        if (fd == -1) {
-                gf_log ("glusterfs-fuse", GF_LOG_INFO,
-                        "direct mount failed (%s), "
-                        "retry to mount via fusermount",
-                        strerror (errno));
-
-                efsname = escape (fsname);
-                if (!efsname) {
-                        GFFUSE_LOGERR ("Out of memory");
-
-                        return -1;
+        /* start mount agent */
+        pid = fork();
+        switch (pid) {
+        case 0:
+                /* hello it's mount agent */
+                if (!mnt_pid) {
+                        /* daemonize mount agent, caller is
+                         * not interested in waiting for it
+                         */
+                        pid = fork ();
+                        if (pid)
+                                exit (pid == -1 ? 1 : 0);
                 }
-                rv = asprintf (&fm_mnt_params,
-                               "%s,fsname=%s,nonempty,subtype=glusterfs",
-                               mnt_param, efsname);
-                FREE (efsname);
-                if (rv == -1) {
-                        GFFUSE_LOGERR ("Out of memory");
 
-                        return -1;
-                }
-
-                fd = fuse_mount_fusermount (mountpoint, fm_mnt_params);
-                if (fd == -1) {
-                        p = fm_mnt_params + strlen (fm_mnt_params);
-                        while (*--p != ',');
-                        *p = '\0';
+                ret = fuse_mount_sys (mountpoint, fsname, mnt_param, fd);
+                if (ret == -1) {
+                        gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                                "direct mount failed (%s), "
+                                "retry to mount via fusermount",
+                                strerror (errno));
 
-                        fd = fuse_mount_fusermount (mountpoint, fm_mnt_params);
+                        ret = fuse_mount_fusermount (mountpoint, fsname,
+                                                     mnt_param, fd);
                 }
 
-                FREE (fm_mnt_params);
+                if (ret == -1)
+                        GFFUSE_LOGERR ("mount failed");
 
-                if (fd == -1)
-                       GFFUSE_LOGERR ("mount failed");
+                if (status_fd >= 0)
+                        (void)write (status_fd, &ret, sizeof (ret));
+                exit (!!ret);
+                /* bye mount agent */
+        case -1:
+                close (fd);
+                fd = -1;
         }
 
+        if (mnt_pid)
+               *mnt_pid = pid;
+
         return fd;
 }
 #endif