fuse-bridge: Handle graph-switch.

The purpose of this patch is to let protocol/client know when its transports can
be disconnected, without application running on gluster mount noticing any
effects of graph switch.

In order to do this, we migrate all fds and blocked locks to new graph.
Once this migration is complete and there are no in-transit frames as viewed
by fuse-bridge, we send a PARENT_DOWN event to its children. protocol/client
on receiving this event, can disconnect up its transports.

Change-Id: Idcea4bc43e23fb077ac16538b61335ebad84ba16
BUG: 767862
Reviewed-on: http://review.gluster.com/2734
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

1 files changed, 434 insertions, 48 deletions

diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index e44aca1d0c1..db2deaca29c 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -205,11 +205,11 @@ fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
                 inode_t *inode, struct iatt *buf)
 {
-        fuse_state_t            *state = NULL;
-        fuse_in_header_t        *finh = NULL;
-        struct fuse_entry_out    feo = {0, };
-        fuse_private_t          *priv = NULL;
-        inode_t                 *linked_inode = NULL;
+        fuse_state_t          *state        = NULL;
+        fuse_in_header_t      *finh         = NULL;
+        struct fuse_entry_out  feo          = {0, };
+        fuse_private_t        *priv         = NULL;
+        inode_t               *linked_inode = NULL;
 
         priv = this->private;
         state = frame->root->state;
@@ -630,11 +630,11 @@ static int
 fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
              int32_t op_ret, int32_t op_errno, fd_t *fd)
 {
-        fuse_state_t         *state = NULL;
-        fuse_in_header_t     *finh  = NULL;
-        fuse_private_t       *priv  = NULL;
-        int32_t               ret   = 0;
-        struct fuse_open_out  foo   = {0, };
+        fuse_state_t         *state  = NULL;
+        fuse_in_header_t     *finh   = NULL;
+        fuse_private_t       *priv   = NULL;
+        int32_t               ret    = 0;
+        struct fuse_open_out  foo    = {0, };
 
         priv = this->private;
         state = frame->root->state;
@@ -677,12 +677,10 @@ fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         goto err;
                 }
 
-                fd_ref (fd);
-
                 if (send_fuse_obj (this, finh, &foo) == ENOENT) {
                         gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
                                 "open(%s) got EINTR", state->loc.path);
-                        fd_unref (fd);
+                        gf_fd_put (priv->fdtable, state->fd_no);
                         goto out;
                 }
 
@@ -695,6 +693,7 @@ fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         strerror (op_errno));
 
                 send_fuse_err (this, finh, op_errno);
+                gf_fd_put (priv->fdtable, state->fd_no);
         }
 out:
         free_fuse_state (state);
@@ -1141,6 +1140,11 @@ fuse_mknod_resume (fuse_state_t *state)
                 return;
         }
 
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
         if (state->loc.inode) {
                 gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
                 inode_unref (state->loc.inode);
@@ -1236,6 +1240,11 @@ fuse_mkdir_resume (fuse_state_t *state)
                 return;
         }
 
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
         if (state->loc.inode) {
                 gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
                 inode_unref (state->loc.inode);
@@ -1399,6 +1408,11 @@ fuse_symlink_resume (fuse_state_t *state)
                 return;
         }
 
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
         if (state->loc.inode) {
                 gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
                 inode_unref (state->loc.inode);
@@ -1516,6 +1530,9 @@ fuse_rename_resume (fuse_state_t *state)
                 return;
         }
 
+        state->resolve.op_ret = 0;
+        state->resolve2.op_ret = 0;
+
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": RENAME `%s (%s)' -> `%s (%s)'",
                 state->finh->unique, state->loc.path, loc_uuid,
@@ -1558,6 +1575,9 @@ fuse_link_resume (fuse_state_t *state)
                 return;
         }
 
+        state->resolve.op_ret = 0;
+        state->resolve2.op_ret = 0;
+
 	if (state->loc.inode) {
 		inode_unref (state->loc.inode);
 		state->loc.inode = NULL;
@@ -1599,14 +1619,14 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  fd_t *fd, inode_t *inode, struct iatt *buf,
                  struct iatt *preparent, struct iatt *postparent)
 {
-        fuse_state_t            *state = NULL;
-        fuse_in_header_t        *finh = NULL;
-        fuse_private_t          *priv = NULL;
-        struct fuse_out_header   fouh = {0, };
-        struct fuse_entry_out    feo = {0, };
-        struct fuse_open_out     foo = {0, };
-        struct iovec             iov_out[3];
-        inode_t                 *linked_inode = NULL;
+        fuse_state_t           *state        = NULL;
+        fuse_in_header_t       *finh         = NULL;
+        fuse_private_t         *priv         = NULL;
+        struct fuse_out_header  fouh         = {0, };
+        struct fuse_entry_out   feo          = {0, };
+        struct fuse_open_out    foo          = {0, };
+        struct iovec            iov_out[3];
+        inode_t                *linked_inode = NULL;
 
         state    = frame->root->state;
         priv     = this->private;
@@ -1645,8 +1665,6 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 inode_unref (linked_inode);
 
-                fd_ref (fd);
-
                 feo.nodeid = inode_to_fuse_nodeid (linked_inode);
 
                 feo.entry_valid = calc_timeout_sec (priv->entry_timeout);
@@ -1671,7 +1689,7 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
                                 "create(%s) got EINTR", state->loc.path);
                         inode_forget (inode, 1);
-                        fd_unref (fd);
+                        gf_fd_put (priv->fdtable, state->fd_no);
                         goto out;
                 }
 
@@ -1681,6 +1699,7 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%"PRIu64": %s => -1 (%s)", finh->unique,
                         state->loc.path, strerror (op_errno));
                 send_fuse_err (this, finh, op_errno);
+                gf_fd_put (priv->fdtable, state->fd_no);
         }
 out:
         free_fuse_state (state);
@@ -1693,7 +1712,8 @@ out:
 void
 fuse_create_resume (fuse_state_t *state)
 {
-        fd_t *fd = NULL;
+        fd_t           *fd   = NULL;
+        fuse_private_t *priv = NULL;
 
         if (!state->loc.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
@@ -1705,15 +1725,26 @@ fuse_create_resume (fuse_state_t *state)
                 return;
         }
 
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
         if (state->loc.inode) {
-                gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
+                gf_log (state->this->name, GF_LOG_DEBUG,
+                        "inode already present");
                 inode_unref (state->loc.inode);
         }
 
         state->loc.inode = inode_new (state->loc.parent->table);
 
         fd = fd_create (state->loc.inode, state->finh->pid);
-        state->fd = fd;
+
+        priv = state->this->private;
+
+        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
+
+        state->fd = fd_ref (fd);
         fd->flags = state->flags;
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
@@ -1799,7 +1830,8 @@ fuse_create (xlator_t *this, fuse_in_header_t *finh, void *msg)
 void
 fuse_open_resume (fuse_state_t *state)
 {
-        fd_t *fd = NULL;
+        fd_t           *fd   = NULL;
+        fuse_private_t *priv = NULL;
 
         if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
@@ -1820,7 +1852,10 @@ fuse_open_resume (fuse_state_t *state)
                 return;
         }
 
-        state->fd = fd;
+        priv = state->this->private;
+
+        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
+        state->fd = fd_ref (fd);
         fd->flags = state->flags;
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
@@ -2084,18 +2119,21 @@ fuse_flush (xlator_t *this, fuse_in_header_t *finh, void *msg)
 static void
 fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_release_in *fri        = msg;
-        fd_t                   *new_fd     = NULL;
-        fd_t                   *fd         = NULL;
-        uint64_t                val        = 0;
-        int                     ret        = 0;
-        fuse_state_t           *state      = NULL;
-        fuse_fd_ctx_t          *fdctx      = NULL;
+        struct fuse_release_in *fri    = msg;
+        fd_t                   *new_fd = NULL;
+        fd_t                   *fd     = NULL;
+        uint64_t                val    = 0;
+        int                     ret    = 0;
+        fuse_state_t           *state  = NULL;
+        fuse_fd_ctx_t          *fdctx  = NULL;
+        fuse_private_t         *priv   = NULL;
 
         GET_STATE (this, finh, state);
         fd = FH_TO_FD (fri->fh);
         state->fd = fd;
 
+        priv = this->private;
+
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": RELEASE %p", finh->unique, state->fd);
 
@@ -2113,6 +2151,10 @@ fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
         }
         fd_unref (fd);
 
+        state->fd = NULL;
+
+        gf_fdptr_put (priv->fdtable, fd);
+
         send_fuse_err (this, finh, 0);
 
         free_fuse_state (state);
@@ -2156,7 +2198,10 @@ fuse_fsync (xlator_t *this, fuse_in_header_t *finh, void *msg)
 void
 fuse_opendir_resume (fuse_state_t *state)
 {
-        fd_t *fd = NULL;
+        fd_t           *fd   = NULL;
+        fuse_private_t *priv = NULL;
+
+        priv = state->this->private;
 
         if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
@@ -2168,7 +2213,8 @@ fuse_opendir_resume (fuse_state_t *state)
         }
 
         fd = fd_create (state->loc.inode, state->finh->pid);
-        state->fd = fd;
+        state->fd = fd_ref (fd);
+        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": OPENDIR %s", state->finh->unique,
@@ -2331,16 +2377,19 @@ fuse_readdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 static void
 fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_release_in *fri        = msg;
-        fd_t                   *new_fd     = NULL;
-        uint64_t                val        = 0;
-        int                     ret        = 0;
-        fuse_state_t           *state      = NULL;
-        fuse_fd_ctx_t          *fdctx      = NULL;
+        struct fuse_release_in *fri    = msg;
+        fd_t                   *new_fd = NULL;
+        uint64_t                val    = 0;
+        int                     ret    = 0;
+        fuse_state_t           *state  = NULL;
+        fuse_fd_ctx_t          *fdctx  = NULL;
+        fuse_private_t         *priv   = NULL;
 
         GET_STATE (this, finh, state);
         state->fd = FH_TO_FD (fri->fh);
 
+        priv = this->private;
+
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": RELEASEDIR %p", finh->unique, state->fd);
 
@@ -2360,6 +2409,10 @@ fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         fd_unref (state->fd);
 
+        gf_fdptr_put (priv->fdtable, state->fd);
+
+        state->fd = NULL;
+
         send_fuse_err (this, finh, 0);
 
         free_fuse_state (state);
@@ -3409,11 +3462,321 @@ fuse_first_lookup (xlator_t *this)
 
 
 int
+fuse_nameless_lookup (xlator_t *xl, uuid_t gfid, loc_t *loc)
+{
+        int          ret          = -1;
+        dict_t      *xattr_req    = NULL;
+        struct iatt  iatt         = {0, };
+
+        if ((loc == NULL) || (xl == NULL)) {
+                goto out;
+        }
+
+        if (loc->inode == NULL) {
+                loc->inode = inode_new (xl->itable);
+                if (loc->inode == NULL) {
+                        goto out;
+                }
+        }
+
+        uuid_copy (loc->gfid, gfid);
+
+        xattr_req = dict_new ();
+        if (xattr_req == NULL) {
+                goto out;
+        }
+
+        ret = syncop_lookup (xl, loc, xattr_req, &iatt, NULL, NULL);
+        if (ret < 0) {
+                goto out;
+        }
+
+        inode_link (loc->inode, NULL, NULL, &iatt);
+
+        ret = 0;
+out:
+        if (xattr_req != NULL) {
+                dict_unref (xattr_req);
+        }
+
+        return ret;
+}
+
+
+int
+fuse_migrate_fd (xlator_t *this, fd_t *fd, xlator_t *old_subvol,
+                 xlator_t *new_subvol)
+{
+        int      ret                = -1;
+        loc_t    loc                = {0, };
+        char     create_in_progress = 0;
+        inode_t *old_inode          = NULL;
+
+        /* could've used pthread_cond_wait, but that requires a cond variable to
+         * be mainted for each fd and that is a bit too much overhead.
+         */
+        do {
+                LOCK (&fd->inode->lock);
+                {
+                        if (uuid_is_null (fd->inode->gfid)) {
+                                create_in_progress = 1;
+                        } else {
+                                create_in_progress = 0;
+                        }
+                }
+                UNLOCK (&fd->inode->lock);
+
+                if (create_in_progress) {
+                        gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                                "create call on fd (%p) is in progress, "
+                                "hence waiting", fd);
+                        sleep (1);
+                }
+
+        } while (create_in_progress);
+
+        if (fd->inode->table->xl == old_subvol) {
+                ret = syncop_fsync (old_subvol, fd);
+                if (ret < 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "syncop_fsync failed (%s)", strerror (errno));
+                }
+        } else {
+                gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "fd (%p) was not "
+                        "migrated during previous graph switch", fd);
+        }
+
+        loc.path = "";
+        loc.name = NULL;
+
+        ret = fuse_nameless_lookup (new_subvol, fd->inode->gfid, &loc);
+        if (ret < 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "name-less lookup of gfid (%s) failed (%s)",
+                        uuid_utoa (fd->inode->gfid), strerror (errno));
+                goto out;
+        }
+
+        old_inode = fd->inode;
+
+        inode_ref (loc.inode);
+
+        LOCK (&fd->inode->lock);
+        {
+                list_del_init (&fd->inode_list);
+        }
+        UNLOCK (&fd->inode->lock);
+
+        LOCK (&fd->lock);
+        {
+                fd->inode = loc.inode;
+        }
+        UNLOCK (&fd->lock);
+
+        if (IA_ISDIR (fd->inode->ia_type)) {
+                ret = syncop_opendir (new_subvol, &loc, fd);
+        } else {
+                ret = syncop_open (new_subvol, &loc, fd->flags, fd);
+        }
+
+        if (ret < 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "open on gfid (%s) failed (%s)",
+                        uuid_utoa (fd->inode->gfid), strerror (errno));
+                goto out;
+        }
+
+        fd_bind (fd);
+
+        ret = 0;
+out:
+        if (loc.inode != NULL) {
+                inode_unref (loc.inode);
+        }
+
+        if (old_inode != NULL) {
+                inode_unref (old_inode);
+        }
+
+        return ret;
+}
+
+
+int
+fuse_handle_opened_fds (xlator_t *this, xlator_t *old_subvol,
+                        xlator_t *new_subvol)
+{
+        fuse_private_t *priv      = NULL;
+        fdentry_t      *fdentries = NULL;
+        uint32_t        count     = 0;
+        fdtable_t      *fdtable   = NULL;
+        int             i         = 0;
+        fd_t           *fd        = NULL;
+
+        priv = this->private;
+
+        fdtable = priv->fdtable;
+
+        fdentries = gf_fd_fdtable_copy_all_fds (fdtable, &count);
+        if (fdentries != NULL) {
+                for (i = 0; i < count; i++) {
+                        fd = fdentries[i].fd;
+                        if (fd != NULL) {
+                                fuse_migrate_fd (this, fd, old_subvol,
+                                                 new_subvol);
+                        }
+                }
+
+                GF_FREE (fdentries);
+        }
+
+        return 0;
+}
+
+
+static int
+fuse_handle_blocked_locks (xlator_t *this, xlator_t *old_subvol,
+                           xlator_t *new_subvol)
+{
+        return 0;
+}
+
+
+static int
+fuse_graph_switch_task (void *data)
+{
+        fuse_graph_switch_args_t *args = NULL;
+
+        args = data;
+        if (args == NULL) {
+                goto out;
+        }
+
+        /* don't change the order of handling open fds and blocked locks, since
+         * the act of opening files also reacquires granted locks in new graph.
+         */
+        fuse_handle_opened_fds (args->this, args->old_subvol, args->new_subvol);
+
+        fuse_handle_blocked_locks (args->this, args->old_subvol,
+                                   args->new_subvol);
+
+        pthread_mutex_lock (&args->lock);
+        {
+                args->complete = 1;
+                pthread_cond_broadcast (&args->cond);
+        }
+        pthread_mutex_unlock (&args->lock);
+out:
+        return 0;
+}
+
+
+fuse_graph_switch_args_t *
+fuse_graph_switch_args_alloc (void)
+{
+        fuse_graph_switch_args_t *args = NULL;
+
+        args = GF_CALLOC (1, sizeof (*args), gf_fuse_mt_graph_switch_args_t);
+        if (args == NULL) {
+                goto out;
+        }
+
+        pthread_cond_init (&args->cond, NULL);
+        pthread_mutex_init (&args->lock, NULL);
+
+out:
+        return args;
+}
+
+
+void
+fuse_graph_switch_args_destroy (fuse_graph_switch_args_t *args)
+{
+        if (args == NULL) {
+                goto out;
+        }
+
+        pthread_cond_destroy (&args->cond);
+        pthread_mutex_destroy (&args->lock);
+
+        GF_FREE (args);
+out:
+        return;
+}
+
+
+static int
+fuse_graph_switch_complete (int ret, call_frame_t *frame, void *data)
+{
+        return 0;
+}
+
+
+int
+fuse_handle_graph_switch (xlator_t *this, xlator_t *old_subvol,
+                          xlator_t *new_subvol)
+{
+        call_frame_t             *frame = NULL;
+        int32_t                   ret   = -1;
+        fuse_graph_switch_args_t *args  = NULL;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (frame == NULL) {
+                goto out;
+        }
+
+        args = fuse_graph_switch_args_alloc ();
+        if (args == NULL) {
+                goto out;
+        }
+
+        args->this = this;
+        args->old_subvol = old_subvol;
+        args->new_subvol = new_subvol;
+
+        ret = synctask_new (this->ctx->env, fuse_graph_switch_task,
+                            fuse_graph_switch_complete, frame, args);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING, "starting sync-task to "
+                        "handle graph switch failed");
+                goto out;
+        }
+
+        pthread_mutex_lock (&args->lock);
+        {
+                while (!args->complete) {
+                        ret = pthread_cond_wait (&args->cond, &args->lock);
+                        if (ret != 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "cond_wait failed ret:%d errno:%d", ret,
+                                        errno);
+                        }
+                }
+        }
+        pthread_mutex_unlock (&args->lock);
+
+        ret = 0;
+out:
+        if (args != NULL) {
+                fuse_graph_switch_args_destroy (args);
+        }
+
+        if (frame != NULL) {
+                STACK_DESTROY (frame->root);
+        }
+
+        return ret;
+}
+
+
+int
 fuse_graph_sync (xlator_t *this)
 {
-        fuse_private_t   *priv = NULL;
-        int               need_first_lookup = 0;
-        int               ret = 0;
+        fuse_private_t *priv                = NULL;
+        int             need_first_lookup   = 0;
+        int             ret                 = 0;
+        xlator_t       *old_subvol          = NULL, *new_subvol = NULL;
+        uint64_t        winds_on_old_subvol = 0;
 
         priv = this->private;
 
@@ -3422,7 +3785,8 @@ fuse_graph_sync (xlator_t *this)
                 if (!priv->next_graph)
                         goto unlock;
 
-                priv->active_subvol = priv->next_graph->top;
+                old_subvol = priv->active_subvol;
+                new_subvol = priv->active_subvol = priv->next_graph->top;
                 priv->next_graph = NULL;
                 need_first_lookup = 1;
 
@@ -3444,6 +3808,22 @@ unlock:
                 fuse_first_lookup (this);
         }
 
+        if ((old_subvol != NULL) && (new_subvol != NULL)) {
+                fuse_handle_graph_switch (this, old_subvol, new_subvol);
+
+                pthread_mutex_lock (&priv->sync_mutex);
+                {
+                        old_subvol->switched = 1;
+                        winds_on_old_subvol = old_subvol->winds;
+                }
+                pthread_mutex_unlock (&priv->sync_mutex);
+
+                if (winds_on_old_subvol == 0) {
+                        xlator_notify (old_subvol, GF_EVENT_PARENT_DOWN,
+                                       old_subvol, NULL);
+                }
+        }
+
         return 0;
 }
 
@@ -4068,6 +4448,12 @@ init (xlator_t *this_xl)
         if (!fsname)
                 fsname = "glusterfs";
 
+        priv->fdtable = gf_fd_fdtable_alloc ();
+        if (priv->fdtable == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR, "Out of memory");
+                goto cleanup_exit;
+        }
+
         gf_asprintf (&mnt_args, "%s%sallow_other,max_read=131072",
                      priv->read_only ? "ro," : "",
                      priv->acl ? "" : "default_permissions,");