fuse-resolve: Attempt fd-migration in resolver, if migration

was never attempted.

Since fd is always associated with an inode, we can create an
fd only after resolver resolves an inode. So, there is a possibility
that graph-switch can happen after resolver kicks in, but before
it can complete, thereby resulting in the newly created fd not
migrated to new graph. So, instead of migrating fds only during
graph-switch, we give a second chance during fd-resolution.

As an example, consider following sequence of events during a
create call:
1. create wants to resolve parent inode, hence it starts
   resolution for parent
2. graph-switch happens (it can happen since fuse-request reader
   thread returns after winding lookup calls)
3. fd-migration of all the fds which are currently in fdtable is
   attempted (Note that the fd corresponding to current create call
   is not yet created and added to fd-table, hence it will not be
   migrated as part of graph switch)
4. resolution of parent triggered as part of create, completes
5. fd is created in fuse_create_resume and this fd is not migrated
   to new graph
6. Any future fops on this fd will fail with EBADF errors
   (create call itself will succeed)

Change-Id: Iae06ecfaca24eaacb2e166ffefbbbb57446332ba
BUG: 804592
Signed-off-by: Raghavendra G <raghavendra@gluster.com>
Reviewed-on: http://review.gluster.com/3181
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

3 files changed, 107 insertions, 50 deletions

diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index f14ccf5c178..86deaea0790 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -26,7 +26,7 @@ static int gf_fuse_xattr_enotsup_log;
 void fini (xlator_t *this_xl);
 
 fuse_fd_ctx_t *
-__fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
+__fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd)
 {
         uint64_t       val    = 0;
         int32_t        ret    = 0;
@@ -54,7 +54,7 @@ __fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
 }
 
 fuse_fd_ctx_t *
-fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
+fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd)
 {
         fuse_fd_ctx_t *fd_ctx = NULL;
 
@@ -64,7 +64,7 @@ fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
 
         LOCK (&fd->lock);
         {
-                fd_ctx = __fuse_fd_ctx_check_n_create (fd, this);
+                fd_ctx = __fuse_fd_ctx_check_n_create (this, fd);
         }
         UNLOCK (&fd->lock);
 
@@ -618,7 +618,7 @@ fuse_fd_inherit_directio (xlator_t *this, fd_t *fd, struct fuse_open_out *foo)
         GF_VALIDATE_OR_GOTO_WITH_ERROR ("glusterfs-fuse", foo, out, ret,
                                         -EINVAL);
 
-        fdctx = fuse_fd_ctx_check_n_create (fd, this);
+        fdctx = fuse_fd_ctx_check_n_create (this, fd);
         if (!fdctx) {
                 ret = -ENOMEM;
                 goto out;
@@ -3506,6 +3506,7 @@ fuse_nameless_lookup (xlator_t *xl, uuid_t gfid, loc_t *loc)
         int          ret          = -1;
         dict_t      *xattr_req    = NULL;
         struct iatt  iatt         = {0, };
+        inode_t     *linked_inode = NULL;
 
         if ((loc == NULL) || (xl == NULL)) {
                 goto out;
@@ -3530,7 +3531,9 @@ fuse_nameless_lookup (xlator_t *xl, uuid_t gfid, loc_t *loc)
                 goto out;
         }
 
-        inode_link (loc->inode, NULL, NULL, &iatt);
+        linked_inode = inode_link (loc->inode, NULL, NULL, &iatt);
+        inode_unref (loc->inode);
+        loc->inode = linked_inode;
 
         ret = 0;
 out:
@@ -3589,13 +3592,17 @@ fuse_migrate_fd (xlator_t *this, fd_t *fd, xlator_t *old_subvol,
         loc.path = "";
         loc.name = NULL;
 
-        ret = fuse_nameless_lookup (new_subvol, fd->inode->gfid, &loc);
-        if (ret < 0) {
-                ret = -2;
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "name-less lookup of gfid (%s) failed (%s)",
-                        uuid_utoa (fd->inode->gfid), strerror (errno));
-                goto out;
+        loc.inode = inode_find (new_subvol->itable, fd->inode->gfid);
+
+        if (loc.inode == NULL) {
+                ret = fuse_nameless_lookup (new_subvol, fd->inode->gfid, &loc);
+                if (ret < 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "name-less lookup of gfid (%s) failed (%s)",
+                                uuid_utoa (fd->inode->gfid), strerror (errno));
+                        goto out;
+                }
+
         }
 
         old_inode = fd->inode;
@@ -3669,34 +3676,13 @@ fuse_handle_opened_fds (xlator_t *this, xlator_t *old_subvol,
                                 continue;
 
                         ret = fuse_migrate_fd (this, fd, old_subvol,
-                                                       new_subvol);
-                        if (ret < 0) {
-                                if (ret == -1) {
-                                        fdctx = fuse_fd_ctx_check_n_create (fd,
-                                                                         this);
-                                        if (fdctx != NULL) {
-                                                fdctx->migration_failed = 1;
-                                                gf_log_callingfn ("glusterfs-"
-                                                                  "fuse",
-                                                                  GF_LOG_ERROR,
-                                                                  "fd migration"
-                                                                  " for the fd "
-                                                                  "(%p), with"
-                                                                  "context (%p)"
-                                                                  " failed", fd,
-                                                                  fdctx);
-                                        }
+                                               new_subvol);
+
+                        fdctx = fuse_fd_ctx_check_n_create (this, fd);
+                        if (fdctx) {
+                                if (ret < 0) {
+                                        fdctx->migration_failed = 1;
                                 } else {
-                                        /* nameless lookup has failed,
-                                         * it can be identified using
-                                         * fd->inode->table->xl
-                                         * != active_subvol. so, do
-                                         * nothing
-                                         */
-                                }
-                        } else {
-                                fdctx = fuse_fd_ctx_get (this, fd);
-                                if (fdctx != NULL) {
                                         fdctx->migration_failed = 0;
                                 }
                         }
@@ -3707,6 +3693,7 @@ fuse_handle_opened_fds (xlator_t *this, xlator_t *old_subvol,
                         if (fd)
                                 fd_unref (fd);
                 }
+
                 GF_FREE (fdentries);
         }
 
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index 3711ca54da0..c13c2dc76fe 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -350,8 +350,8 @@ inode_t *fuse_ino_to_inode (uint64_t ino, xlator_t *fuse);
 int send_fuse_err (xlator_t *this, fuse_in_header_t *finh, int error);
 int fuse_gfid_set (fuse_state_t *state);
 int fuse_flip_xattr_ns (struct fuse_private *priv, char *okey, char **nkey);
-fuse_fd_ctx_t * __fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this);
-fuse_fd_ctx_t * fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this);
+fuse_fd_ctx_t * __fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd);
+fuse_fd_ctx_t * fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd);
 
 int fuse_resolve_and_resume (fuse_state_t *state, fuse_resume_fn_t fn);
 int fuse_resolve_inode_init (fuse_state_t *state, fuse_resolve_t *resolve,
diff --git a/xlators/mount/fuse/src/fuse-resolve.c b/xlators/mount/fuse/src/fuse-resolve.c
index 5df0ac4b8be..d7446a71153 100644
--- a/xlators/mount/fuse/src/fuse-resolve.c
+++ b/xlators/mount/fuse/src/fuse-resolve.c
@@ -30,6 +30,8 @@ fuse_resolve_all (fuse_state_t *state);
 int fuse_resolve_continue (fuse_state_t *state);
 int fuse_resolve_entry_simple (fuse_state_t *state);
 int fuse_resolve_inode_simple (fuse_state_t *state);
+int fuse_migrate_fd (xlator_t *this, fd_t *fd, xlator_t *old_subvol,
+                     xlator_t *new_subvol);
 
 fuse_fd_ctx_t *
 fuse_fd_ctx_get (xlator_t *this, fd_t *fd);
@@ -334,29 +336,97 @@ fuse_resolve_inode (fuse_state_t *state)
         return 0;
 }
 
+
+int
+fuse_migrate_fd_task (void *data)
+{
+        int                     ret   = -1;
+        fuse_fd_ctx_t          *fdctx = NULL;
+        fuse_state_t           *state = NULL;
+
+        state = data;
+        if (state == NULL) {
+                goto out;
+        }
+
+        ret = fuse_migrate_fd (state->this, state->fd,
+                               state->fd->inode->table->xl,
+                               state->active_subvol);
+
+        fdctx = fuse_fd_ctx_check_n_create (state->this, state->fd);
+        if (fdctx != NULL) {
+                if (ret < 0) {
+                        fdctx->migration_failed = 1;
+                } else {
+                        fdctx->migration_failed = 0;
+                }
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+static inline int
+fuse_migrate_fd_error (xlator_t *this, fd_t *fd)
+{
+        fuse_fd_ctx_t *fdctx = NULL;
+        char           error = 0;
+
+        fdctx = fuse_fd_ctx_get (this, fd);
+        if (fdctx != NULL) {
+                if (fdctx->migration_failed) {
+                        error = 1;
+                }
+        }
+
+        return error;
+}
+
+
 static int
 fuse_resolve_fd (fuse_state_t *state)
 {
-        fuse_resolve_t *resolve       = NULL;
-	fd_t           *fd            = NULL;
-	xlator_t       *active_subvol = NULL;
-        fuse_fd_ctx_t  *fdctx         = NULL;
+        fuse_resolve_t         *resolve            = NULL;
+	fd_t                   *fd                 = NULL;
+	xlator_t               *active_subvol      = NULL;
+        int                     ret                = 0;
+        char                    fd_migration_error = 0;
 
         resolve = state->resolve_now;
 
         fd = resolve->fd;
 	active_subvol = fd->inode->table->xl;
 
-        if (state->active_subvol != active_subvol) {
+        fd_migration_error = fuse_migrate_fd_error (state->this, fd);
+        if (fd_migration_error) {
                 resolve->op_ret = -1;
                 resolve->op_errno = EBADF;
-        }
+        } else if (state->active_subvol != active_subvol) {
+                ret = synctask_new (state->this->ctx->env, fuse_migrate_fd_task,
+                                    NULL, NULL, state);
+
+                fd_migration_error = fuse_migrate_fd_error (state->this, fd);
+
+                if ((ret == -1) || fd_migration_error
+                    || (state->active_subvol != fd->inode->table->xl)) {
+                        if (ret == -1) {
+                                gf_log (state->this->name, GF_LOG_WARNING,
+                                        "starting sync-task to migrate fd (%p)"
+                                        " failed", fd);
+                        } else {
+                                gf_log (state->this->name, GF_LOG_WARNING,
+                                        "fd migration of fd (%p) failed", fd);
+                        }
 
-        fdctx = fuse_fd_ctx_get (state->this, fd);
-        if (fdctx != NULL) {
-                if (fdctx->migration_failed) {
                         resolve->op_ret = -1;
                         resolve->op_errno = EBADF;
+                } else {
+                        gf_log (state->this->name, GF_LOG_DEBUG,
+                                "fd (%p) migrated successfully in resolver",
+                                fd);
                 }
         }