protocol/client: Periodically attempt reopens

If the brick is taken down and the hard disk is replaced
and the brick is brought back up, the re-opens of the open-fds
will fail because the file is not present on the brick.
Re-opens are not attempted even if the files are re-created by
self-heal until the brick is brought down after the files are
re-created and brought back up. This is a problem with a VM-store
in a replica-setup.  Until the fd is re-opened the writes will
never happen on the brick where the hard-disk is replaced.
To handle this situation gracefully, client xlator is enhanced
to perform finodelk, fxattrop, writev, readv using anonymous fds
if the file is yet to be re-opened. If the fop succeeds then client
xlator attempts re-open.

Change-Id: I1cc6d1bbf8227cd996868ab2ed0a57fb05e00017
BUG: 821056
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/4358
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>

6 files changed, 265 insertions, 93 deletions

diff --git a/libglusterfs/src/fd.h b/libglusterfs/src/fd.h
index 1077cfa8632..54290b1980b 100644
--- a/libglusterfs/src/fd.h
+++ b/libglusterfs/src/fd.h
@@ -24,6 +24,8 @@
 #include "fd-lk.h"
 #include "common-utils.h"
 
+#define GF_ANON_FD_NO -2
+
 struct _inode;
 struct _dict;
 struct fd_lk_ctx;
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 7862635fcc6..75c58afe8ba 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -25,6 +25,7 @@
 #include "portmap-xdr.h"
 #include "rpc-common-xdr.h"
 
+#define CLIENT_REOPEN_MAX_ATTEMPTS 1024
 extern rpc_clnt_prog_t clnt3_3_fop_prog;
 extern rpc_clnt_prog_t clnt_pmap_prog;
 
@@ -886,22 +887,16 @@ client_default_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
 }
 
 void
-client_child_up_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
+client_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
         clnt_conf_t  *conf    = NULL;
-        uint64_t     fd_count = 0;
         gf_boolean_t destroy  = _gf_false;
 
         conf = this->private;
 
-        LOCK (&conf->rec_lock);
-        {
-                fd_count = --(conf->reopen_fd_count);
-        }
-        UNLOCK (&conf->rec_lock);
-
         pthread_mutex_lock (&conf->lock);
         {
+                fdctx->reopen_attempts = 0;
                 if (!fdctx->released)
                         list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
                 else
@@ -910,14 +905,31 @@ client_child_up_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
         }
         pthread_mutex_unlock (&conf->lock);
 
+        if (destroy)
+                client_fdctx_destroy (this, fdctx);
+}
+
+void
+client_child_up_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
+{
+        clnt_conf_t  *conf    = NULL;
+        uint64_t     fd_count = 0;
+
+        conf = this->private;
+
+        LOCK (&conf->rec_lock);
+        {
+                fd_count = --(conf->reopen_fd_count);
+        }
+        UNLOCK (&conf->rec_lock);
+
+        client_reopen_done (fdctx, this);
         if (fd_count == 0) {
                 gf_log (this->name, GF_LOG_INFO,
                         "last fd open'd/lock-self-heal'd - notifying CHILD-UP");
                 client_set_lk_version (this);
                 client_notify_parents_child_up (this);
         }
-        if (destroy)
-                client_fdctx_destroy (this, fdctx);
 }
 
 int
@@ -1075,8 +1087,8 @@ out:
         return 0;
 }
 
-int
-protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx)
+static int
+protocol_client_reopendir (clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
         int               ret   = -1;
         gfs3_opendir_req  req   = {{0,},};
@@ -1139,8 +1151,8 @@ out:
 
 }
 
-int
-protocol_client_reopen (xlator_t *this, clnt_fd_ctx_t *fdctx)
+static int
+protocol_client_reopenfile (clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
         int            ret   = -1;
         gfs3_open_req  req   = {{0,},};
@@ -1204,6 +1216,60 @@ out:
 
 }
 
+static void
+protocol_client_reopen (clnt_fd_ctx_t *fdctx, xlator_t *this)
+{
+        if (fdctx->is_dir)
+                protocol_client_reopendir (fdctx, this);
+        else
+                protocol_client_reopenfile (fdctx, this);
+}
+
+gf_boolean_t
+__is_fd_reopen_in_progress (clnt_fd_ctx_t *fdctx)
+{
+        if (fdctx->reopen_done == client_default_reopen_done)
+                return _gf_false;
+        return _gf_true;
+}
+
+void
+client_attempt_reopen (fd_t *fd, xlator_t *this)
+{
+        clnt_conf_t   *conf  = NULL;
+        clnt_fd_ctx_t *fdctx = NULL;
+        gf_boolean_t  reopen = _gf_false;
+
+        if (!fd || !this)
+                goto out;
+
+        conf = this->private;
+        pthread_mutex_lock (&conf->lock);
+        {
+                fdctx = this_fd_get_ctx (fd, this);
+                if (!fdctx)
+                        goto unlock;
+                if (__is_fd_reopen_in_progress (fdctx))
+                        goto unlock;
+                if (fdctx->remote_fd != -1)
+                        goto unlock;
+
+                if (fdctx->reopen_attempts == CLIENT_REOPEN_MAX_ATTEMPTS) {
+                        reopen = _gf_true;
+                        fdctx->reopen_done = client_reopen_done;
+                        list_del_init (&fdctx->sfd_pos);
+                } else {
+                        fdctx->reopen_attempts++;
+                }
+        }
+unlock:
+        pthread_mutex_unlock (&conf->lock);
+        if (reopen)
+                protocol_client_reopen (fdctx, this);
+out:
+        return;
+}
+
 int
 client_post_handshake (call_frame_t *frame, xlator_t *this)
 {
@@ -1246,10 +1312,7 @@ client_post_handshake (call_frame_t *frame, xlator_t *this)
                 list_for_each_entry_safe (fdctx, tmp, &reopen_head, sfd_pos) {
                         list_del_init (&fdctx->sfd_pos);
 
-                        if (fdctx->is_dir)
-                                protocol_client_reopendir (this, fdctx);
-                        else
-                                protocol_client_reopen (this, fdctx);
+                        protocol_client_reopen (fdctx, this);
                 }
         } else {
                 gf_log (this->name, GF_LOG_DEBUG,
diff --git a/xlators/protocol/client/src/client-helpers.c b/xlators/protocol/client/src/client-helpers.c
index 77b416c476c..5d9f00fdc70 100644
--- a/xlators/protocol/client/src/client-helpers.c
+++ b/xlators/protocol/client/src/client-helpers.c
@@ -16,7 +16,6 @@
 #include "client.h"
 #include "fd.h"
 
-
 int
 client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx, gf_boolean_t try_lock)
 {
@@ -276,3 +275,75 @@ clnt_readdir_rsp_cleanup (gfs3_readdir_rsp *rsp)
 
         return 0;
 }
+
+int
+client_get_remote_fd (xlator_t *this, fd_t *fd, int flags, int64_t *remote_fd)
+{
+        clnt_fd_ctx_t *fdctx    = NULL;
+        clnt_conf_t   *conf     = NULL;
+
+        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+        GF_VALIDATE_OR_GOTO (this->name, remote_fd, out);
+
+        conf = this->private;
+        pthread_mutex_lock (&conf->lock);
+        {
+                fdctx = this_fd_get_ctx (fd, this);
+                if (!fdctx)
+                        *remote_fd = GF_ANON_FD_NO;
+                else if (__is_fd_reopen_in_progress (fdctx))
+                        *remote_fd = -1;
+                else
+                        *remote_fd = fdctx->remote_fd;
+        }
+        pthread_mutex_unlock (&conf->lock);
+
+        if ((flags & FALLBACK_TO_ANON_FD) && (*remote_fd == -1))
+                *remote_fd = GF_ANON_FD_NO;
+
+        return 0;
+out:
+        return -1;
+}
+
+gf_boolean_t
+client_is_reopen_needed (fd_t *fd, xlator_t *this, int64_t remote_fd)
+{
+        clnt_fd_ctx_t   *fdctx = NULL;
+
+        fdctx = this_fd_get_ctx (fd, this);
+        if (fdctx && (fdctx->remote_fd == -1) &&
+            (remote_fd == GF_ANON_FD_NO))
+                return _gf_true;
+        return _gf_false;
+}
+
+int
+client_fd_fop_prepare_local (call_frame_t *frame, fd_t *fd, int64_t remote_fd)
+{
+        xlator_t     *this  = NULL;
+        clnt_conf_t  *conf  = NULL;
+        clnt_local_t *local = NULL;
+        int          ret    = 0;
+
+        this = frame->this;
+        conf = this->private;
+
+        if (!frame || !fd) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        frame->local = mem_get0 (this->local_pool);
+        if (frame->local == NULL) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        local = frame->local;
+        local->fd = fd_ref (fd);
+        local->attempt_reopen = client_is_reopen_needed (fd, this, remote_fd);
+        return 0;
+out:
+        return ret;
+}
diff --git a/xlators/protocol/client/src/client-rpc-fops.c b/xlators/protocol/client/src/client-rpc-fops.c
index 36c05ed17c7..f524c1a373a 100644
--- a/xlators/protocol/client/src/client-rpc-fops.c
+++ b/xlators/protocol/client/src/client-rpc-fops.c
@@ -836,11 +836,13 @@ client3_3_writev_cbk (struct rpc_req *req, struct iovec *iov, int count,
         int ret = 0;
         xlator_t *this       = NULL;
         dict_t  *xdata       = NULL;
+        clnt_local_t    *local = NULL;
 
 
         this = THIS;
 
         frame = myframe;
+        local = frame->local;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -869,6 +871,9 @@ out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
+        } else if (rsp.op_ret >= 0) {
+                if (local->attempt_reopen)
+                        client_attempt_reopen (local->fd, this);
         }
         CLIENT_STACK_UNWIND (writev, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
@@ -1547,16 +1552,17 @@ int
 client3_3_finodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
+        call_frame_t  *frame = NULL;
+        gf_common_rsp rsp    = {0,};
+        int           ret    = 0;
+        xlator_t      *this  = NULL;
+        dict_t        *xdata = NULL;
+        clnt_local_t  *local = NULL;
 
-        this = THIS;
 
         frame = myframe;
+        this = frame->this;
+        local = frame->local;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -1580,6 +1586,9 @@ out:
             (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
+        } else if (rsp.op_ret == 0) {
+                if (local->attempt_reopen)
+                        client_attempt_reopen (local->fd, this);
         }
         CLIENT_STACK_UNWIND (finodelk, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), xdata);
@@ -1811,6 +1820,9 @@ out:
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s",
                         strerror (gf_error_to_errno (op_errno)));
+        } else if (rsp.op_ret == 0) {
+                if (local->attempt_reopen)
+                        client_attempt_reopen (local->fd, this);
         }
         CLIENT_STACK_UNWIND (fxattrop, frame, rsp.op_ret,
                              gf_error_to_errno (op_errno), dict, xdata);
@@ -2683,6 +2695,9 @@ out:
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
+        } else if (rsp.op_ret >= 0) {
+                if (local->attempt_reopen)
+                        client_attempt_reopen (local->fd, this);
         }
         CLIENT_STACK_UNWIND (readv, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), vector, rspcount,
@@ -3119,7 +3134,8 @@ client3_3_ftruncate (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.offset = args->offset;
         req.fd     = remote_fd;
@@ -3879,13 +3895,13 @@ client3_3_readv (call_frame_t *frame, xlator_t *this,
         clnt_args_t    *args       = NULL;
         int64_t         remote_fd  = -1;
         clnt_conf_t    *conf       = NULL;
+        clnt_local_t   *local      = NULL;
         int             op_errno   = ESTALE;
         gfs3_read_req   req        = {{0,},};
         int             ret        = 0;
         struct iovec    rsp_vec    = {0, };
         struct iobuf   *rsp_iobuf  = NULL;
         struct iobref  *rsp_iobref = NULL;
-        clnt_local_t   *local      = NULL;
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -3893,7 +3909,14 @@ client3_3_readv (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
+                              remote_fd, op_errno, unwind);
+        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
+        if (ret) {
+                op_errno = -ret;
+                goto unwind;
+        }
+        local = frame->local;
 
         req.size   = args->size;
         req.offset = args->offset;
@@ -3931,15 +3954,8 @@ client3_3_readv (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
         local->iobref = rsp_iobref;
         rsp_iobref = NULL;
-        frame->local = local;
 
         GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
                                     req.xdata.xdata_len, op_errno, unwind);
@@ -3950,17 +3966,12 @@ client3_3_readv (call_frame_t *frame, xlator_t *this,
                                      local->iobref,
                                      (xdrproc_t)xdr_gfs3_read_req);
         if (ret) {
+                //unwind is done in the cbk
                 gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
         GF_FREE (req.xdata.xdata_val);
 
-        if (rsp_iobuf)
-                iobuf_unref (rsp_iobuf);
-
-        if (rsp_iobref)
-                iobref_unref (rsp_iobref);
-
         return 0;
 unwind:
         if (rsp_iobuf)
@@ -3984,7 +3995,7 @@ client3_3_writev (call_frame_t *frame, xlator_t *this, void *data)
         clnt_conf_t    *conf     = NULL;
         gfs3_write_req  req      = {{0,},};
         int             op_errno = ESTALE;
-        int             ret        = 0;
+        int             ret      = 0;
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -3992,7 +4003,13 @@ client3_3_writev (call_frame_t *frame, xlator_t *this, void *data)
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
+                              remote_fd, op_errno, unwind);
+        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
+        if (ret) {
+                op_errno = -ret;
+                goto unwind;
+        }
 
         req.size   = args->size;
         req.offset = args->offset;
@@ -4056,7 +4073,8 @@ client3_3_flush (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         conf = this->private;
 
@@ -4115,7 +4133,8 @@ client3_3_fsync (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd   = remote_fd;
         req.data = args->flags;
@@ -4163,7 +4182,8 @@ client3_3_fstat (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd = remote_fd;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
@@ -4272,7 +4292,8 @@ client3_3_fsyncdir (call_frame_t *frame, xlator_t *this, void *data)
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd   = remote_fd;
         req.data = args->flags;
@@ -4441,7 +4462,8 @@ client3_3_fsetxattr (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd    = remote_fd;
         req.flags = args->flags;
@@ -4505,7 +4527,8 @@ client3_3_fgetxattr (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         local = mem_get0 (this->local_pool);
         if (!local) {
@@ -4848,11 +4871,11 @@ client3_3_fxattrop (call_frame_t *frame, xlator_t *this,
         clnt_args_t       *args       = NULL;
         int64_t            remote_fd  = -1;
         clnt_conf_t       *conf       = NULL;
+        clnt_local_t      *local      = NULL;
         gfs3_fxattrop_req  req        = {{0,},};
         int                op_errno   = ESTALE;
         int                ret        = 0;
         int                count      = 0;
-        clnt_local_t    *local      = NULL;
         struct iobref     *rsp_iobref = NULL;
         struct iobuf      *rsp_iobuf  = NULL;
         struct iovec      *rsphdr     = NULL;
@@ -4864,19 +4887,20 @@ client3_3_fxattrop (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
+                              remote_fd, op_errno, unwind);
+        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
+        if (ret) {
+                op_errno = -ret;
+                goto unwind;
+        }
+
+        local = frame->local;
 
         req.fd     = remote_fd;
         req.flags  = args->flags;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        frame->local = local;
-
         rsp_iobref = iobref_new ();
         if (rsp_iobref == NULL) {
                 op_errno = ENOMEM;
@@ -4924,12 +4948,6 @@ client3_3_fxattrop (call_frame_t *frame, xlator_t *this,
 
         GF_FREE (req.xdata.xdata_val);
 
-        if (rsp_iobuf)
-                iobuf_unref (rsp_iobuf);
-
-        if (rsp_iobref)
-                iobref_unref (rsp_iobref);
-
         return 0;
 unwind:
         CLIENT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
@@ -5021,7 +5039,8 @@ client3_3_fremovexattr (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         memcpy (req.gfid,  args->fd->inode->gfid, 16);
         req.name = (char *)args->name;
@@ -5074,7 +5093,8 @@ client3_3_lk (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         ret = client_cmd_to_gf_cmd (args->cmd, &gf_cmd);
         if (ret) {
@@ -5224,15 +5244,20 @@ client3_3_finodelk (call_frame_t *frame, xlator_t *this,
         int64_t            remote_fd = -1;
         clnt_conf_t       *conf     = NULL;
         int                op_errno = ESTALE;
-        int           ret        = 0;
+        int                ret      = 0;
 
         if (!frame || !this || !data)
                 goto unwind;
 
         args = data;
         conf = this->private;
-
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
+                              remote_fd, op_errno, unwind);
+        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
+        if (ret) {
+                op_errno = -ret;
+                goto unwind;
+        }
 
         if (args->cmd == F_GETLK || args->cmd == F_GETLK64)
                 gf_cmd = GF_LK_GETLK;
@@ -5365,7 +5390,8 @@ client3_3_fentrylk (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd  = remote_fd;
         req.cmd = args->cmd_entrylk;
@@ -5418,7 +5444,8 @@ client3_3_rchecksum (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.len    = args->len;
         req.offset = args->offset;
@@ -5474,7 +5501,8 @@ client3_3_readdir (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         readdir_rsp_size = xdr_sizeof ((xdrproc_t) xdr_gfs3_readdir_rsp, &rsp)
                 + args->size;
@@ -5583,7 +5611,8 @@ client3_3_readdirp (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         readdirp_rsp_size = xdr_sizeof ((xdrproc_t) xdr_gfs3_readdirp_rsp, &rsp)
                 + args->size;
@@ -5736,7 +5765,8 @@ client3_3_fsetattr (call_frame_t *frame, xlator_t *this, void *data)
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd = remote_fd;
         req.valid = args->valid;
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 03e7f838739..0a27c095c88 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -34,27 +34,27 @@ typedef enum {
         GF_LK_HEAL_DONE,
 } lk_heal_state_t;
 
-#define CLIENT_GET_REMOTE_FD(conf, fd, remote_fd, op_errno, label)      \
+typedef enum {
+        DEFAULT_REMOTE_FD = 0,
+        FALLBACK_TO_ANON_FD = 1
+} clnt_remote_fd_flags_t;
+
+#define CLIENT_GET_REMOTE_FD(xl, fd, flags, remote_fd, op_errno, label) \
         do {                                                            \
-                clnt_fd_ctx_t      *fdctx    = NULL;                    \
-                pthread_mutex_lock (&conf->lock);                       \
-                {                                                       \
-                        fdctx = this_fd_get_ctx (fd, THIS);             \
-                }                                                       \
-                pthread_mutex_unlock (&conf->lock);                     \
-                if (!fdctx) {                                           \
-                        remote_fd = -2;                                 \
-                } else {                                                \
-                        remote_fd = fdctx->remote_fd;                   \
+                int     _ret    = 0;                                    \
+                _ret = client_get_remote_fd (xl, fd, flags, &remote_fd);\
+                if (_ret < 0) {                                         \
+                        op_errno = errno;                               \
+                        goto label;                                     \
                 }                                                       \
                 if (remote_fd == -1) {                                  \
-                        gf_log (THIS->name, GF_LOG_WARNING, " (%s) "    \
+                        gf_log (xl->name, GF_LOG_WARNING, " (%s) "      \
                                 "remote_fd is -1. EBADFD",              \
                                 uuid_utoa (fd->inode->gfid));           \
                         op_errno = EBADFD;                              \
                         goto label;                                     \
                 }                                                       \
-        } while (0);
+        } while (0)
 
 #define CLIENT_STACK_UNWIND(op, frame, params ...) do {             \
                 clnt_local_t *__local = frame->local;               \
@@ -132,6 +132,7 @@ typedef struct _client_fd_ctx {
         uuid_t            gfid;
         void (*reopen_done) (struct _client_fd_ctx*, xlator_t *);
         struct list_head  lock_list;     /* List of all granted locks on this fd */
+        int32_t           reopen_attempts;
 } clnt_fd_ctx_t;
 
 typedef struct _client_posix_lock {
@@ -159,7 +160,8 @@ typedef struct client_local {
         int32_t              cmd;
         struct list_head     lock_list;
         pthread_mutex_t      mutex;
-        char           *name;
+        char                *name;
+        gf_boolean_t         attempt_reopen;
 } clnt_local_t;
 
 typedef struct client_args {
@@ -211,9 +213,6 @@ int client_submit_request (xlator_t *this, void *req,
                            struct iovec *rsp_payload, int rsp_count,
                            struct iobref *rsp_iobref, xdrproc_t xdrproc);
 
-int protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx);
-int protocol_client_reopen (xlator_t *this, clnt_fd_ctx_t *fdctx);
-
 int unserialize_rsp_dirent (struct gfs3_readdir_rsp *rsp, gf_dirent_t *entries);
 int unserialize_rsp_direntp (xlator_t *this, fd_t *fd,
                              struct gfs3_readdirp_rsp *rsp, gf_dirent_t *entries);
@@ -244,4 +243,11 @@ int client_set_lk_version (xlator_t *this);
 
 int client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx, gf_boolean_t use_try_lock);
 void client_default_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this);
+void client_attempt_reopen (fd_t *fd, xlator_t *this);
+int client_get_remote_fd (xlator_t *this, fd_t *fd, int flags,
+                          int64_t *remote_fd);
+int client_fd_fop_prepare_local (call_frame_t *frame, fd_t *fd,
+                                 int64_t remote_fd);
+gf_boolean_t
+__is_fd_reopen_in_progress (clnt_fd_ctx_t *fdctx);
 #endif /* !_CLIENT_H */
diff --git a/xlators/protocol/server/src/server-resolve.c b/xlators/protocol/server/src/server-resolve.c
index 11b48818725..159f6386d32 100644
--- a/xlators/protocol/server/src/server-resolve.c
+++ b/xlators/protocol/server/src/server-resolve.c
@@ -469,7 +469,7 @@ server_resolve_fd (call_frame_t *frame)
 
         fd_no = resolve->fd_no;
 
-        if (fd_no == -2) {
+        if (fd_no == GF_ANON_FD_NO) {
                 server_resolve_anonfd (frame);
                 return 0;
         }