client: move reopen{,dir} code to handshake file

and minor log level changes

Signed-off-by: Amar Tumballi <amar@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 875 (Implement a new protocol to provide proper backward/forward compatibility)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=875

5 files changed, 300 insertions, 312 deletions

diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 3dec4900605..26aabfeec70 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -367,6 +367,300 @@ client_notify_parents_child_up (xlator_t *this)
 }
 
 int
+client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                      void           *myframe)
+{
+        int32_t        ret                   = -1;
+        gfs3_open_rsp  rsp                   = {0,};
+        int            attempt_lock_recovery = _gf_false;
+        uint64_t       fd_count              = 0;
+        clnt_local_t  *local                 = NULL;
+        clnt_conf_t   *conf                  = NULL;
+        clnt_fd_ctx_t *fdctx                 = NULL;
+        call_frame_t  *frame                 = NULL;
+
+        frame = myframe;
+        local = frame->local;
+        conf  = frame->this->private;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_open_rsp (*iov, &rsp);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_ERROR, "error");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        gf_log (frame->this->name, GF_LOG_DEBUG,
+                "reopen on %s returned %d (%"PRId64")",
+                local->loc.path, rsp.op_ret, rsp.fd);
+
+        if (rsp.op_ret == -1)
+                goto out;
+
+        fdctx = local->fdctx;
+
+        if (!fdctx)
+                goto out;
+
+        pthread_mutex_lock (&conf->lock);
+        {
+                fdctx->remote_fd = rsp.fd;
+                if (!fdctx->released) {
+                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
+                        if (!list_empty (&fdctx->lock_list))
+                                attempt_lock_recovery = _gf_true;
+                        fdctx = NULL;
+                }
+        }
+        pthread_mutex_unlock (&conf->lock);
+
+        attempt_lock_recovery = _gf_false; /* temporarily */
+
+        if (attempt_lock_recovery) {
+                ret = client_attempt_lock_recovery (frame->this, local->fdctx);
+                if (ret < 0) {
+                        gf_log (frame->this->name, GF_LOG_DEBUG,
+                                "No locks on fd to recover");
+                } else {
+                        gf_log (frame->this->name, GF_LOG_DEBUG,
+                                "Need to attempt lock recovery on %lld open fds",
+                                (unsigned long long) fd_count);
+                }
+        } else {
+                fd_count = decrement_reopen_fd_count (frame->this, conf);
+        }
+
+
+out:
+        if (fdctx)
+                client_fdctx_destroy (frame->this, fdctx);
+
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
+
+        client_local_wipe (local);
+
+        return 0;
+}
+
+int
+client3_1_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                         void           *myframe)
+{
+        int32_t        ret   = -1;
+        gfs3_open_rsp  rsp   = {0,};
+        clnt_local_t  *local = NULL;
+        clnt_conf_t   *conf  = NULL;
+        clnt_fd_ctx_t *fdctx = NULL;
+        call_frame_t  *frame = NULL;
+
+        frame = myframe;
+        if (!frame || !frame->this)
+                goto out;
+
+        local        = frame->local;
+        frame->local = NULL;
+        conf         = frame->this->private;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_opendir_rsp (*iov, &rsp);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_ERROR, "error");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        gf_log (frame->this->name, GF_LOG_DEBUG,
+                "reopendir on %s returned %d (%"PRId64")",
+                local->loc.path, rsp.op_ret, rsp.fd);
+
+	if (-1 != rsp.op_ret) {
+                fdctx = local->fdctx;
+                if (fdctx) {
+                        pthread_mutex_lock (&conf->lock);
+                        {
+                                fdctx->remote_fd = rsp.fd;
+
+                                if (!fdctx->released) {
+                                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
+                                        fdctx = NULL;
+                                }
+                        }
+                        pthread_mutex_unlock (&conf->lock);
+                }
+        }
+
+        decrement_reopen_fd_count (frame->this, conf);
+
+out:
+        if (fdctx)
+                client_fdctx_destroy (frame->this, fdctx);
+
+        if (frame) {
+                frame->local = NULL;
+                STACK_DESTROY (frame->root);
+        }
+
+        client_local_wipe (local);
+
+        return 0;
+}
+
+int
+protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx)
+{
+        int               ret   = -1;
+        gfs3_opendir_req  req   = {{0,},};
+        clnt_local_t     *local = NULL;
+        inode_t          *inode = NULL;
+        char             *path  = NULL;
+        call_frame_t     *frame = NULL;
+        clnt_conf_t      *conf  = NULL;
+
+        if (!this || !fdctx)
+                goto out;
+
+        inode = fdctx->inode;
+        conf = this->private;
+
+        ret = inode_path (inode, NULL, &path);
+        if (ret < 0) {
+                goto out;
+        }
+
+        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        if (!local) {
+                goto out;
+        }
+
+        local->fdctx    = fdctx;
+        local->loc.path = path;
+        path            = NULL;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                goto out;
+        }
+
+        memcpy (req.gfid,  inode->gfid, 16);
+        req.path  = (char *)local->loc.path;
+
+        gf_log (frame->this->name, GF_LOG_DEBUG,
+                "attempting reopen on %s", local->loc.path);
+
+        frame->local = local; local = NULL;
+
+        ret = client_submit_request (this, &req, frame, conf->fops,
+                                     GFS3_OP_OPENDIR,
+                                     client3_1_reopendir_cbk, NULL,
+                                     xdr_from_opendir_req, NULL, 0, NULL, 0,
+                                     NULL);
+        if (ret)
+                goto out;
+
+        return ret;
+
+out:
+        if (frame) {
+                frame->local = NULL;
+                STACK_DESTROY (frame->root);
+        }
+
+        if (local)
+                client_local_wipe (local);
+
+        if (path)
+                GF_FREE (path);
+
+        return 0;
+
+}
+
+int
+protocol_client_reopen (xlator_t *this, clnt_fd_ctx_t *fdctx)
+{
+        int            ret   = -1;
+        gfs3_open_req  req   = {{0,},};
+        clnt_local_t  *local = NULL;
+        inode_t       *inode = NULL;
+        char          *path  = NULL;
+        call_frame_t  *frame = NULL;
+        clnt_conf_t   *conf  = NULL;
+
+        if (!this || !fdctx)
+                goto out;
+
+        inode = fdctx->inode;
+        conf  = this->private;
+
+        ret = inode_path (inode, NULL, &path);
+        if (ret < 0) {
+                goto out;
+        }
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                goto out;
+        }
+
+        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        if (!local) {
+                goto out;
+        }
+
+        local->fdctx    = fdctx;
+        local->loc.path = path;
+        path            = NULL;
+        frame->local    = local;
+
+        memcpy (req.gfid,  inode->gfid, 16);
+        req.flags    = gf_flags_from_flags (fdctx->flags);
+        req.wbflags  = fdctx->wbflags;
+        req.path     = (char *)local->loc.path;
+
+        gf_log (frame->this->name, GF_LOG_DEBUG,
+                "attempting reopen on %s", local->loc.path);
+
+        local = NULL;
+        ret = client_submit_request (this, &req, frame, conf->fops,
+                                     GFS3_OP_OPEN, client3_1_reopen_cbk, NULL,
+                                     xdr_from_open_req, NULL, 0, NULL, 0, NULL);
+        if (ret)
+                goto out;
+
+        return ret;
+
+out:
+        if (frame) {
+                frame->local = NULL;
+                STACK_DESTROY (frame->root);
+        }
+
+        if (local)
+                client_local_wipe (local);
+
+        if (path)
+                GF_FREE (path);
+
+        return 0;
+
+}
+
+
+int
 client_post_handshake (call_frame_t *frame, xlator_t *this)
 {
         clnt_conf_t            *conf = NULL;
@@ -399,7 +693,7 @@ client_post_handshake (call_frame_t *frame, xlator_t *this)
         /* Delay notifying CHILD_UP to parents
            until all locks are recovered */
         if (count > 0) {
-                gf_log (this->name, GF_LOG_TRACE,
+                gf_log (this->name, GF_LOG_INFO,
                         "%d fds open - Delaying child_up until they are re-opened",
                         count);
                 client_save_number_fds (conf, count);
@@ -413,7 +707,7 @@ client_post_handshake (call_frame_t *frame, xlator_t *this)
                                 protocol_client_reopen (this, fdctx);
                 }
         } else {
-                gf_log (this->name, GF_LOG_TRACE,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "No open fds - notifying all parents child up");
                 client_notify_parents_child_up (this);
 
diff --git a/xlators/protocol/client/src/client-helpers.c b/xlators/protocol/client/src/client-helpers.c
index 8c90fd9a961..4dd51257b6e 100644
--- a/xlators/protocol/client/src/client-helpers.c
+++ b/xlators/protocol/client/src/client-helpers.c
@@ -25,6 +25,7 @@
 #include "client.h"
 #include "fd.h"
 
+
 clnt_fd_ctx_t *
 this_fd_del_ctx (fd_t *file, xlator_t *this)
 {
diff --git a/xlators/protocol/client/src/client-lk.c b/xlators/protocol/client/src/client-lk.c
index 386af4a07eb..ee595225735 100644
--- a/xlators/protocol/client/src/client-lk.c
+++ b/xlators/protocol/client/src/client-lk.c
@@ -614,7 +614,7 @@ decrement_reopen_fd_count (xlator_t *this, clnt_conf_t *conf)
 
         if (fd_count == 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
-                        "All locks healed on the last fd - notifying CHILDUP");
+                        "last fd open'd/lock-self-heal'd - notifying CHILD-UP");
                 client_notify_parents_child_up (this);
         }
 
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 39dfc36ea89..e353bb68a57 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -99,7 +99,6 @@ typedef struct client_local {
         uint32_t           flags;
         uint32_t           wbflags;
         struct iobref     *iobref;
-        fop_cbk_fn_t       op;
 
         client_posix_lock_t *client_lock;
         uint64_t           owner;
@@ -178,4 +177,6 @@ int client_notify_parents_child_up (xlator_t *this);
 int32_t is_client_dump_locks_cmd (char *name);
 int32_t client_dump_locks (char *name, inode_t *inode,
                            dict_t *dict);
+int client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx);
+
 #endif /* !_CLIENT_H */
diff --git a/xlators/protocol/client/src/client3_1-fops.c b/xlators/protocol/client/src/client3_1-fops.c
index 6c48421f413..6d20e342961 100644
--- a/xlators/protocol/client/src/client3_1-fops.c
+++ b/xlators/protocol/client/src/client3_1-fops.c
@@ -276,11 +276,6 @@ client3_1_open_cbk (struct rpc_req *req, struct iovec *iov, int count,
         frame = myframe;
         local = frame->local;
 
-        if (local->op) {
-                local->op (req, iov, 1, myframe);
-                return 0;
-        }
-
         frame->local = NULL;
         conf  = frame->this->private;
         fd    = local->fd;
@@ -1777,11 +1772,6 @@ client3_1_opendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
         frame = myframe;
         local = frame->local;
 
-        if (local->op) {
-                local->op (req, iov, 1, myframe);
-                return 0;
-        }
-
         frame->local = NULL;
         conf  = frame->this->private;
         fd    = local->fd;
@@ -2046,304 +2036,6 @@ out:
         return ret;
 }
 
-int
-client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                      void           *myframe)
-{
-        int32_t        ret                   = -1;
-        gfs3_open_rsp  rsp                   = {0,};
-        int            attempt_lock_recovery = _gf_false;
-        uint64_t       fd_count              = 0;
-        clnt_local_t  *local                 = NULL;
-        clnt_conf_t   *conf                  = NULL;
-        clnt_fd_ctx_t *fdctx                 = NULL;
-        call_frame_t  *frame                 = NULL;
-
-        frame = myframe;
-        local = frame->local;
-        conf  = frame->this->private;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_open_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "reopen on %s returned %d (%"PRId64")",
-                local->loc.path, rsp.op_ret, rsp.fd);
-
-        if (rsp.op_ret == -1)
-                goto out;
-
-        fdctx = local->fdctx;
-
-        if (!fdctx)
-                goto out;
-
-        pthread_mutex_lock (&conf->lock);
-        {
-                fdctx->remote_fd = rsp.fd;
-                if (!fdctx->released) {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                        if (!list_empty (&fdctx->lock_list))
-                                attempt_lock_recovery = _gf_true;
-                        fdctx = NULL;
-                }
-        }
-        pthread_mutex_unlock (&conf->lock);
-
-        attempt_lock_recovery = _gf_false; /* temporarily */
-
-        if (attempt_lock_recovery) {
-                ret = client_attempt_lock_recovery (frame->this, local->fdctx);
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "No locks on fd to recover");
-                } else {
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "Need to attempt lock recovery on %lld open fds",
-                                (unsigned long long) fd_count);
-                }
-        } else {
-                fd_count = decrement_reopen_fd_count (frame->this, conf);
-        }
-
-
-out:
-        if (fdctx)
-                client_fdctx_destroy (frame->this, fdctx);
-
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-int
-client3_1_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                         void           *myframe)
-{
-        int32_t        ret   = -1;
-        gfs3_open_rsp  rsp   = {0,};
-        clnt_local_t  *local = NULL;
-        clnt_conf_t   *conf  = NULL;
-        clnt_fd_ctx_t *fdctx = NULL;
-        call_frame_t  *frame = NULL;
-
-        frame = myframe;
-        if (!frame || !frame->this)
-                goto out;
-
-        local        = frame->local;
-        frame->local = NULL;
-        conf         = frame->this->private;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_opendir_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "reopendir on %s returned %d (%"PRId64")",
-                local->loc.path, rsp.op_ret, rsp.fd);
-
-	if (-1 != rsp.op_ret) {
-	  fdctx = local->fdctx;
-	  if (fdctx) {
-                pthread_mutex_lock (&conf->lock);
-                {
-                        fdctx->remote_fd = rsp.fd;
-
-                        if (!fdctx->released) {
-                                list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                                fdctx = NULL;
-                        }
-                }
-                pthread_mutex_unlock (&conf->lock);
-
-	  }
-        }
-
-        decrement_reopen_fd_count (frame->this, conf);
-
-out:
-        if (fdctx)
-                client_fdctx_destroy (frame->this, fdctx);
-
-        if (frame) {
-                frame->local = NULL;
-                STACK_DESTROY (frame->root);
-        }
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-int
-protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx)
-{
-        int               ret   = -1;
-        gfs3_opendir_req  req   = {{0,},};
-        clnt_local_t     *local = NULL;
-        inode_t          *inode = NULL;
-        char             *path  = NULL;
-        call_frame_t     *frame = NULL;
-        clnt_conf_t      *conf  = NULL;
-
-        if (!this || !fdctx)
-                goto out;
-
-        inode = fdctx->inode;
-        conf = this->private;
-
-        ret = inode_path (inode, NULL, &path);
-        if (ret < 0) {
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
-        if (!local) {
-                goto out;
-        }
-
-        local->fdctx    = fdctx;
-        local->op       = client3_1_reopendir_cbk;
-        local->loc.path = path;
-        path            = NULL;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                goto out;
-        }
-
-        memcpy (req.gfid,  inode->gfid, 16);
-        req.path  = (char *)local->loc.path;
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "attempting reopen on %s", local->loc.path);
-
-        frame->local = local; local = NULL;
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_OPENDIR,
-                                     client3_1_opendir_cbk, NULL,
-                                     xdr_from_opendir_req, NULL, 0, NULL, 0,
-                                     NULL);
-        if (ret)
-                goto out;
-
-        return ret;
-
-out:
-        if (frame) {
-                frame->local = NULL;
-                STACK_DESTROY (frame->root);
-        }
-
-        if (local)
-                client_local_wipe (local);
-
-        if (path)
-                GF_FREE (path);
-
-        return 0;
-
-}
-
-int
-protocol_client_reopen (xlator_t *this, clnt_fd_ctx_t *fdctx)
-{
-        int            ret   = -1;
-        gfs3_open_req  req   = {{0,},};
-        clnt_local_t  *local = NULL;
-        inode_t       *inode = NULL;
-        char          *path  = NULL;
-        call_frame_t  *frame = NULL;
-        clnt_conf_t   *conf  = NULL;
-
-        if (!this || !fdctx)
-                goto out;
-
-        inode = fdctx->inode;
-        conf  = this->private;
-
-        ret = inode_path (inode, NULL, &path);
-        if (ret < 0) {
-                goto out;
-        }
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
-        if (!local) {
-                goto out;
-        }
-
-        local->fdctx    = fdctx;
-        local->op       = client3_1_reopen_cbk;
-        local->loc.path = path;
-        path            = NULL;
-        frame->local    = local;
-
-        memcpy (req.gfid,  inode->gfid, 16);
-        req.flags    = gf_flags_from_flags (fdctx->flags);
-        req.wbflags  = fdctx->wbflags;
-        req.path     = (char *)local->loc.path;
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "attempting reopen on %s", local->loc.path);
-
-        local = NULL;
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_OPEN, client3_1_open_cbk, NULL,
-                                     xdr_from_open_req, NULL, 0, NULL, 0, NULL);
-        if (ret)
-                goto out;
-
-        return ret;
-
-out:
-        if (frame) {
-                frame->local = NULL;
-                STACK_DESTROY (frame->root);
-        }
-
-        if (local)
-                client_local_wipe (local);
-
-        if (path)
-                GF_FREE (path);
-
-        return 0;
-
-}
-
-
-
 int32_t
 client3_1_releasedir (call_frame_t *frame, xlator_t *this,
                       void *data)