From 3e78ea991b213422fc423ff94994e1eb295569c7 Mon Sep 17 00:00:00 2001 From: Anoop C S Date: Thu, 15 Oct 2015 15:04:34 +0530 Subject: protocol: Remove lock recovery logic from client and server Change-Id: I27f5e1e34fe3eac96c7dd88e90753fb5d3d14550 BUG: 1272030 Signed-off-by: Anoop C S --- xlators/protocol/client/src/client-handshake.c | 365 +---------------------- xlators/protocol/client/src/client-rpc-fops.c | 19 +- xlators/protocol/client/src/client-rpc-fops_v2.c | 18 +- xlators/protocol/client/src/client.c | 177 +---------- xlators/protocol/client/src/client.h | 24 -- xlators/protocol/server/src/server-handshake.c | 81 +---- xlators/protocol/server/src/server-helpers.c | 44 --- xlators/protocol/server/src/server-helpers.h | 3 - xlators/protocol/server/src/server.c | 200 ++----------- xlators/protocol/server/src/server.h | 5 - 10 files changed, 36 insertions(+), 900 deletions(-) (limited to 'xlators/protocol') diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c index 2d779fa39b7..fe6e923c077 100644 --- a/xlators/protocol/client/src/client-handshake.c +++ b/xlators/protocol/client/src/client-handshake.c @@ -27,11 +27,6 @@ extern rpc_clnt_prog_t clnt3_3_fop_prog; extern rpc_clnt_prog_t clnt4_0_fop_prog; extern rpc_clnt_prog_t clnt_pmap_prog; -int client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov, - int count, void *myframe); - -int client_set_lk_version (xlator_t *this); - typedef struct client_fd_lk_local { gf_atomic_t ref; gf_boolean_t error; @@ -168,7 +163,6 @@ clnt_fd_lk_reacquire_failed (xlator_t *this, clnt_fd_ctx_t *fdctx, pthread_spin_lock (&conf->fd_lock); { fdctx->remote_fd = -1; - fdctx->lk_heal_state = GF_LK_HEAL_DONE; } pthread_spin_unlock (&conf->fd_lock); @@ -177,90 +171,6 @@ out: return ret; } -int -client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov, - int count, void *myframe) -{ - int32_t ret = -1; - call_frame_t *fr = NULL; - gf_set_lk_ver_rsp rsp = {0,}; - - fr = (call_frame_t *) myframe; - GF_VALIDATE_OR_GOTO ("client", fr, out); - - if (req->rpc_status == -1) { - gf_msg (fr->this->name, GF_LOG_WARNING, ENOTCONN, - PC_MSG_RPC_STATUS_ERROR, "received RPC status error"); - goto out; - } - - ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_set_lk_ver_rsp); - if (ret < 0) - gf_msg (fr->this->name, GF_LOG_WARNING, 0, - PC_MSG_XDR_DECODING_FAILED, "xdr decoding failed"); - else - gf_msg (fr->this->name, GF_LOG_INFO, 0, - PC_MSG_LOCK_VERSION_SERVER, - "Server lk version = %d", rsp.lk_ver); - - ret = 0; -out: - if (fr) - STACK_DESTROY (fr->root); - - return ret; -} - -//TODO: Check for all released fdctx and destroy them -int -client_set_lk_version (xlator_t *this) -{ - int ret = -1; - clnt_conf_t *conf = NULL; - call_frame_t *frame = NULL; - gf_set_lk_ver_req req = {0, }; - char *process_uuid = NULL; - - GF_VALIDATE_OR_GOTO ("client", this, err); - - conf = (clnt_conf_t *) this->private; - - req.lk_ver = client_get_lk_ver (conf); - ret = dict_get_str (this->options, "process-uuid", &process_uuid); - if (!process_uuid) { - ret = -1; - goto err; - } - req.uid = gf_strdup (process_uuid); - if (!req.uid) { - ret = -1; - goto err; - } - - frame = create_frame (this, this->ctx->pool); - if (!frame) { - ret = -1; - goto out; - } - - gf_msg_debug (this->name, 0, "Sending SET_LK_VERSION"); - - ret = client_submit_request (this, &req, frame, - conf->handshake, - GF_HNDSK_SET_LK_VER, - client_set_lk_version_cbk, - NULL, NULL, 0, NULL, 0, NULL, - (xdrproc_t)xdr_gf_set_lk_ver_req); -out: - GF_FREE (req.uid); - return ret; -err: - gf_msg (this->name, GF_LOG_WARNING, 0, PC_MSG_SET_LK_VERSION_ERROR, - "Failed to send SET_LK_VERSION to server"); - - return ret; -} - int client_fd_lk_count (fd_lk_ctx_t *lk_ctx) { @@ -440,186 +350,6 @@ out: return ret; } -int -client_reacquire_lock_cbk (struct rpc_req *req, struct iovec *iov, - int count, void *myframe) -{ - int32_t ret = -1; - xlator_t *this = NULL; - gfs3_lk_rsp rsp = {0,}; - call_frame_t *frame = NULL; - clnt_conf_t *conf = NULL; - clnt_fd_ctx_t *fdctx = NULL; - clnt_fd_lk_local_t *local = NULL; - struct gf_flock lock = {0,}; - - frame = (call_frame_t *) myframe; - this = frame->this; - local = (clnt_fd_lk_local_t *) frame->local; - conf = (clnt_conf_t *) this->private; - - if (req->rpc_status == -1) { - gf_msg ("client", GF_LOG_WARNING, 0, PC_MSG_CLIENT_REQ_FAIL, - "request failed at rpc"); - goto out; - } - - ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_lk_rsp); - if (ret < 0) { - gf_msg (this->name, GF_LOG_ERROR, EINVAL, - PC_MSG_XDR_DECODING_FAILED, "XDR decoding failed"); - goto out; - } - - if (rsp.op_ret == -1) { - gf_msg (this->name, GF_LOG_ERROR, 0, PC_MSG_LOCK_REQ_FAIL, - "lock request failed"); - ret = -1; - goto out; - } - - fdctx = local->fdctx; - - gf_proto_flock_to_flock (&rsp.flock, &lock); - - gf_msg_debug (this->name, 0, "%s type lock reacquired on file " - "with gfid %s from %"PRIu64 " to %"PRIu64, - get_lk_type (lock.l_type), uuid_utoa (fdctx->gfid), - lock.l_start, lock.l_start + lock.l_len); - - if (!clnt_fd_lk_local_error_status (this, local) && - clnt_fd_lk_local_unref (this, local) == 0) { - pthread_spin_lock (&conf->fd_lock); - { - fdctx->lk_heal_state = GF_LK_HEAL_DONE; - } - pthread_spin_unlock (&conf->fd_lock); - - fdctx->reopen_done (fdctx, fdctx->remote_fd, this); - } - - ret = 0; -out: - if (ret < 0) { - clnt_fd_lk_local_mark_error (this, local); - - clnt_fd_lk_local_unref (this, local); - } - - frame->local = NULL; - STACK_DESTROY (frame->root); - - return ret; -} - -int -_client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx) -{ - int32_t ret = -1; - int32_t gf_cmd = 0; - int32_t gf_type = 0; - gfs3_lk_req req = {{0,},}; - struct gf_flock flock = {0,}; - fd_lk_ctx_t *lk_ctx = NULL; - clnt_fd_lk_local_t *local = NULL; - fd_lk_ctx_node_t *fd_lk = NULL; - call_frame_t *frame = NULL; - clnt_conf_t *conf = NULL; - - conf = (clnt_conf_t *) this->private; - lk_ctx = fdctx->lk_ctx; - - local = clnt_fd_lk_local_create (fdctx); - if (!local) { - gf_msg (this->name, GF_LOG_WARNING, 0, PC_MSG_LOCK_ERROR, - "clnt_fd_lk_local_create failed, aborting reacquring " - "of locks on %s.", uuid_utoa (fdctx->gfid)); - clnt_reacquire_lock_error (this, fdctx, conf); - goto out; - } - - list_for_each_entry (fd_lk, &lk_ctx->lk_list, next) { - memcpy (&flock, &fd_lk->user_flock, - sizeof (struct gf_flock)); - - /* Always send F_SETLK even if the cmd was F_SETLKW */ - /* to avoid frame being blocked if lock cannot be granted. */ - ret = client_cmd_to_gf_cmd (F_SETLK, &gf_cmd); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, 0, - PC_MSG_LOCK_ERROR, "client_cmd_to_gf_cmd " - "failed, aborting reacquiring of locks"); - break; - } - - gf_type = client_type_to_gf_type (flock.l_type); - req.fd = fdctx->remote_fd; - req.cmd = gf_cmd; - req.type = gf_type; - (void) gf_proto_flock_from_flock (&req.flock, - &flock); - - memcpy (req.gfid, fdctx->gfid, 16); - - frame = create_frame (this, this->ctx->pool); - if (!frame) { - ret = -1; - break; - } - - frame->local = clnt_fd_lk_local_ref (this, local); - frame->root->lk_owner = fd_lk->user_flock.l_owner; - - ret = client_submit_request (this, &req, frame, - conf->fops, GFS3_OP_LK, - client_reacquire_lock_cbk, - NULL, NULL, 0, NULL, 0, NULL, - (xdrproc_t)xdr_gfs3_lk_req); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, 0, - PC_MSG_LOCK_REACQUIRE, "reacquiring locks " - "failed on file with gfid %s", - uuid_utoa (fdctx->gfid)); - break; - } - - ret = 0; - frame = NULL; - } - - if (local) - (void) clnt_fd_lk_local_unref (this, local); -out: - return ret; -} - -int -client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx) -{ - int32_t ret = -1; - fd_lk_ctx_t *lk_ctx = NULL; - - GF_VALIDATE_OR_GOTO ("client", this, out); - GF_VALIDATE_OR_GOTO (this->name, fdctx, out); - - if (client_fd_lk_list_empty (fdctx->lk_ctx, _gf_false)) { - gf_msg_debug (this->name, 0, - "fd lock list is empty"); - fdctx->reopen_done (fdctx, fdctx->remote_fd, this); - } else { - lk_ctx = fdctx->lk_ctx; - - LOCK (&lk_ctx->lock); - { - (void) _client_reacquire_lock (this, fdctx); - } - UNLOCK (&lk_ctx->lock); - } - ret = 0; -out: - return ret; -} - void client_default_reopen_done (clnt_fd_ctx_t *fdctx, int64_t rfd, xlator_t *this) { @@ -670,7 +400,6 @@ client_child_up_reopen_done (clnt_fd_ctx_t *fdctx, int64_t rfd, xlator_t *this) if (fd_count == 0) { gf_msg (this->name, GF_LOG_INFO, 0, PC_MSG_CHILD_UP_NOTIFY, "last fd open'd/lock-self-heal'd - notifying CHILD-UP"); - client_set_lk_version (this); client_notify_parents_child_up (this); } } @@ -681,16 +410,13 @@ client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count, { int32_t ret = -1; gfs3_open_rsp rsp = {0,}; - gf_boolean_t attempt_lock_recovery = _gf_false; clnt_local_t *local = NULL; - clnt_conf_t *conf = NULL; clnt_fd_ctx_t *fdctx = NULL; call_frame_t *frame = NULL; xlator_t *this = NULL; frame = myframe; this = frame->this; - conf = this->private; local = frame->local; fdctx = local->fdctx; @@ -727,38 +453,10 @@ client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count, goto out; } - pthread_spin_lock (&conf->fd_lock); - { - if (!fdctx->released) { - if (conf->lk_heal && - !client_fd_lk_list_empty (fdctx->lk_ctx, - _gf_false)) { - attempt_lock_recovery = _gf_true; - fdctx->lk_heal_state = GF_LK_HEAL_IN_PROGRESS; - } - } - } - pthread_spin_unlock (&conf->fd_lock); - ret = 0; - if (attempt_lock_recovery) { - /* Delay decrementing the reopen fd count until all the - locks corresponding to this fd are acquired.*/ - gf_msg_debug (this->name, 0, "acquiring locks " - "on %s", local->loc.path); - ret = client_reacquire_lock (frame->this, local->fdctx); - if (ret) { - clnt_reacquire_lock_error (this, local->fdctx, conf); - gf_msg (this->name, GF_LOG_WARNING, 0, - PC_MSG_LOCK_ERROR, "acquiring locks failed " - "on %s", local->loc.path); - } - } - out: - if (!attempt_lock_recovery) - fdctx->reopen_done (fdctx, (rsp.op_ret) ? -1 : rsp.fd, this); + fdctx->reopen_done (fdctx, (rsp.op_ret) ? -1 : rsp.fd, this); frame->local = NULL; STACK_DESTROY (frame->root); @@ -1051,7 +749,6 @@ client_post_handshake (call_frame_t *frame, xlator_t *this) gf_msg_debug (this->name, 0, "No fds to open - notifying all parents child " "up"); - client_set_lk_version (this); client_notify_parents_child_up (this); } out: @@ -1073,7 +770,6 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m int32_t op_ret = 0; int32_t op_errno = 0; gf_boolean_t auth_fail = _gf_false; - uint32_t lk_ver = 0; glusterfs_ctx_t *ctx = NULL; frame = myframe; @@ -1189,16 +885,6 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m conf->child_up = (child_up_int != 0); } - ret = dict_get_uint32 (reply, "clnt-lk-version", &lk_ver); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, 0, PC_MSG_DICT_GET_FAILED, - "failed to find key 'clnt-lk-version' in the options"); - goto out; - } - - gf_msg_debug (this->name, 0, "clnt-lk-version = %d, " - "server-lk-version = %d", client_get_lk_ver (conf), - lk_ver); /* TODO: currently setpeer path is broken */ /* if (process_uuid && req->conn && @@ -1234,22 +920,7 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m conf->connecting = 0; conf->connected = 1; - if (lk_ver != client_get_lk_ver (conf)) { - gf_msg (this->name, GF_LOG_INFO, 0, PC_MSG_LOCK_MISMATCH, - "Server and Client lk-version numbers are not same, " - "reopening the fds"); - client_mark_fd_bad (this); - client_post_handshake (frame, frame->this); - } else { - /*TODO: Traverse the saved fd list, and send - release to the server on fd's that were closed - during grace period */ - gf_msg (this->name, GF_LOG_INFO, 0, PC_MSG_LOCK_MATCH, - "Server and Client lk-version numbers are same, no " - "need to reopen the fds"); - client_notify_parents_child_up (frame->this); - } - + client_post_handshake (frame, frame->this); out: if (auth_fail) { gf_msg (this->name, GF_LOG_INFO, 0, PC_MSG_AUTH_FAILED, @@ -1334,21 +1005,16 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc) } } - /* When lock-heal is enabled: - * With multiple graphs possible in the same process, we need a - field to bring the uniqueness. Graph-ID should be enough to get the - job done. - * When lock-heal is disabled, connection-id should always be unique so - * that server never gets to reuse the previous connection resources - * so it cleans up the resources on every disconnect. Otherwise - * it may lead to stale resources, i.e. leaked file desciptors, - * inode/entry locks - */ - if (!conf->lk_heal) { - snprintf (counter_str, sizeof (counter_str), + /* + * Connection-id should always be unique so that server never gets to + * reuse the previous connection resources so it cleans up the resources + * on every disconnect. Otherwise it may lead to stale resources, i.e. + * leaked file desciptors, inode/entry locks + */ + + snprintf (counter_str, sizeof (counter_str), "-%"PRIu64, conf->setvol_count); - conf->setvol_count++; - } + conf->setvol_count++; if (gethostname (hostname, 256) == -1) { gf_msg (this->name, GF_LOG_ERROR, errno, @@ -1417,14 +1083,6 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc) } } - ret = dict_set_int16 (options, "clnt-lk-version", - client_get_lk_ver (conf)); - if (ret < 0) { - gf_msg (this->name, GF_LOG_WARNING, 0, PC_MSG_DICT_SET_FAILED, - "failed to set clnt-lk-version(%"PRIu32") in handshake " - "msg", client_get_lk_ver (conf)); - } - ret = dict_set_int32 (options, "opversion", GD_OP_VERSION_MAX); if (ret < 0) { gf_msg (this->name, GF_LOG_ERROR, 0, PC_MSG_DICT_SET_FAILED, @@ -1777,7 +1435,6 @@ char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = { [GF_HNDSK_SETVOLUME] = "SETVOLUME", [GF_HNDSK_GETSPEC] = "GETSPEC", [GF_HNDSK_PING] = "PING", - [GF_HNDSK_SET_LK_VER] = "SET_LK_VER" }; rpc_clnt_prog_t clnt_handshake_prog = { diff --git a/xlators/protocol/client/src/client-rpc-fops.c b/xlators/protocol/client/src/client-rpc-fops.c index ead439eab7e..4279a32a054 100644 --- a/xlators/protocol/client/src/client-rpc-fops.c +++ b/xlators/protocol/client/src/client-rpc-fops.c @@ -363,7 +363,6 @@ client_add_fd_to_saved_fds (xlator_t *this, fd_t *fd, loc_t *loc, int32_t flags, fdctx->remote_fd = remote_fd; fdctx->flags = flags; fdctx->lk_ctx = fd_lk_ctx_ref (fd->lk_ctx); - fdctx->lk_heal_state = GF_LK_HEAL_DONE; fdctx->reopen_done = client_default_reopen_done; INIT_LIST_HEAD (&fdctx->sfd_pos); @@ -2437,20 +2436,6 @@ client3_3_lk_cbk (struct rpc_req *req, struct iovec *iov, int count, goto out; } - /* Save the lock to the client lock cache to be able - to recover in the case of server reboot.*/ - /* - temporarily - if (local->cmd == F_SETLK || local->cmd == F_SETLKW) { - ret = client_add_lock_for_recovery (local->fd, &lock, - local->owner, local->cmd); - if (ret < 0) { - rsp.op_ret = -1; - rsp.op_errno = -ret; - } - } - */ - out: if ((rsp.op_ret == -1) && (EAGAIN != gf_error_to_errno (rsp.op_errno))) { @@ -3277,7 +3262,6 @@ client3_3_release (call_frame_t *frame, xlator_t *this, clnt_conf_t *conf = NULL; clnt_fd_ctx_t *fdctx = NULL; clnt_args_t *args = NULL; - lk_heal_state_t lk_heal_state = GF_LK_HEAL_DONE; gf_boolean_t destroy = _gf_false; if (!this || !data) @@ -3291,7 +3275,6 @@ client3_3_release (call_frame_t *frame, xlator_t *this, pthread_spin_lock (&conf->fd_lock); { remote_fd = fdctx->remote_fd; - lk_heal_state = fdctx->lk_heal_state; /* fdctx->remote_fd == -1 indicates a reopen attempt in progress. Just mark ->released = 1 and let @@ -3299,7 +3282,7 @@ client3_3_release (call_frame_t *frame, xlator_t *this, */ if (remote_fd == -1) { fdctx->released = 1; - } else if (lk_heal_state == GF_LK_HEAL_DONE) { + } else { list_del_init (&fdctx->sfd_pos); destroy = _gf_true; } diff --git a/xlators/protocol/client/src/client-rpc-fops_v2.c b/xlators/protocol/client/src/client-rpc-fops_v2.c index 4a4df478a67..d30d56e9929 100644 --- a/xlators/protocol/client/src/client-rpc-fops_v2.c +++ b/xlators/protocol/client/src/client-rpc-fops_v2.c @@ -2115,20 +2115,6 @@ client4_0_lk_cbk (struct rpc_req *req, struct iovec *iov, int count, goto out; } - /* Save the lock to the client lock cache to be able - to recover in the case of server reboot.*/ - /* - temporarily - if (local->cmd == F_SETLK || local->cmd == F_SETLKW) { - ret = client_add_lock_for_recovery (local->fd, &lock, - local->owner, local->cmd); - if (ret < 0) { - rsp.op_ret = -1; - rsp.op_errno = -ret; - } - } - */ - out: if ((rsp.op_ret == -1) && (EAGAIN != gf_error_to_errno (rsp.op_errno))) { @@ -2782,7 +2768,6 @@ client4_0_release (call_frame_t *frame, xlator_t *this, clnt_conf_t *conf = NULL; clnt_fd_ctx_t *fdctx = NULL; clnt_args_t *args = NULL; - lk_heal_state_t lk_heal_state = GF_LK_HEAL_DONE; gf_boolean_t destroy = _gf_false; if (!this || !data) @@ -2796,7 +2781,6 @@ client4_0_release (call_frame_t *frame, xlator_t *this, fdctx = this_fd_del_ctx (args->fd, this); if (fdctx != NULL) { remote_fd = fdctx->remote_fd; - lk_heal_state = fdctx->lk_heal_state; /* fdctx->remote_fd == -1 indicates a reopen attempt in progress. Just mark ->released = 1 and let @@ -2804,7 +2788,7 @@ client4_0_release (call_frame_t *frame, xlator_t *this, */ if (remote_fd == -1) { fdctx->released = 1; - } else if (lk_heal_state == GF_LK_HEAL_DONE) { + } else { list_del_init (&fdctx->sfd_pos); destroy = _gf_true; } diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c index 955db86e1c2..74bf2104edc 100644 --- a/xlators/protocol/client/src/client.c +++ b/xlators/protocol/client/src/client.c @@ -126,96 +126,6 @@ client_type_to_gf_type (short l_type) return gf_type; } -uint32_t -client_get_lk_ver (clnt_conf_t *conf) -{ - uint32_t lk_ver = 0; - - GF_VALIDATE_OR_GOTO ("client", conf, out); - - pthread_mutex_lock (&conf->lock); - { - lk_ver = conf->lk_version; - } - pthread_mutex_unlock (&conf->lock); -out: - return lk_ver; -} - -void -client_grace_timeout (void *data) -{ - int ver = 0; - xlator_t *this = NULL; - struct clnt_conf *conf = NULL; - - GF_VALIDATE_OR_GOTO ("client", data, out); - - this = THIS; - - conf = (struct clnt_conf *) this->private; - - pthread_mutex_lock (&conf->lock); - { - ver = ++conf->lk_version; - /* ver == 0 is a special value used by server - to notify client that this is a fresh connect.*/ - if (ver == 0) - ver = ++conf->lk_version; - - gf_timer_call_cancel (this->ctx, conf->grace_timer); - conf->grace_timer = NULL; - } - pthread_mutex_unlock (&conf->lock); - - gf_msg (this->name, GF_LOG_WARNING, 0, PC_MSG_TIMER_EXPIRED, - "client grace timer expired, updating " - "the lk-version to %d", ver); - - client_mark_fd_bad (this); -out: - return; -} - -int32_t -client_register_grace_timer (xlator_t *this, clnt_conf_t *conf) -{ - int32_t ret = -1; - struct timespec grace_ts = {0, }; - - GF_VALIDATE_OR_GOTO ("client", this, out); - GF_VALIDATE_OR_GOTO (this->name, conf, out); - - grace_ts.tv_sec = conf->grace_timeout; - grace_ts.tv_nsec = 0; - - pthread_mutex_lock (&conf->lock); - { - if (conf->grace_timer || !conf->grace_timer_needed) { - gf_msg_trace (this->name, 0, - "Client grace timer is already set " - "or a grace-timer has already time " - "out, not registering a new timer"); - } else { - gf_msg (this->name, GF_LOG_INFO, 0, PC_MSG_TIMER_REG, - "Registering a grace timer"); - - conf->grace_timer_needed = _gf_false; - - conf->grace_timer = - gf_timer_call_after (this->ctx, - grace_ts, - client_grace_timeout, - conf->rpc); - } - } - pthread_mutex_unlock (&conf->lock); - - ret = 0; -out: - return ret; -} - int client_submit_request (xlator_t *this, void *req, call_frame_t *frame, rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn, @@ -2298,34 +2208,12 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event, gf_msg (this->name, GF_LOG_WARNING, 0, PC_MSG_HANDSHAKE_RETURN, "handshake " "msg returned %d", ret); - - /* Cancel grace timer if set */ - pthread_mutex_lock (&conf->lock); - { - conf->grace_timer_needed = _gf_true; - - if (conf->grace_timer) { - gf_msg (this->name, GF_LOG_WARNING, 0, - PC_MSG_GRACE_TIMER_CANCELLED, - "Cancelling the grace timer"); - - gf_timer_call_cancel (this->ctx, - conf->grace_timer); - - conf->grace_timer = NULL; - } - } - pthread_mutex_unlock (&conf->lock); - break; } case RPC_CLNT_DISCONNECT: gf_msg_debug (this->name, 0, "got RPC_CLNT_DISCONNECT"); - if (!conf->lk_heal) - client_mark_fd_bad (this); - else - client_register_grace_timer (this, conf); + client_mark_fd_bad (this); if (!conf->skip_notify) { if (conf->connected) { @@ -2378,9 +2266,7 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event, } else { rpc->conn.config.remote_port = 0; } - break; - case RPC_CLNT_DESTROY: ret = client_fini_complete (this); break; @@ -2614,33 +2500,6 @@ out: return ret; } - -int -client_init_grace_timer (xlator_t *this, dict_t *options, - clnt_conf_t *conf) -{ - int32_t ret = -1; - - GF_VALIDATE_OR_GOTO ("client", this, out); - GF_VALIDATE_OR_GOTO (this->name, options, out); - GF_VALIDATE_OR_GOTO (this->name, conf, out); - - GF_OPTION_RECONF ("lk-heal", conf->lk_heal, options, bool, out); - - gf_msg_debug (this->name, 0, "lk-heal = %s", - (conf->lk_heal) ? "on" : "off"); - - GF_OPTION_RECONF ("grace-timeout", conf->grace_timeout, - options, uint32, out); - - gf_msg_debug (this->name, 0, "Client grace timeout value = %d", - conf->grace_timeout); - - ret = 0; -out: - return ret; -} - int client_check_event_threads (xlator_t *this, clnt_conf_t *conf, int32_t old, int32_t new) @@ -2722,10 +2581,6 @@ reconfigure (xlator_t *this, dict_t *options) GF_OPTION_RECONF ("send-gids", conf->send_gids, options, bool, out); - ret = client_init_grace_timer (this, options, conf); - if (ret) - goto out; - ret = 0; out: return ret; @@ -2761,11 +2616,6 @@ init (xlator_t *this) conf->child_up = _gf_false; - /* Initialize parameters for lock self healing*/ - conf->lk_version = 1; - conf->grace_timer = NULL; - conf->grace_timer_needed = _gf_true; - /* Set event threads to the configured default */ GF_OPTION_INIT("event-threads", conf->event_threads, int32, out); ret = client_check_event_threads (this, conf, STARTING_EVENT_THREADS, @@ -2773,10 +2623,6 @@ init (xlator_t *this) if (ret) goto out; - ret = client_init_grace_timer (this, this->options, conf); - if (ret) - goto out; - LOCK_INIT (&conf->rec_lock); conf->last_sent_event = -1; /* To start with we don't have any events */ @@ -3078,27 +2924,6 @@ struct volume_options options[] = { { .key = {"client-bind-insecure"}, .type = GF_OPTION_TYPE_BOOL }, - { .key = {"lk-heal"}, - .type = GF_OPTION_TYPE_BOOL, - .default_value = "off", - .description = "When the connection to client is lost, server " - "cleans up all the locks held by the client. After " - "the connection is restored, the client reacquires " - "(heals) the fcntl locks released by the server.", - .op_version = {1}, - .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC - }, - { .key = {"grace-timeout"}, - .type = GF_OPTION_TYPE_INT, - .min = 10, - .max = 1800, - .default_value = "10", - .description = "Specifies the duration for the lock state to be " - "maintained on the client after a network " - "disconnection. Range 10-1800 seconds.", - .op_version = {1}, - .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC - }, { .key = {"tcp-window-size"}, .type = GF_OPTION_TYPE_SIZET, .min = GF_MIN_SOCKET_WINDOW_SIZE, diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h index 003a2bef397..207c778c20c 100644 --- a/xlators/protocol/client/src/client.h +++ b/xlators/protocol/client/src/client.h @@ -33,11 +33,6 @@ #define GF_MAX_SOCKET_WINDOW_SIZE (1 * GF_UNIT_MB) #define GF_MIN_SOCKET_WINDOW_SIZE (0) -typedef enum { - GF_LK_HEAL_IN_PROGRESS, - GF_LK_HEAL_DONE, -} lk_heal_state_t; - typedef enum { DEFAULT_REMOTE_FD = 0, FALLBACK_TO_ANON_FD = 1 @@ -218,18 +213,6 @@ typedef struct clnt_conf { char disconnect_err_logged; /* flag used to prevent excessive disconnect logging */ - gf_boolean_t lk_heal; - uint16_t lk_version; /* this variable is used to distinguish - client-server transaction while - performing lock healing */ - uint32_t grace_timeout; - gf_timer_t *grace_timer; - gf_boolean_t grace_timer_needed; /* The state of this flag will - be used to decide whether - a new grace-timer must be - registered or not. False - means dont register, true - means register */ char parent_down; gf_boolean_t quick_reconnect; /* When reconnecting after portmap query, do not let @@ -266,7 +249,6 @@ typedef struct _client_fd_ctx { int32_t flags; fd_lk_ctx_t *lk_ctx; pthread_mutex_t mutex; - lk_heal_state_t lk_heal_state; uuid_t gfid; void (*reopen_done)(struct _client_fd_ctx*, int64_t rfd, xlator_t *); struct list_head lock_list; /* List of all granted locks on this fd */ @@ -376,8 +358,6 @@ int clnt_readdir_rsp_cleanup (gfs3_readdir_rsp *rsp); int clnt_readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp); int client_attempt_lock_recovery (xlator_t *this, clnt_fd_ctx_t *fdctx); int32_t delete_granted_locks_owner (fd_t *fd, gf_lkowner_t *owner); -int client_add_lock_for_recovery (fd_t *fd, struct gf_flock *flock, - gf_lkowner_t *owner, int32_t cmd); int32_t delete_granted_locks_fd (clnt_fd_ctx_t *fdctx); int32_t client_cmd_to_gf_cmd (int32_t cmd, int32_t *gf_cmd); void client_save_number_fds (clnt_conf_t *conf, int count); @@ -388,14 +368,10 @@ int32_t client_dump_locks (char *name, inode_t *inode, dict_t *dict); int client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx); -uint32_t client_get_lk_ver (clnt_conf_t *conf); - int32_t client_type_to_gf_type (short l_type); int client_mark_fd_bad (xlator_t *this); -int client_set_lk_version (xlator_t *this); - int client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx, gf_boolean_t use_try_lock); void client_default_reopen_done (clnt_fd_ctx_t *fdctx, int64_t rfd, xlator_t *this); diff --git a/xlators/protocol/server/src/server-handshake.c b/xlators/protocol/server/src/server-handshake.c index acc1cb6ca75..f6057da3b68 100644 --- a/xlators/protocol/server/src/server-handshake.c +++ b/xlators/protocol/server/src/server-handshake.c @@ -465,9 +465,7 @@ server_setvolume (rpcsvc_request_t *req) int32_t ret = -1; int32_t op_ret = -1; int32_t op_errno = EINVAL; - uint32_t lk_version = 0; char *buf = NULL; - gf_boolean_t cancelled = _gf_false; uint32_t opversion = 0; rpc_transport_t *xprt = NULL; int32_t fop_version = 0; @@ -617,20 +615,6 @@ server_setvolume (rpcsvc_request_t *req) client_name = "unknown"; } - /*lk_verion :: [1..2^31-1]*/ - ret = dict_get_uint32 (params, "clnt-lk-version", &lk_version); - if (ret < 0) { - ret = dict_set_str (reply, "ERROR", - "lock state version not supplied"); - if (ret < 0) - gf_msg_debug (this->name, 0, "failed to set error " - "msg"); - - op_ret = -1; - op_errno = EINVAL; - goto fail; - } - client = gf_client_get (this, &req->cred, client_uid, subdir_mount); if (client == NULL) { op_ret = -1; @@ -641,24 +625,6 @@ server_setvolume (rpcsvc_request_t *req) client->client_name = gf_strdup(client_name); gf_msg_debug (this->name, 0, "Connected to %s", client->client_uid); - cancelled = server_cancel_grace_timer (this, client); - if (cancelled) { - /* If timer has been successfully cancelled then it means - * that the client has reconnected within grace period. - * Since we've bumped up the bind count with a gf_client_get() - * for this connect attempt, we need to drop the bind count - * for earlier connect, since grace timer handler couldn't - * drop it since the timer was cancelled. - */ - gf_client_put (client, NULL); - - /* We need to drop the ref count for this reconnected client - * since one ref was taken before delegating to the grace - * timer handler. Since grace timer handler was cancelled, - * it couldn't run and drop the ref either. - */ - gf_client_unref (client); - } serv_ctx = server_ctx_get (client, client->this); if (serv_ctx == NULL) { @@ -668,12 +634,6 @@ server_setvolume (rpcsvc_request_t *req) goto fail; } - if (serv_ctx->lk_version != 0 && - serv_ctx->lk_version != lk_version) { - (void) server_connection_cleanup (this, client, - INTERNAL_LOCKS | POSIX_LOCKS); - } - if (req->trans->xl_private != client) req->trans->xl_private = client; @@ -878,12 +838,6 @@ server_setvolume (rpcsvc_request_t *req) if (ret) gf_msg_debug (this->name, 0, "failed to set 'process-uuid'"); - ret = dict_set_uint32 (reply, "clnt-lk-version", serv_ctx->lk_version); - if (ret) - gf_msg (this->name, GF_LOG_WARNING, 0, - PS_MSG_CLIENT_LK_VERSION_ERROR, "failed to set " - "'clnt-lk-version'"); - ret = dict_set_uint64 (reply, "transport-ptr", ((uint64_t) (long) req->trans)); if (ret) @@ -976,47 +930,20 @@ server_ping (rpcsvc_request_t *req) int server_set_lk_version (rpcsvc_request_t *req) { - int op_ret = -1; - int op_errno = EINVAL; + int ret = -1; gf_set_lk_ver_req args = {0,}; gf_set_lk_ver_rsp rsp = {0,}; - client_t *client = NULL; - server_ctx_t *serv_ctx = NULL; - xlator_t *this = NULL; - this = req->svc->xl; - //TODO: Decide on an appropriate errno for the error-path - //below - if (!this) - goto fail; - - op_ret = xdr_to_generic (req->msg[0], &args, + ret = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gf_set_lk_ver_req); - if (op_ret < 0) { - //failed to decode msg; + if (ret < 0) { + /* failed to decode msg */ req->rpc_err = GARBAGE_ARGS; goto fail; } - client = gf_client_get (this, &req->cred, args.uid, NULL); - serv_ctx = server_ctx_get (client, client->this); - if (serv_ctx == NULL) { - gf_msg (this->name, GF_LOG_INFO, 0, - PS_MSG_SERVER_CTX_GET_FAILED, "server_ctx_get() " - "failed"); - goto fail; - } - - serv_ctx->lk_version = args.lk_ver; rsp.lk_ver = args.lk_ver; - - op_ret = 0; fail: - if (client) - gf_client_put (client, NULL); - - rsp.op_ret = op_ret; - rsp.op_errno = op_errno; server_submit_reply (NULL, req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_set_lk_ver_rsp); diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c index 7a58d2592ba..31eb2510cf8 100644 --- a/xlators/protocol/server/src/server-helpers.c +++ b/xlators/protocol/server/src/server-helpers.c @@ -1277,49 +1277,6 @@ gf_server_check_setxattr_cmd (call_frame_t *frame, dict_t *dict) return 0; } - -gf_boolean_t -server_cancel_grace_timer (xlator_t *this, client_t *client) -{ - server_ctx_t *serv_ctx = NULL; - gf_timer_t *timer = NULL; - gf_boolean_t cancelled = _gf_false; - - if (!this || !client) { - gf_msg (THIS->name, GF_LOG_ERROR, EINVAL, PS_MSG_INVALID_ENTRY, - "Invalid arguments to cancel connection timer"); - return cancelled; - } - - serv_ctx = server_ctx_get (client, client->this); - - if (serv_ctx == NULL) { - gf_msg (this->name, GF_LOG_INFO, 0, - PS_MSG_SERVER_CTX_GET_FAILED, - "server_ctx_get() failed"); - goto out; - } - - LOCK (&serv_ctx->fdtable_lock); - { - if (serv_ctx->grace_timer) { - gf_msg (this->name, GF_LOG_INFO, 0, - PS_MSG_GRACE_TIMER_CANCELLED, - "Cancelling the grace timer"); - timer = serv_ctx->grace_timer; - serv_ctx->grace_timer = NULL; - } - } - UNLOCK (&serv_ctx->fdtable_lock); - - if (timer) { - gf_timer_call_cancel (this->ctx, timer); - cancelled = _gf_true; - } -out: - return cancelled; -} - server_ctx_t* server_ctx_get (client_t *client, xlator_t *xlator) { @@ -1339,7 +1296,6 @@ server_ctx_get (client_t *client, xlator_t *xlator) if (ctx == NULL) goto out; - /* ctx->lk_version = 0; redundant */ ctx->fdtable = gf_fd_fdtable_alloc (); if (ctx->fdtable == NULL) { diff --git a/xlators/protocol/server/src/server-helpers.h b/xlators/protocol/server/src/server-helpers.h index 9fe60fd231b..b89105a355f 100644 --- a/xlators/protocol/server/src/server-helpers.h +++ b/xlators/protocol/server/src/server-helpers.h @@ -44,9 +44,6 @@ int server_connection_cleanup (xlator_t *this, struct _client *client, int32_t flags); -gf_boolean_t -server_cancel_grace_timer (xlator_t *this, struct _client *client); - int server_build_config (xlator_t *this, server_conf_t *conf); diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c index bae4a119b1f..7bd276cdece 100644 --- a/xlators/protocol/server/src/server.c +++ b/xlators/protocol/server/src/server.c @@ -31,61 +31,6 @@ rpcsvc_cbk_program_t server_cbk_prog = { .progver = GLUSTER_CBK_VERSION, }; -void -grace_time_handler (void *data) -{ - client_t *client = NULL; - xlator_t *this = NULL; - gf_timer_t *timer = NULL; - server_ctx_t *serv_ctx = NULL; - gf_boolean_t cancelled = _gf_false; - gf_boolean_t detached = _gf_false; - - client = data; - this = client->this; - - GF_VALIDATE_OR_GOTO (THIS->name, this, out); - - gf_msg (this->name, GF_LOG_INFO, 0, PS_MSG_GRACE_TIMER_EXPD, "grace " - "timer expired for %s", client->client_uid); - - serv_ctx = server_ctx_get (client, this); - - if (serv_ctx == NULL) { - gf_msg (this->name, GF_LOG_INFO, 0, - PS_MSG_SERVER_CTX_GET_FAILED, "server_ctx_get() " - "failed"); - goto out; - } - - LOCK (&serv_ctx->fdtable_lock); - { - if (serv_ctx->grace_timer) { - timer = serv_ctx->grace_timer; - serv_ctx->grace_timer = NULL; - } - } - UNLOCK (&serv_ctx->fdtable_lock); - if (timer) { - gf_timer_call_cancel (this->ctx, timer); - cancelled = _gf_true; - } - if (cancelled) { - - /* - * ref has already been taken in server_rpc_notify() - */ - gf_client_put (client, &detached); - - if (client && detached) /* reconnection did not happen :-( */ - server_connection_cleanup (this, client, - INTERNAL_LOCKS | POSIX_LOCKS); - gf_client_unref (client); - } -out: - return; -} - struct iobuf * gfs_serialize_reply (rpcsvc_request_t *req, void *arg, struct iovec *outmsg, xdrproc_t xdrproc) @@ -145,7 +90,6 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg, server_state_t *state = NULL; char new_iobref = 0; client_t *client = NULL; - gf_boolean_t lk_heal = _gf_false; GF_VALIDATE_OR_GOTO ("server", req, ret); @@ -155,9 +99,6 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg, client = frame->root->client; } - if (client) - lk_heal = ((server_conf_t *) client->this->private)->lk_heal; - if (!iobref) { iobref = iobref_new (); if (!iobref) { @@ -193,7 +134,7 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg, gf_msg_callingfn ("", GF_LOG_ERROR, 0, PS_MSG_REPLY_SUBMIT_FAILED, "Reply submission failed"); - if (frame && client && !lk_heal) { + if (frame && client) { server_connection_cleanup (frame->this, client, INTERNAL_LOCKS | POSIX_LOCKS); } else { @@ -472,8 +413,6 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event, rpc_transport_t *trans = NULL; server_conf_t *conf = NULL; client_t *client = NULL; - server_ctx_t *serv_ctx = NULL; - struct timespec grace_ts = {0, }; char *auth_path = NULL; int ret = -1; @@ -542,76 +481,24 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event, auth_path = NULL; } - /* If lock self heal is off, then destroy the - conn object, else register a grace timer event */ - if (!conf->lk_heal) { - gf_client_ref (client); - gf_client_put (client, &detached); - if (client && detached) { - server_connection_cleanup (this, client, - INTERNAL_LOCKS | POSIX_LOCKS); - - gf_event (EVENT_CLIENT_DISCONNECT, - "client_uid=%s;" - "client_identifier=%s;" - "server_identifier=%s;" - "brick_path=%s", - client->client_uid, - trans->peerinfo.identifier, - trans->myinfo.identifier, - auth_path); - } - - /* - * gf_client_unref will be done while handling - * RPC_EVENT_TRANSPORT_DESTROY - */ - goto unref_transport; - } - - serv_ctx = server_ctx_get (client, this); - - if (serv_ctx == NULL) { - gf_msg (this->name, GF_LOG_INFO, 0, - PS_MSG_SERVER_CTX_GET_FAILED, - "server_ctx_get() failed"); - goto unref_transport; - } - - grace_ts.tv_sec = conf->grace_timeout; - grace_ts.tv_nsec = 0; - - LOCK (&serv_ctx->fdtable_lock); - { - if (!serv_ctx->grace_timer) { - - gf_msg (this->name, GF_LOG_INFO, 0, - PS_MSG_GRACE_TIMER_START, - "starting a grace timer for %s", - client->client_uid); - - /* ref to protect against client destruction - * in RPCSVC_EVENT_TRANSPORT_DESTROY while - * we are starting a grace timer - */ - gf_client_ref (client); - - serv_ctx->grace_timer = - gf_timer_call_after (this->ctx, - grace_ts, - grace_time_handler, - client); - } + gf_client_ref (client); + gf_client_put (client, &detached); + if (detached) { + server_connection_cleanup (this, client, + INTERNAL_LOCKS | POSIX_LOCKS); + gf_event (EVENT_CLIENT_DISCONNECT, "client_uid=%s;" + "client_identifier=%s;server_identifier=%s;" + "brick_path=%s", + client->client_uid, + trans->peerinfo.identifier, + trans->myinfo.identifier, + auth_path); } - UNLOCK (&serv_ctx->fdtable_lock); - gf_event (EVENT_CLIENT_DISCONNECT, "client_uid=%s;" - "client_identifier=%s;server_identifier=%s;" - "brick_path=%s", - client->client_uid, - trans->peerinfo.identifier, - trans->myinfo.identifier, - auth_path); + /* + * gf_client_unref will be done while handling + * RPC_EVENT_TRANSPORT_DESTROY + */ unref_transport: /* rpc_transport_unref() causes a RPCSVC_EVENT_TRANSPORT_DESTROY @@ -619,17 +506,13 @@ unref_transport: * So no code should ideally be after this unref */ rpc_transport_unref (trans); - break; - case RPCSVC_EVENT_TRANSPORT_DESTROY: client = trans->xl_private; if (!client) break; - /* unref only for if (!client->lk_heal) */ - if (!conf->lk_heal) - gf_client_unref (client); + gf_client_unref (client); trans->xl_private = NULL; break; @@ -703,33 +586,6 @@ _copy_auth_opt (dict_t *unused, char *key, data_t *value, void *xl_dict) return 0; } - -int -server_init_grace_timer (xlator_t *this, dict_t *options, - server_conf_t *conf) -{ - int32_t ret = -1; - - GF_VALIDATE_OR_GOTO ("server", this, out); - GF_VALIDATE_OR_GOTO (this->name, options, out); - GF_VALIDATE_OR_GOTO (this->name, conf, out); - - GF_OPTION_RECONF ("lk-heal", conf->lk_heal, options, bool, out); - - gf_msg_debug (this->name, 0, "lk-heal = %s", - (conf->lk_heal) ? "on" : "off"); - - GF_OPTION_RECONF ("grace-timeout", conf->grace_timeout, - options, uint32, out); - - gf_msg_debug (this->name, 0, "Server grace timeout value = %d", - conf->grace_timeout); - - ret = 0; -out: - return ret; -} - int server_check_event_threads (xlator_t *this, server_conf_t *conf, int32_t new) { @@ -987,8 +843,6 @@ do_rpc: if (ret) goto out; - ret = server_init_grace_timer (this, options, conf); - out: THIS = oldTHIS; gf_msg_debug ("", 0, "returning %d", ret); @@ -1085,10 +939,6 @@ server_init (xlator_t *this) if (ret) goto out; - ret = server_init_grace_timer (this, this->options, conf); - if (ret) - goto out; - ret = server_build_config (this, conf); if (ret) goto out; @@ -1772,20 +1622,6 @@ struct volume_options server_options[] = { .op_version = {1}, .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC }, - { .key = {"lk-heal"}, - .type = GF_OPTION_TYPE_BOOL, - .default_value = "off", - .op_version = {1}, - .flags = OPT_FLAG_SETTABLE - }, - {.key = {"grace-timeout"}, - .type = GF_OPTION_TYPE_INT, - .min = 10, - .max = 1800, - .default_value = "10", - .op_version = {1}, - .flags = OPT_FLAG_SETTABLE - }, {.key = {"tcp-window-size"}, .type = GF_OPTION_TYPE_SIZET, .min = GF_MIN_SOCKET_WINDOW_SIZE, diff --git a/xlators/protocol/server/src/server.h b/xlators/protocol/server/src/server.h index fe407689ceb..393219bf290 100644 --- a/xlators/protocol/server/src/server.h +++ b/xlators/protocol/server/src/server.h @@ -107,11 +107,8 @@ struct server_conf { int inode_lru_limit; gf_boolean_t verify_volfile; gf_boolean_t trace; - gf_boolean_t lk_heal; /* If true means lock self - heal is on else off. */ char *conf_dir; struct _volfile_ctx *volfile; - uint32_t grace_timeout; dict_t *auth_modules; pthread_mutex_t mutex; struct list_head xprt_list; @@ -244,8 +241,6 @@ extern struct rpcsvc_program glusterfs4_0_fop_prog; typedef struct _server_ctx { gf_lock_t fdtable_lock; fdtable_t *fdtable; - struct _gf_timer *grace_timer; - uint32_t lk_version; } server_ctx_t; -- cgit