diff options
Diffstat (limited to 'xlators/protocol/client')
| -rw-r--r-- | xlators/protocol/client/src/client-handshake.c | 163 | ||||
| -rw-r--r-- | xlators/protocol/client/src/client-lk.c | 360 | ||||
| -rw-r--r-- | xlators/protocol/client/src/client-rpc-fops.c | 1 | ||||
| -rw-r--r-- | xlators/protocol/client/src/client.h | 3 |
4 files changed, 76 insertions, 451 deletions
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c index 61708562a06..7862635fcc6 100644 --- a/xlators/protocol/client/src/client-handshake.c +++ b/xlators/protocol/client/src/client-handshake.c @@ -415,9 +415,6 @@ clnt_fd_lk_reacquire_failed (xlator_t *this, clnt_fd_ctx_t *fdctx, { fdctx->remote_fd = -1; fdctx->lk_heal_state = GF_LK_HEAL_DONE; - - list_add_tail (&fdctx->sfd_pos, - &conf->saved_fds); } pthread_mutex_unlock (&conf->lock); @@ -603,7 +600,7 @@ clnt_release_reopen_fd_cbk (struct rpc_req *req, struct iovec *iov, clnt_fd_lk_reacquire_failed (this, fdctx, conf); - decrement_reopen_fd_count (this, conf); + fdctx->reopen_done (fdctx, this); frame->local = NULL; STACK_DESTROY (frame->root); @@ -635,8 +632,8 @@ clnt_release_reopen_fd (xlator_t *this, clnt_fd_ctx_t *fdctx) (xdrproc_t)xdr_gfs3_releasedir_req); out: if (ret) { - decrement_reopen_fd_count (this, conf); clnt_fd_lk_reacquire_failed (this, fdctx, conf); + fdctx->reopen_done (fdctx, this); if (frame) { frame->local = NULL; STACK_DESTROY (frame->root); @@ -754,13 +751,10 @@ client_reacquire_lock_cbk (struct rpc_req *req, struct iovec *iov, pthread_mutex_lock (&conf->lock); { fdctx->lk_heal_state = GF_LK_HEAL_DONE; - - list_add_tail (&fdctx->sfd_pos, - &conf->saved_fds); } pthread_mutex_unlock (&conf->lock); - decrement_reopen_fd_count (this, conf); + fdctx->reopen_done (fdctx, this); } ret = 0; @@ -869,8 +863,7 @@ client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx) if (client_fd_lk_list_empty (fdctx->lk_ctx, _gf_false)) { gf_log (this->name, GF_LOG_DEBUG, "fd lock list is empty"); - decrement_reopen_fd_count (this, - (clnt_conf_t *)this->private); + fdctx->reopen_done (fdctx, this); } else { lk_ctx = fdctx->lk_ctx; @@ -885,14 +878,55 @@ out: return ret; } +void +client_default_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this) +{ + gf_log_callingfn (this->name, GF_LOG_WARNING, + "This function should never be called"); +} + +void +client_child_up_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this) +{ + clnt_conf_t *conf = NULL; + uint64_t fd_count = 0; + gf_boolean_t destroy = _gf_false; + + conf = this->private; + + LOCK (&conf->rec_lock); + { + fd_count = --(conf->reopen_fd_count); + } + UNLOCK (&conf->rec_lock); + + pthread_mutex_lock (&conf->lock); + { + if (!fdctx->released) + list_add_tail (&fdctx->sfd_pos, &conf->saved_fds); + else + destroy = _gf_true; + fdctx->reopen_done = client_default_reopen_done; + } + pthread_mutex_unlock (&conf->lock); + + if (fd_count == 0) { + gf_log (this->name, GF_LOG_INFO, + "last fd open'd/lock-self-heal'd - notifying CHILD-UP"); + client_set_lk_version (this); + client_notify_parents_child_up (this); + } + if (destroy) + client_fdctx_destroy (this, fdctx); +} + int client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count, void *myframe) { int32_t ret = -1; gfs3_open_rsp rsp = {0,}; - int attempt_lock_recovery = _gf_false; - uint64_t fd_count = 0; + gf_boolean_t attempt_lock_recovery = _gf_false; clnt_local_t *local = NULL; clnt_conf_t *conf = NULL; clnt_fd_ctx_t *fdctx = NULL; @@ -900,12 +934,10 @@ client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count, xlator_t *this = NULL; frame = myframe; - if (!frame || !frame->this) - goto out; - this = frame->this; + conf = this->private; local = frame->local; - conf = frame->this->private; + fdctx = local->fdctx; if (-1 == req->rpc_status) { gf_log (frame->this->name, GF_LOG_WARNING, @@ -938,32 +970,23 @@ client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count, goto out; } - fdctx = local->fdctx; - if (!fdctx) { - gf_log (frame->this->name, GF_LOG_WARNING, "fdctx not found"); - ret = -1; - goto out; - } - pthread_mutex_lock (&conf->lock); { fdctx->remote_fd = rsp.fd; if (!fdctx->released) { - if (!client_fd_lk_list_empty (fdctx->lk_ctx, _gf_false)) { + if (conf->lk_heal && + !client_fd_lk_list_empty (fdctx->lk_ctx, + _gf_false)) { attempt_lock_recovery = _gf_true; fdctx->lk_heal_state = GF_LK_HEAL_IN_PROGRESS; - } else { - list_add_tail (&fdctx->sfd_pos, - &conf->saved_fds); } - fdctx = NULL; } } pthread_mutex_unlock (&conf->lock); ret = 0; - if (conf->lk_heal && attempt_lock_recovery) { + if (attempt_lock_recovery) { /* Delay decrementing the reopen fd count untill all the locks corresponding to this fd are acquired.*/ gf_log (this->name, GF_LOG_DEBUG, "acquiring locks " @@ -973,23 +996,15 @@ client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count, clnt_reacquire_lock_error (this, local->fdctx, conf); gf_log (this->name, GF_LOG_WARNING, "acquiring locks " "failed on %s", local->loc.path); - ret = 0; } - } else { - fd_count = decrement_reopen_fd_count (frame->this, conf); } out: - if (fdctx) { - clnt_release_reopen_fd (this, fdctx); - } else if ((ret < 0) && frame && frame->this && conf) { - decrement_reopen_fd_count (frame->this, conf); - } + if (!attempt_lock_recovery) + fdctx->reopen_done (fdctx, this); - if (frame) { - frame->local = NULL; - STACK_DESTROY (frame->root); - } + frame->local = NULL; + STACK_DESTROY (frame->root); client_local_wipe (local); @@ -1008,11 +1023,10 @@ client3_3_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count, call_frame_t *frame = NULL; frame = myframe; - if (!frame || !frame->this) - goto out; + local = frame->local; + fdctx = local->fdctx; + conf = frame->this->private; - local = frame->local; - conf = frame->this->private; if (-1 == req->rpc_status) { gf_log (frame->this->name, GF_LOG_WARNING, @@ -1045,39 +1059,17 @@ client3_3_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count, goto out; } - fdctx = local->fdctx; - if (!fdctx) { - gf_log (frame->this->name, GF_LOG_WARNING, "fdctx not found"); - ret = -1; - goto out; - } - pthread_mutex_lock (&conf->lock); { fdctx->remote_fd = rsp.fd; - - if (!fdctx->released) { - list_add_tail (&fdctx->sfd_pos, &conf->saved_fds); - fdctx = NULL; - } } pthread_mutex_unlock (&conf->lock); - decrement_reopen_fd_count (frame->this, conf); - ret = 0; - out: - if (fdctx) - client_fdctx_destroy (frame->this, fdctx); - - if ((ret < 0) && frame && frame->this && conf) - decrement_reopen_fd_count (frame->this, conf); - - if (frame) { - frame->local = NULL; - STACK_DESTROY (frame->root); - } + fdctx->reopen_done (fdctx, frame->this); + frame->local = NULL; + STACK_DESTROY (frame->root); client_local_wipe (local); return 0; @@ -1092,9 +1084,6 @@ protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx) call_frame_t *frame = NULL; clnt_conf_t *conf = NULL; - if (!this || !fdctx) - goto out; - conf = this->private; local = mem_get0 (this->local_pool); @@ -1120,7 +1109,7 @@ protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx) gf_log (frame->this->name, GF_LOG_DEBUG, "attempting reopen on %s", local->loc.path); - frame->local = local; local = NULL; + frame->local = local; ret = client_submit_request (this, &req, frame, conf->fops, GFS3_OP_OPENDIR, @@ -1128,11 +1117,12 @@ protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx) NULL, 0, NULL, 0, NULL, (xdrproc_t)xdr_gfs3_opendir_req); if (ret) { - gf_log (THIS->name, GF_LOG_ERROR, + gf_log (this->name, GF_LOG_ERROR, "failed to send the re-opendir request"); + goto out; } - return ret; + return 0; out: if (frame) { @@ -1143,9 +1133,7 @@ out: if (local) client_local_wipe (local); - if ((ret < 0) && this && conf) { - decrement_reopen_fd_count (this, conf); - } + fdctx->reopen_done (fdctx, this); return 0; @@ -1160,9 +1148,6 @@ protocol_client_reopen (xlator_t *this, clnt_fd_ctx_t *fdctx) call_frame_t *frame = NULL; clnt_conf_t *conf = NULL; - if (!this || !fdctx) - goto out; - conf = this->private; frame = create_frame (this, this->ctx->pool); @@ -1192,17 +1177,17 @@ protocol_client_reopen (xlator_t *this, clnt_fd_ctx_t *fdctx) gf_log (frame->this->name, GF_LOG_DEBUG, "attempting reopen on %s", local->loc.path); - local = NULL; ret = client_submit_request (this, &req, frame, conf->fops, GFS3_OP_OPEN, client3_3_reopen_cbk, NULL, NULL, 0, NULL, 0, NULL, (xdrproc_t)xdr_gfs3_open_req); if (ret) { - gf_log (THIS->name, GF_LOG_ERROR, + gf_log (this->name, GF_LOG_ERROR, "failed to send the re-open request"); + goto out; } - return ret; + return 0; out: if (frame) { @@ -1213,15 +1198,12 @@ out: if (local) client_local_wipe (local); - if ((ret < 0) && this && conf) { - decrement_reopen_fd_count (this, conf); - } + fdctx->reopen_done (fdctx, this); return 0; } - int client_post_handshake (call_frame_t *frame, xlator_t *this) { @@ -1245,6 +1227,7 @@ client_post_handshake (call_frame_t *frame, xlator_t *this) if (fdctx->remote_fd != -1) continue; + fdctx->reopen_done = client_child_up_reopen_done; list_del_init (&fdctx->sfd_pos); list_add_tail (&fdctx->sfd_pos, &reopen_head); count++; diff --git a/xlators/protocol/client/src/client-lk.c b/xlators/protocol/client/src/client-lk.c index 5611c7b83e2..1fd8f0d5029 100644 --- a/xlators/protocol/client/src/client-lk.c +++ b/xlators/protocol/client/src/client-lk.c @@ -16,9 +16,6 @@ static void __insert_and_merge (clnt_fd_ctx_t *fdctx, client_posix_lock_t *lock); -static int -client_send_recovery_lock (call_frame_t *frame, xlator_t *this, - client_posix_lock_t *lock); static void __dump_client_lock (client_posix_lock_t *lock) { @@ -430,21 +427,6 @@ delete_granted_locks_fd (clnt_fd_ctx_t *fdctx) return ret; } -static void -client_mark_bad_fd (fd_t *fd, clnt_fd_ctx_t *fdctx) -{ - xlator_t *this = NULL; - - this = THIS; - if (fdctx) - fdctx->remote_fd = -1; - - gf_log (this->name, GF_LOG_WARNING, - "marking the file descriptor (%p) bad", fd); - - this_fd_set_ctx (fd, this, NULL, fdctx); -} - int32_t client_cmd_to_gf_cmd (int32_t cmd, int32_t *gf_cmd) { @@ -553,348 +535,6 @@ out: } -static int -construct_reserve_unlock (struct gf_flock *lock, call_frame_t *frame, - client_posix_lock_t *client_lock) -{ - GF_ASSERT (lock); - GF_ASSERT (frame); - - lock->l_type = F_UNLCK; - lock->l_start = 0; - lock->l_whence = SEEK_SET; - lock->l_len = 0; /* Whole file */ - lock->l_pid = (uint64_t)(unsigned long)frame->root; - lock->l_owner = client_lock->owner; - - frame->root->lk_owner = client_lock->owner; - - return 0; -} - -static int -construct_reserve_lock (client_posix_lock_t *client_lock, call_frame_t *frame, - struct gf_flock *lock) -{ - GF_ASSERT (client_lock); - - memcpy (lock, &(client_lock->user_flock), sizeof (struct gf_flock)); - - frame->root->lk_owner = client_lock->owner; - - return 0; -} - -uint64_t -decrement_reopen_fd_count (xlator_t *this, clnt_conf_t *conf) -{ - uint64_t fd_count = 0; - - LOCK (&conf->rec_lock); - { - fd_count = --(conf->reopen_fd_count); - } - UNLOCK (&conf->rec_lock); - - if (fd_count == 0) { - gf_log (this->name, GF_LOG_INFO, - "last fd open'd/lock-self-heal'd - notifying CHILD-UP"); - client_set_lk_version (this); - client_notify_parents_child_up (this); - } - - return fd_count; -} - -int32_t -client_remove_reserve_lock_cbk (call_frame_t *frame, - void *cookie, - xlator_t *this, - int32_t op_ret, - int32_t op_errno, - struct gf_flock *lock, dict_t *xdata) -{ - clnt_local_t *local = NULL; - clnt_conf_t *conf = NULL; - - uint64_t fd_count = 0; - - local = frame->local; - conf = this->private; - - if (op_ret < 0) { - gf_log (this->name, GF_LOG_WARNING, - "removing reserver lock on fd failed: %s", - strerror(op_errno)); - goto cleanup; - } - - gf_log (this->name, GF_LOG_DEBUG, - "Removing reserve lock was successful."); - -cleanup: - frame->local = NULL; - - client_mark_bad_fd (local->client_lock->fd, local->fdctx); - - destroy_client_lock (local->client_lock); - client_local_wipe (local); - STACK_DESTROY (frame->root); - - fd_count = decrement_reopen_fd_count (this, conf); - gf_log (this->name, GF_LOG_TRACE, - "Need to attempt lock recovery on %lld open fds", - (unsigned long long) fd_count); - return 0; -} - -static void -client_remove_reserve_lock (xlator_t *this, call_frame_t *frame, - client_posix_lock_t *lock) -{ - struct gf_flock unlock; - - construct_reserve_unlock (&unlock, frame, lock); - - STACK_WIND (frame, client_remove_reserve_lock_cbk, - this, this->fops->lk, - lock->fd, F_RESLK_UNLCK, &unlock, NULL); -} - -static client_posix_lock_t * -get_next_recovery_lock (xlator_t *this, clnt_local_t *local) -{ - client_posix_lock_t *lock = NULL; - - pthread_mutex_lock (&local->mutex); - { - if (list_empty (&local->lock_list)) { - gf_log (this->name, GF_LOG_DEBUG, - "lock-list empty"); - goto unlock; - } - - lock = list_entry ((local->lock_list).next, typeof (*lock), list); - list_del_init (&lock->list); - } -unlock: - pthread_mutex_unlock (&local->mutex); - - return lock; - -} - -int32_t -client_reserve_lock_cbk (call_frame_t *frame, - void *cookie, - xlator_t *this, - int32_t op_ret, - int32_t op_errno, - struct gf_flock *lock, dict_t *xdata) -{ - - clnt_local_t *local = NULL; - clnt_conf_t *conf = NULL; - - uint64_t fd_count = 0; - - local = frame->local; - conf = this->private; - - /* Got the reserve lock. Check if lock is grantable and proceed - with the real lk call */ - - if (op_ret >= 0) { - /* Lock is grantable if flock reflects a successful getlk() call*/ - if (lock->l_type == F_UNLCK && lock->l_pid) { - gf_log (this->name, GF_LOG_INFO, - "Got the reservelk, but the lock is not grantable. "); - client_remove_reserve_lock (this, frame, local->client_lock); - goto out; - } - - gf_log (this->name, GF_LOG_DEBUG, "reserve lock succeeded"); - client_send_recovery_lock (frame, this, local->client_lock); - goto out; - } - - /* Somebody else has a reserve lk. Lock conflict detected. - Mark fd as bad */ - - gf_log (this->name, GF_LOG_WARNING, - "reservelk OP failed. aborting lock recovery"); - - client_mark_bad_fd (local->client_lock->fd, - local->fdctx); - destroy_client_lock (local->client_lock); - frame->local = NULL; - client_local_wipe (local); - STACK_DESTROY (frame->root); - - fd_count = decrement_reopen_fd_count (this, conf); - gf_log (this->name, GF_LOG_DEBUG, - "need to attempt lock recovery on %"PRIu64" open fds", - fd_count); - -out: - return 0; -} - -int32_t -client_recovery_lock_cbk (call_frame_t *frame, - void *cookie, - xlator_t *this, - int32_t op_ret, - int32_t op_errno, - struct gf_flock *lock, dict_t *xdata) -{ - clnt_local_t *local = NULL; - clnt_fd_ctx_t *fdctx = NULL; - clnt_conf_t *conf = NULL; - client_posix_lock_t *next_lock = NULL; - - struct gf_flock reserve_flock; - uint64_t fd_count = 0; - - local = frame->local; - conf = this->private; - - if (op_ret < 0) { - gf_log (this->name, GF_LOG_ERROR, - "lock recovery failed: %s", - strerror(op_errno)); - - client_mark_bad_fd (local->client_lock->fd, - local->fdctx); - goto cleanup; - - /* Lock recovered. Continue with reserve lock for next lock */ - } else { - gf_log (this->name, GF_LOG_DEBUG, - "lock recovered successfully - continuing with next lock."); - - next_lock = get_next_recovery_lock (this, local); - if (!next_lock) { - gf_log (this->name, GF_LOG_DEBUG, - "all locks recovered on fd"); - goto cleanup; - } - - construct_reserve_lock (next_lock, frame, &reserve_flock); - local->fdctx = fdctx; - local->client_lock = next_lock; - - STACK_WIND (frame, client_reserve_lock_cbk, - this, this->fops->lk, - next_lock->fd, F_RESLK_LCK, &reserve_flock, NULL); - goto out; - - } - -cleanup: - - frame->local = NULL; - client_local_wipe (local); - - if (local->client_lock) - destroy_client_lock (local->client_lock); - - STACK_DESTROY (frame->root); - - fd_count = decrement_reopen_fd_count (this, conf); - - gf_log (this->name, GF_LOG_DEBUG, - "need to attempt lock recovery on %"PRIu64" open fds", - fd_count); - -out: - return 0; -} - -static int -client_send_recovery_lock (call_frame_t *frame, xlator_t *this, - client_posix_lock_t *lock) -{ - frame->root->lk_owner = lock->owner; - - /* Send all locks as F_SETLK to prevent the frame - from blocking if there is a conflict */ - - STACK_WIND (frame, client_recovery_lock_cbk, - this, this->fops->lk, - lock->fd, F_SETLK, - &(lock->user_flock), NULL); - - return 0; -} - -static int -client_lockrec_init (clnt_fd_ctx_t *fdctx, clnt_local_t *local) -{ - - INIT_LIST_HEAD (&local->lock_list); - pthread_mutex_init (&local->mutex, NULL); - - pthread_mutex_lock (&fdctx->mutex); - { - list_splice_init (&fdctx->lock_list, &local->lock_list); - } - pthread_mutex_unlock (&fdctx->mutex); - - return 0; -} - - -int -client_attempt_lock_recovery (xlator_t *this, clnt_fd_ctx_t *fdctx) -{ - call_frame_t *frame = NULL; - clnt_local_t *local = NULL; - client_posix_lock_t *lock = NULL; - - struct gf_flock reserve_flock; - int ret = 0; - - local = mem_get0 (this->local_pool); - if (!local) { - ret = -ENOMEM; - goto out; - } - - client_lockrec_init (fdctx, local); - - lock = get_next_recovery_lock (this, local); - if (!lock) { - gf_log (this->name, GF_LOG_DEBUG, - "no locks found on fd"); - ret = -1; - goto out; - } - - frame = create_frame (this, this->ctx->pool); - if (!frame) { - gf_log (this->name, GF_LOG_ERROR, - "creating of frame failed, lock recovery failed"); - ret = -1; - goto out; - } - - construct_reserve_lock (lock, frame, &reserve_flock); - - frame->local = local; - local->fdctx = fdctx; - local->client_lock = lock; - - STACK_WIND (frame, client_reserve_lock_cbk, - this, this->fops->lk, - lock->fd, F_RESLK_LCK, &reserve_flock, NULL); - -out: - return ret; - - -} - int32_t client_dump_locks (char *name, inode_t *inode, dict_t *dict) diff --git a/xlators/protocol/client/src/client-rpc-fops.c b/xlators/protocol/client/src/client-rpc-fops.c index 9a30081a44c..36c05ed17c7 100644 --- a/xlators/protocol/client/src/client-rpc-fops.c +++ b/xlators/protocol/client/src/client-rpc-fops.c @@ -396,6 +396,7 @@ client_add_fd_to_saved_fds (xlator_t *this, fd_t *fd, loc_t *loc, int32_t flags, fdctx->flags = flags; fdctx->lk_ctx = fd_lk_ctx_ref (fd->lk_ctx); fdctx->lk_heal_state = GF_LK_HEAL_DONE; + fdctx->reopen_done = client_default_reopen_done; INIT_LIST_HEAD (&fdctx->sfd_pos); INIT_LIST_HEAD (&fdctx->lock_list); diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h index 9241eefa3aa..03e7f838739 100644 --- a/xlators/protocol/client/src/client.h +++ b/xlators/protocol/client/src/client.h @@ -130,6 +130,7 @@ typedef struct _client_fd_ctx { pthread_mutex_t mutex; lk_heal_state_t lk_heal_state; uuid_t gfid; + void (*reopen_done) (struct _client_fd_ctx*, xlator_t *); struct list_head lock_list; /* List of all granted locks on this fd */ } clnt_fd_ctx_t; @@ -223,7 +224,6 @@ int client_attempt_lock_recovery (xlator_t *this, clnt_fd_ctx_t *fdctx); int32_t delete_granted_locks_owner (fd_t *fd, gf_lkowner_t *owner); int client_add_lock_for_recovery (fd_t *fd, struct gf_flock *flock, gf_lkowner_t *owner, int32_t cmd); -uint64_t decrement_reopen_fd_count (xlator_t *this, clnt_conf_t *conf); int32_t delete_granted_locks_fd (clnt_fd_ctx_t *fdctx); int32_t client_cmd_to_gf_cmd (int32_t cmd, int32_t *gf_cmd); void client_save_number_fds (clnt_conf_t *conf, int count); @@ -243,4 +243,5 @@ int client_mark_fd_bad (xlator_t *this); int client_set_lk_version (xlator_t *this); int client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx, gf_boolean_t use_try_lock); +void client_default_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this); #endif /* !_CLIENT_H */ |