summaryrefslogtreecommitdiffstats
path: root/xlators/protocol/client/src
diff options
context:
space:
mode:
Diffstat (limited to 'xlators/protocol/client/src')
-rw-r--r--xlators/protocol/client/src/client-handshake.c481
-rw-r--r--xlators/protocol/client/src/client-lk.c1
-rw-r--r--xlators/protocol/client/src/client-mem-types.h1
-rw-r--r--xlators/protocol/client/src/client.c165
-rw-r--r--xlators/protocol/client/src/client.h16
-rw-r--r--xlators/protocol/client/src/client3_1-fops.c16
6 files changed, 654 insertions, 26 deletions
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 91cda6d0c45..1896e6b6391 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -22,6 +22,7 @@
#include "config.h"
#endif
+#include "fd-lk.h"
#include "client.h"
#include "xlator.h"
#include "defaults.h"
@@ -39,6 +40,18 @@ extern rpc_clnt_prog_t clnt_pmap_prog;
int client_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
void *myframe);
+int client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe);
+
+int client_set_lk_version (xlator_t *this);
+
+typedef struct client_fd_lk_local {
+ int ref;
+ gf_boolean_t error;
+ gf_lock_t lock;
+ clnt_fd_ctx_t *fdctx;
+}clnt_fd_lk_local_t;
+
/* Handshake */
void
@@ -391,6 +404,411 @@ client_notify_parents_child_up (xlator_t *this)
}
int
+client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ int32_t ret = -1;
+ call_frame_t *fr = NULL;
+ gf_set_lk_ver_rsp rsp = {0,};
+
+ fr = (call_frame_t *) myframe;
+ GF_VALIDATE_OR_GOTO ("client", fr, out);
+
+ if (req->rpc_status == -1) {
+ gf_log (fr->this->name, GF_LOG_WARNING,
+ "received RPC status error");
+ goto out;
+ }
+
+ ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_set_lk_ver_rsp);
+ if (ret < 0)
+ gf_log (fr->this->name, GF_LOG_WARNING,
+ "xdr decoding failed");
+ else
+ gf_log (fr->this->name, GF_LOG_DEBUG,
+ "Server lk version = %d", rsp.lk_ver);
+
+ ret = 0;
+out:
+ if (fr)
+ STACK_DESTROY (fr->root);
+
+ return ret;
+}
+
+int
+client_set_lk_version (xlator_t *this)
+{
+ int ret = -1;
+ clnt_conf_t *conf = NULL;
+ call_frame_t *frame = NULL;
+ gf_set_lk_ver_req req = {0, };
+
+ conf = (clnt_conf_t *) this->private;
+
+ req.lk_ver = client_get_lk_ver (conf);
+ req.uid = this->ctx->process_uuid;
+
+ gf_log (this->name, GF_LOG_DEBUG, "Sending SET_LK_VERSION");
+
+ frame = create_frame (this, this->ctx->pool);
+ if (!frame)
+ goto out;
+
+ ret = client_submit_request (this, &req, frame,
+ conf->handshake,
+ GF_HNDSK_SET_LK_VER,
+ client_set_lk_version_cbk,
+ NULL, NULL, 0, NULL, 0, NULL,
+ (xdrproc_t)xdr_gf_set_lk_ver_req);
+out:
+ if (ret < 0)
+ gf_log (this->name, GF_LOG_WARNING,
+ "Failed to send SET_LK_VERSION to server");
+
+ return ret;
+}
+
+int
+client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx)
+{
+ int ret = 1;
+
+ GF_VALIDATE_OR_GOTO ("client", lk_ctx, out);
+
+ LOCK (&lk_ctx->lock);
+ {
+ ret = list_empty (&lk_ctx->lk_list);
+ }
+ UNLOCK (&lk_ctx->lock);
+out:
+ return ret;
+}
+
+int
+client_fd_lk_count (fd_lk_ctx_t *lk_ctx)
+{
+ int count = 0;
+ fd_lk_ctx_node_t *fd_lk = NULL;
+
+ GF_VALIDATE_OR_GOTO ("client", lk_ctx, err);
+
+ LOCK (&lk_ctx->lock);
+ {
+ list_for_each_entry (fd_lk, &lk_ctx->lk_list, next)
+ count++;
+ }
+ UNLOCK (&lk_ctx->lock);
+
+ return count;
+err:
+ return -1;
+}
+
+clnt_fd_lk_local_t *
+clnt_fd_lk_local_ref (xlator_t *this, clnt_fd_lk_local_t *local)
+{
+ GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+ LOCK (&local->lock);
+ {
+ local->ref++;
+ }
+ UNLOCK (&local->lock);
+out:
+ return local;
+}
+
+int
+clnt_fd_lk_local_unref (xlator_t *this, clnt_fd_lk_local_t *local)
+{
+ int ref = -1;
+
+ GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+ LOCK (&local->lock);
+ {
+ ref = --local->ref;
+ }
+ UNLOCK (&local->lock);
+
+ if (ref == 0) {
+ LOCK_DESTROY (&local->lock);
+ GF_FREE (local);
+ }
+ ref = 0;
+out:
+ return ref;
+}
+
+clnt_fd_lk_local_t *
+clnt_fd_lk_local_create (clnt_fd_ctx_t *fdctx)
+{
+ clnt_fd_lk_local_t *local = NULL;
+
+ local = GF_CALLOC (1, sizeof (clnt_fd_lk_local_t),
+ gf_client_mt_clnt_fd_lk_local_t);
+ if (!local)
+ goto out;
+
+ local->ref = 1;
+ local->error = _gf_false;
+ local->fdctx = fdctx;
+
+ LOCK_INIT (&local->lock);
+out:
+ return local;
+}
+
+void
+clnt_mark_fd_bad (clnt_conf_t *conf, clnt_fd_ctx_t *fdctx)
+{
+ pthread_mutex_lock (&conf->lock);
+ {
+ fdctx->remote_fd = -1;
+ }
+ pthread_mutex_unlock (&conf->lock);
+}
+
+// call decrement_reopen_fd_count
+int
+clnt_release_reopen_fd_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ xlator_t *this = NULL;
+ call_frame_t *frame = NULL;
+ clnt_conf_t *conf = NULL;
+ clnt_fd_ctx_t *fdctx = NULL;
+
+ frame = myframe;
+ this = frame->this;
+ fdctx = (clnt_fd_ctx_t *) frame->local;
+ conf = (clnt_conf_t *) this->private;
+
+ clnt_mark_fd_bad (conf, fdctx);
+
+ decrement_reopen_fd_count (this, conf);
+
+ frame->local = NULL;
+ STACK_DESTROY (frame->root);
+
+ return 0;
+}
+
+int
+clnt_release_reopen_fd (xlator_t *this, clnt_fd_ctx_t *fdctx)
+{
+ int ret = -1;
+ clnt_conf_t *conf = NULL;
+ call_frame_t *frame = NULL;
+ gfs3_release_req req = {{0,},};
+
+ conf = (clnt_conf_t *) this->private;
+
+ frame = create_frame (THIS, THIS->ctx->pool);
+ if (!frame)
+ goto out;
+
+ frame->local = (void *) fdctx;
+ req.fd = fdctx->remote_fd;
+
+ ret = client_submit_request (this, &req, frame, conf->fops,
+ GFS3_OP_RELEASE,
+ clnt_release_reopen_fd_cbk, NULL,
+ NULL, 0, NULL, 0, NULL,
+ (xdrproc_t)xdr_gfs3_releasedir_req);
+out:
+ if (ret) {
+ decrement_reopen_fd_count (this, conf);
+ clnt_mark_fd_bad (conf, fdctx);
+ if (frame) {
+ frame->local = NULL;
+ STACK_DESTROY (frame->root);
+ }
+ }
+
+ return 0;
+}
+
+int
+clnt_fd_lk_local_mark_error (xlator_t *this,
+ clnt_fd_lk_local_t *local)
+{
+ gf_boolean_t error = _gf_false;
+
+ LOCK (&local->lock);
+ {
+ error = local->error;
+ local->error = _gf_true;
+ }
+ UNLOCK (&local->lock);
+
+ if (error)
+ clnt_release_reopen_fd (this, local->fdctx);
+
+ return 0;
+}
+
+// Also, I think in reopen_cbk, the fdctx is added to
+// saved_fd list.. avoid that, may cause a problem
+// Reason: While the locks on the fd are reacquired, a release
+// fop may be received by the client-protocol translator
+// which will free the fdctx datastructure.
+int
+client_reacquire_lock_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ int32_t ret = -1;
+ xlator_t *this = NULL;
+ gf_common_rsp rsp = {0,};
+ call_frame_t *frame = NULL;
+ clnt_fd_lk_local_t *local = NULL;
+
+ frame = (call_frame_t *) myframe;
+ this = frame->this;
+ local = (clnt_fd_lk_local_t *) frame->local;
+
+ if (req->rpc_status == -1) {
+ gf_log ("client", GF_LOG_WARNING,
+ "request failed at rpc");
+ goto out;
+ }
+
+ ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+ if (ret < 0) {
+ gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+ goto out;
+ }
+
+ if (rsp.op_ret == -1) {
+ gf_log (this->name, GF_LOG_ERROR, "lock request failed");
+ ret = -1;
+ goto out;
+ }
+
+ // TODO: Add more info to log.
+ gf_log (this->name, GF_LOG_DEBUG, "Reacquired lock");
+
+ ret = 0;
+out:
+ if (ret < 0)
+ clnt_fd_lk_local_mark_error (this, local);
+
+ (void) clnt_fd_lk_local_unref (this, local);
+ frame->local = NULL;
+ STACK_DESTROY (frame->root);
+
+ return ret;
+}
+
+int
+_client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
+{
+ int32_t ret = -1;
+ int32_t gf_cmd = 0;
+ int32_t gf_type = 0;
+ gfs3_lk_req req = {{0,},};
+ struct gf_flock flock = {0,};
+ fd_lk_ctx_t *lk_ctx = NULL;
+ clnt_fd_lk_local_t *local = NULL;
+ fd_lk_ctx_node_t *fd_lk = NULL;
+ call_frame_t *frame = NULL;
+ clnt_conf_t *conf = NULL;
+
+ conf = (clnt_conf_t *) this->private;
+ lk_ctx = fdctx->lk_ctx;
+
+ local = clnt_fd_lk_local_create (fdctx);
+ if (!local) {
+ clnt_release_reopen_fd (this, fdctx);
+ goto out;
+ }
+
+ list_for_each_entry (fd_lk, &lk_ctx->lk_list, next) {
+ memcpy (&flock, &fd_lk->user_flock,
+ sizeof (struct gf_flock));
+
+ ret = client_cmd_to_gf_cmd (fd_lk->cmd, &gf_cmd);
+ if (ret) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "client_cmd_to_gf_cmd failed, "
+ "aborting reacquiring of locks");
+ break;
+ }
+
+ gf_type = client_type_to_gf_type (flock.l_type);
+ req.fd = fdctx->remote_fd;
+ req.cmd = gf_cmd;
+ req.type = gf_type;
+ (void) gf_proto_flock_from_flock (&req.flock,
+ &flock);
+
+ memcpy (req.gfid, fdctx->inode->gfid, 16);
+
+ frame = create_frame (THIS, THIS->ctx->pool);
+ if (!frame) {
+ ret = -1;
+ break;
+ }
+
+ frame->local = clnt_fd_lk_local_ref (this, local);
+ frame->root->lk_owner = fd_lk->user_flock.l_owner;
+
+ ret = client_submit_request (this, &req, frame,
+ conf->fops, GFS3_OP_LK,
+ client_reacquire_lock_cbk,
+ NULL, NULL, 0, NULL, 0, NULL,
+ (xdrproc_t)xdr_gfs3_lk_req);
+ if (ret)
+ break;
+
+ ret = 0;
+ frame = NULL;
+ }
+
+ if (ret) {
+ clnt_fd_lk_local_mark_error (this, local);
+
+ if (frame) {
+ if (frame->local) {
+ clnt_fd_lk_local_unref (this, frame->local);
+ frame->local = NULL;
+ }
+ STACK_DESTROY (frame->root);
+ }
+ }
+ if (local)
+ (void) clnt_fd_lk_local_unref (this, local);
+out:
+ return ret;
+}
+
+int
+client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
+{
+ int32_t ret = -1;
+ fd_lk_ctx_t *lk_ctx = NULL;
+
+ if (client_fd_lk_list_empty (fdctx->lk_ctx)) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "fd lock list is empty");
+ decrement_reopen_fd_count (this, (clnt_conf_t *)this->private);
+ ret = 0;
+ goto out;
+ }
+
+ lk_ctx = fdctx->lk_ctx;
+
+ LOCK (&lk_ctx->lock);
+ {
+ ret = _client_reacquire_lock (this, fdctx);
+ }
+ UNLOCK (&lk_ctx->lock);
+out:
+ return ret;
+}
+
+int
client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
void *myframe)
{
@@ -402,11 +820,13 @@ client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
clnt_conf_t *conf = NULL;
clnt_fd_ctx_t *fdctx = NULL;
call_frame_t *frame = NULL;
+ xlator_t *this = NULL;
frame = myframe;
if (!frame || !frame->this)
goto out;
+ this = frame->this;
local = frame->local;
conf = frame->this->private;
@@ -454,7 +874,7 @@ client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
fdctx->remote_fd = rsp.fd;
if (!fdctx->released) {
list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
- if (!list_empty (&fdctx->lock_list))
+ if (!client_fd_lk_list_empty (fdctx->lk_ctx))
attempt_lock_recovery = _gf_true;
fdctx = NULL;
}
@@ -463,31 +883,27 @@ client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
ret = 0;
- attempt_lock_recovery = _gf_false; /* temporarily */
-
- if (attempt_lock_recovery) {
- ret = client_attempt_lock_recovery (frame->this, local->fdctx);
- if (ret < 0) {
- gf_log (frame->this->name, GF_LOG_DEBUG,
- "lock recovery not attempted on fd");
- } else {
- gf_log (frame->this->name, GF_LOG_INFO,
- "need to attempt lock recovery on %"PRIu64
- " open fds", fd_count);
- }
+ if (conf->lk_heal && attempt_lock_recovery) {
+ /* Delay decrement the reopen fd count untill all the
+ locks corresponding to this fd are acquired.*/
+ gf_log (frame->this->name, GF_LOG_WARNING, "acquiring locks on "
+ "%s", local->loc.path);
+ ret = client_reacquire_lock (frame->this, local->fdctx);
} else {
fd_count = decrement_reopen_fd_count (frame->this, conf);
}
out:
if (fdctx)
- client_fdctx_destroy (frame->this, fdctx);
+ client_fdctx_destroy (this, fdctx);
if ((ret < 0) && frame && frame->this && conf)
decrement_reopen_fd_count (frame->this, conf);
- frame->local = NULL;
- STACK_DESTROY (frame->root);
+ if (frame) {
+ frame->local = NULL;
+ STACK_DESTROY (frame->root);
+ }
client_local_wipe (local);
@@ -792,7 +1208,8 @@ client_post_handshake (call_frame_t *frame, xlator_t *this)
}
} else {
gf_log (this->name, GF_LOG_DEBUG,
- "no open fds - notifying all parents child up");
+ "no fds to open - notifying all parents child up");
+ client_set_lk_version (this);
client_notify_parents_child_up (this);
}
out:
@@ -814,6 +1231,7 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
int32_t op_ret = 0;
int32_t op_errno = 0;
gf_boolean_t auth_fail = _gf_false;
+ uint32_t lk_ver = 0;
frame = myframe;
this = frame->this;
@@ -895,6 +1313,15 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
goto out;
}
+ ret = dict_get_uint32 (reply, "clnt-lk-version", &lk_ver);
+ if (ret) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "failed to find key 'clnt-lk-version' in the options");
+ goto out;
+ }
+
+ gf_log (this->name, GF_LOG_INFO, "clnt-lk-version = %d, "
+ "server-lk-version = %d", client_get_lk_ver (conf), lk_ver);
/* TODO: currently setpeer path is broken */
/*
if (process_uuid && req->conn &&
@@ -930,8 +1357,15 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
conf->need_different_port = 0;
- /* TODO: more to test */
- client_post_handshake (frame, frame->this);
+ if (lk_ver != client_get_lk_ver (conf)) {
+ client_mark_fd_bad (this);
+ client_post_handshake (frame, frame->this);
+ } else {
+ /*TODO: Traverse the saved fd list, and send
+ release to the server on fd's that were closed
+ during grace period */
+ ;
+ }
out:
if (auth_fail) {
@@ -1043,6 +1477,14 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc)
"failed to set 'volfile-checksum'");
}
+ ret = dict_set_int16 (options, "clnt-lk-version",
+ client_get_lk_ver (conf));
+ if (ret < 0) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "failed to set clnt-lk-version(%"PRIu32") in handshake msg",
+ client_get_lk_ver (conf));
+ }
+
req.dict.dict_len = dict_serialized_length (options);
if (req.dict.dict_len < 0) {
gf_log (this->name, GF_LOG_ERROR,
@@ -1366,6 +1808,7 @@ char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
[GF_HNDSK_SETVOLUME] = "SETVOLUME",
[GF_HNDSK_GETSPEC] = "GETSPEC",
[GF_HNDSK_PING] = "PING",
+ [GF_HNDSK_SET_LK_VER] = "SET_LK_VER"
};
rpc_clnt_prog_t clnt_handshake_prog = {
diff --git a/xlators/protocol/client/src/client-lk.c b/xlators/protocol/client/src/client-lk.c
index 842e3ec5b62..e99fe774de6 100644
--- a/xlators/protocol/client/src/client-lk.c
+++ b/xlators/protocol/client/src/client-lk.c
@@ -608,6 +608,7 @@ decrement_reopen_fd_count (xlator_t *this, clnt_conf_t *conf)
if (fd_count == 0) {
gf_log (this->name, GF_LOG_INFO,
"last fd open'd/lock-self-heal'd - notifying CHILD-UP");
+ client_set_lk_version (this);
client_notify_parents_child_up (this);
}
diff --git a/xlators/protocol/client/src/client-mem-types.h b/xlators/protocol/client/src/client-mem-types.h
index c2aa690b1c5..6bc7daad271 100644
--- a/xlators/protocol/client/src/client-mem-types.h
+++ b/xlators/protocol/client/src/client-mem-types.h
@@ -29,6 +29,7 @@ enum gf_client_mem_types_ {
gf_client_mt_clnt_req_buf_t,
gf_client_mt_clnt_fdctx_t,
gf_client_mt_clnt_lock_t,
+ gf_client_mt_clnt_fd_lk_local_t,
gf_client_mt_end,
};
#endif /* __CLIENT_MEM_TYPES_H__ */
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 229e0191725..8955e237dee 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -40,6 +40,81 @@ int client_handshake (xlator_t *this, struct rpc_clnt *rpc);
void client_start_ping (void *data);
int client_init_rpc (xlator_t *this);
int client_destroy_rpc (xlator_t *this);
+int client_mark_fd_bad (xlator_t *this);
+
+int32_t
+client_type_to_gf_type (short l_type)
+{
+ int32_t gf_type;
+
+ switch (l_type) {
+ case F_RDLCK:
+ gf_type = GF_LK_F_RDLCK;
+ break;
+ case F_WRLCK:
+ gf_type = GF_LK_F_WRLCK;
+ break;
+ case F_UNLCK:
+ gf_type = GF_LK_F_UNLCK;
+ break;
+ }
+
+ return gf_type;
+}
+
+uint32_t
+client_get_lk_ver (clnt_conf_t *conf)
+{
+ uint32_t lk_ver = 0;
+
+ GF_VALIDATE_OR_GOTO ("client", conf, out);
+
+ pthread_mutex_lock (&conf->lock);
+ {
+ lk_ver = conf->lk_version;
+ }
+ pthread_mutex_unlock (&conf->lock);
+out:
+ return lk_ver;
+}
+
+void
+client_grace_timeout (void *data)
+{
+ int ver = 0;
+ xlator_t *this = NULL;
+ struct clnt_conf *conf = NULL;
+ struct rpc_clnt *rpc = NULL;
+
+ GF_VALIDATE_OR_GOTO ("client", data, out);
+
+ this = THIS;
+
+ rpc = (struct rpc_clnt *) data;
+
+ conf = (struct clnt_conf *) this->private;
+
+ pthread_mutex_lock (&conf->lock);
+ {
+ ver = ++conf->lk_version;
+ /* ver == 0 is a special value used by server
+ to notify client that this is a fresh connect.*/
+ if (ver == 0)
+ ver = ++conf->lk_version;
+
+ gf_timer_call_cancel (this->ctx, conf->grace_timer);
+ conf->grace_timer = NULL;
+ }
+ pthread_mutex_unlock (&conf->lock);
+
+ gf_log (this->name, GF_LOG_WARNING,
+ "client grace timer expired, updating "
+ "the lk-version to %d", ver);
+
+ client_mark_fd_bad (this);
+out:
+ return;
+}
int
client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
@@ -828,7 +903,6 @@ out:
}
-
int32_t
client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
{
@@ -1455,7 +1529,6 @@ out:
return 0;
}
-
int32_t
client_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
struct gf_flock *lock)
@@ -1841,7 +1914,7 @@ out:
}
- int
+int
client_mark_fd_bad (xlator_t *this)
{
clnt_conf_t *conf = NULL;
@@ -1908,11 +1981,42 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
conf->last_sent_event = GF_EVENT_CHILD_UP;
}
}
+
+ /* Cancel grace timer if set */
+ pthread_mutex_lock (&conf->lock);
+ {
+ if (conf->grace_timer) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "Cancelling the grace timer");
+
+ gf_timer_call_cancel (this->ctx,
+ conf->grace_timer);
+ conf->grace_timer = NULL;
+ }
+ }
+ pthread_mutex_unlock (&conf->lock);
+
break;
}
case RPC_CLNT_DISCONNECT:
+ /* client_mark_fd_bad (this); */
- client_mark_fd_bad (this);
+ pthread_mutex_lock (&conf->lock);
+ {
+ if (conf->grace_timer) {
+ gf_log (this->name, GF_LOG_DEBUG,
+ "Client grace timer is already set");
+ } else {
+ gf_log (this->name, GF_LOG_WARNING,
+ "Registering a grace timer");
+ conf->grace_timer =
+ gf_timer_call_after (this->ctx,
+ conf->grace_tv,
+ client_grace_timeout,
+ conf->rpc);
+ }
+ }
+ pthread_mutex_unlock (&conf->lock);
if (!conf->skip_notify) {
if (conf->connected)
@@ -2107,6 +2211,40 @@ out:
int
+client_init_grace_timer (xlator_t *this, dict_t *options,
+ clnt_conf_t *conf)
+{
+ char *lk_heal = NULL;
+ int32_t ret = -1;
+ int32_t grace_timeout = -1;
+
+ GF_VALIDATE_OR_GOTO ("client", this, out);
+ GF_VALIDATE_OR_GOTO (this->name, options, out);
+ GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+ conf->lk_heal = _gf_true;
+
+ ret = dict_get_str (options, "lk-heal", &lk_heal);
+ if (!ret)
+ gf_string2boolean (lk_heal, &conf->lk_heal);
+
+ ret = dict_get_int32 (options, "grace-timeout", &grace_timeout);
+ if (!ret)
+ conf->grace_tv.tv_sec = grace_timeout;
+ else
+ conf->grace_tv.tv_sec = 10;
+
+ conf->grace_tv.tv_usec = 0;
+
+ gf_log (this->name, GF_LOG_INFO, "lk-heal = %s",
+ (conf->lk_heal) ? "on" : "off");
+
+ ret = 0;
+out:
+ return ret;
+}
+
+int
reconfigure (xlator_t *this, dict_t *options)
{
clnt_conf_t *conf = NULL;
@@ -2153,6 +2291,10 @@ reconfigure (xlator_t *this, dict_t *options)
}
}
+ ret = client_init_grace_timer (this, options, conf);
+ if (ret)
+ goto out;
+
ret = 0;
out:
return ret;
@@ -2186,6 +2328,14 @@ init (xlator_t *this)
pthread_mutex_init (&conf->lock, NULL);
INIT_LIST_HEAD (&conf->saved_fds);
+ /* Initialize parameters for lock self healing*/
+ conf->lk_version = 1;
+ conf->grace_timer = NULL;
+
+ ret = client_init_grace_timer (this, this->options, conf);
+ if (ret)
+ goto out;
+
LOCK_INIT (&conf->rec_lock);
conf->last_sent_event = -1; /* To start with we don't have any events */
@@ -2207,7 +2357,6 @@ init (xlator_t *this)
goto out;
}
-
ret = client_init_rpc (this);
out:
if (ret)
@@ -2409,5 +2558,11 @@ struct volume_options options[] = {
{ .key = {"client-bind-insecure"},
.type = GF_OPTION_TYPE_BOOL
},
+ { .key = {"lk-heal"},
+ .type = GF_OPTION_TYPE_STR
+ },
+ { .key = {"grace-timeout"},
+ .type = GF_OPTION_TYPE_INT
+ },
{ .key = {NULL} },
};
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 2dda451c9cb..00addf34ceb 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -29,6 +29,7 @@
#include "client-mem-types.h"
#include "protocol-common.h"
#include "glusterfs3.h"
+#include "fd-lk.h"
/* FIXME: Needs to be defined in a common file */
#define CLIENT_CMD_CONNECT "trusted.glusterfs.client-connect"
@@ -91,6 +92,12 @@ typedef struct clnt_conf {
char need_different_port; /* flag used to change the
portmap path in case of
'tcp,rdma' on server */
+ gf_boolean_t lk_heal;
+ uint16_t lk_version; /* this variable is used to distinguish
+ client-server transaction while
+ performing lock healing */
+ struct timeval grace_tv;
+ gf_timer_t *grace_timer;
} clnt_conf_t;
typedef struct _client_fd_ctx {
@@ -105,7 +112,7 @@ typedef struct _client_fd_ctx {
char released;
int32_t flags;
int32_t wbflags;
-
+ fd_lk_ctx_t *lk_ctx;
pthread_mutex_t mutex;
struct list_head lock_list; /* List of all granted locks on this fd */
} clnt_fd_ctx_t;
@@ -211,4 +218,11 @@ int32_t client_dump_locks (char *name, inode_t *inode,
dict_t *dict);
int client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx);
+uint32_t client_get_lk_ver (clnt_conf_t *conf);
+
+int32_t client_type_to_gf_type (short l_type);
+
+int client_mark_fd_bad (xlator_t *this);
+
+int client_set_lk_version (xlator_t *this);
#endif /* !_CLIENT_H */
diff --git a/xlators/protocol/client/src/client3_1-fops.c b/xlators/protocol/client/src/client3_1-fops.c
index 76d4fb0d691..4d6d57528f3 100644
--- a/xlators/protocol/client/src/client3_1-fops.c
+++ b/xlators/protocol/client/src/client3_1-fops.c
@@ -351,6 +351,7 @@ client3_1_open_cbk (struct rpc_req *req, struct iovec *iov, int count,
fdctx->inode = inode_ref (fd->inode);
fdctx->flags = local->flags;
fdctx->wbflags = local->wbflags;
+ fdctx->lk_ctx = fd_lk_ctx_ref (fd->lk_ctx);
INIT_LIST_HEAD (&fdctx->sfd_pos);
INIT_LIST_HEAD (&fdctx->lock_list);
@@ -2279,17 +2280,30 @@ client3_1_releasedir_cbk (struct rpc_req *req, struct iovec *iov, int count,
int
client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx)
{
+ clnt_conf_t *conf = NULL;
call_frame_t *fr = NULL;
int32_t ret = -1;
+ fd_lk_ctx_t *lk_ctx = NULL;
if (!fdctx)
goto out;
+ conf = (clnt_conf_t *) this->private;
+
if (fdctx->remote_fd == -1) {
gf_log (this->name, GF_LOG_DEBUG, "not a valid fd");
goto out;
}
+ pthread_mutex_lock (&conf->lock);
+ {
+ lk_ctx = fdctx->lk_ctx;
+ fdctx->lk_ctx = NULL;
+ }
+ pthread_mutex_unlock (&conf->lock);
+
+ fd_lk_ctx_unref (lk_ctx);
+
fr = create_frame (this, this->ctx->pool);
if (fdctx->is_dir) {
@@ -4466,7 +4480,6 @@ unwind:
return 0;
}
-
int32_t
client3_1_lk (call_frame_t *frame, xlator_t *this,
void *data)
@@ -4523,6 +4536,7 @@ client3_1_lk (call_frame_t *frame, xlator_t *this,
req.cmd = gf_cmd;
req.type = gf_type;
gf_proto_flock_from_flock (&req.flock, args->flock);
+
memcpy (req.gfid, args->fd->inode->gfid, 16);
ret = client_submit_request (this, &req, frame, conf->fops, GFS3_OP_LK,
generated by cgit (git 2.34.1) at 2024-11-19 13:25:31 +0000