summaryrefslogtreecommitdiffstats
path: root/xlators/protocol/server
diff options
context:
space:
mode:
authorMohammed Junaid <junaid@redhat.com>2012-02-08 18:06:39 +0530
committerVijay Bellur <vijay@gluster.com>2012-02-20 04:45:31 -0800
commitf764516c2e526624ce0088963924ff2d88304553 (patch)
tree85262797baad440b12853a3a6ad41ab518d9f996 /xlators/protocol/server
parent4d1b040f00e7ec8de997d151b35fa035bba9cb25 (diff)
protocol/client,server: fcntl lock self healing.
Currently(with out this patch), on a disconnect the server cleans up the transport which inturn closes the fd's and releases the locks acquired on those fd's by that client. On a reconnect, client just reopens the fd's but doesn't reacquire the locks. The application that had previously acquired the locks still is under the assumption that it is the owner of those locks which might have been granted to other clients(if they request) by the server leading to data corruption. This patch allows the client to reacquire the fcntl locks (held on the fd's) during client-server handshake. * The server identifies the client via process-uuid-xl (which is a combination of uuid and client-protocol name, it is assumed to be unique) and lk-version number. * The client maintains a list of process-uuid-xl, lk-version pair for each accepted connection. On a connect, the server traverses the list for a matching pair, if a matching pair is not found the the server returns lk-version with value 0, else it returns the lk-version it has in store. * On a disconnect, the server and client enter grace period, and on the completion of the grace period, the client bumps up its lk-version number (which means, it will reacquire the locks the next time) and the server will distroy the connection. If reconnection happens within the grace period, the server will find the matching (process-uuid-xl, lk-version) pair in its list which guarantees that the fd's and there corresponding locks are still valid for this client. Configurable options: To set grace-timeout, the following options are option server.grace-timeout value option client.grace-timeout value To enable or disable the lk-heal, option lk-heal [on|off] gluster volume set command can be used to configurable options Change-Id: Id677ef1087b300d649f278b8b2aa0d94eae85ed2 BUG: 795386 Signed-off-by: Mohammed Junaid <junaid@redhat.com> Reviewed-on: http://review.gluster.com/2766 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Vijay Bellur <vijay@gluster.com>
Diffstat (limited to 'xlators/protocol/server')
-rw-r--r--xlators/protocol/server/src/server-handshake.c78
-rw-r--r--xlators/protocol/server/src/server-helpers.c48
-rw-r--r--xlators/protocol/server/src/server-helpers.h6
-rw-r--r--xlators/protocol/server/src/server-mem-types.h1
-rw-r--r--xlators/protocol/server/src/server.c96
-rw-r--r--xlators/protocol/server/src/server.h5
6 files changed, 213 insertions, 21 deletions
diff --git a/xlators/protocol/server/src/server-handshake.c b/xlators/protocol/server/src/server-handshake.c
index 2c8cf059b..374f5a49a 100644
--- a/xlators/protocol/server/src/server-handshake.c
+++ b/xlators/protocol/server/src/server-handshake.c
@@ -354,6 +354,7 @@ server_setvolume (rpcsvc_request_t *req)
int32_t op_errno = EINVAL;
int32_t fop_version = 0;
int32_t mgmt_version = 0;
+ uint32_t lk_version = 0;
char *buf = NULL;
params = dict_new ();
@@ -408,8 +409,33 @@ server_setvolume (rpcsvc_request_t *req)
goto fail;
}
+ /*lk_verion :: [1..2^31-1]*/
+ ret = dict_get_uint32 (params, "clnt-lk-version", &lk_version);
+ if (ret < 0) {
+ ret = dict_set_str (reply, "ERROR",
+ "lock state verison not supplied");
+ if (ret < 0)
+ gf_log (this->name, GF_LOG_DEBUG,
+ "failed to set error msg");
+
+ op_ret = -1;
+ op_errno = EINVAL;
+ goto fail;
+ }
conn = server_connection_get (this, process_uuid);
+ if (!conn) {
+ op_ret = -1;
+ op_errno = ENOMEM;
+ goto fail;
+ }
+
+ server_cancel_conn_timer (this, conn);
+ if (conn->lk_version != 0 &&
+ conn->lk_version != lk_version) {
+ (void) server_connection_cleanup (this, conn);
+ }
+
if (req->trans->xl_private != conn)
req->trans->xl_private = conn;
@@ -595,6 +621,12 @@ server_setvolume (rpcsvc_request_t *req)
gf_log (this->name, GF_LOG_DEBUG,
"failed to set 'process-uuid'");
+ ret = dict_set_uint32 (reply, "clnt-lk-version",
+ conn->lk_version);
+ if (ret)
+ gf_log (this->name, GF_LOG_WARNING,
+ "failed to set 'clnt-lk-version'");
+
ret = dict_set_uint64 (reply, "transport-ptr",
((uint64_t) (long) req->trans));
if (ret)
@@ -663,12 +695,50 @@ server_ping (rpcsvc_request_t *req)
return 0;
}
+int
+server_set_lk_version (rpcsvc_request_t *req)
+{
+ int op_ret = -1;
+ int op_errno = EINVAL;
+ gf_set_lk_ver_req args = {0, };
+ gf_set_lk_ver_rsp rsp = {0,};
+ server_connection_t *conn = NULL;
+ xlator_t *this = NULL;
+
+ this = req->svc->mydata;
+ //TODO: Decide on an appropriate errno for the error-path
+ //below
+ if (!this)
+ goto fail;
+
+ if (!xdr_to_generic (req->msg[0], &args,
+ (xdrproc_t)xdr_gf_set_lk_ver_req)) {
+ //failed to decode msg;
+ req->rpc_err = GARBAGE_ARGS;
+ goto fail;
+ }
+
+ conn = server_connection_get (this, args.uid);
+ conn->lk_version = args.lk_ver;
+ server_connection_put (this, conn);
+
+ rsp.lk_ver = args.lk_ver;
+
+ op_ret = 0;
+fail:
+ rsp.op_ret = op_ret;
+ rsp.op_errno = op_errno;
+ server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
+ (xdrproc_t)xdr_gf_set_lk_ver_rsp);
+ return 0;
+}
rpcsvc_actor_t gluster_handshake_actors[] = {
- [GF_HNDSK_NULL] = {"NULL", GF_HNDSK_NULL, server_null, NULL, NULL, 0},
- [GF_HNDSK_SETVOLUME] = {"SETVOLUME", GF_HNDSK_SETVOLUME, server_setvolume, NULL, NULL, 0},
- [GF_HNDSK_GETSPEC] = {"GETSPEC", GF_HNDSK_GETSPEC, server_getspec, NULL, NULL, 0},
- [GF_HNDSK_PING] = {"PING", GF_HNDSK_PING, server_ping, NULL, NULL, 0},
+ [GF_HNDSK_NULL] = {"NULL", GF_HNDSK_NULL, server_null, NULL, NULL, 0},
+ [GF_HNDSK_SETVOLUME] = {"SETVOLUME", GF_HNDSK_SETVOLUME, server_setvolume, NULL, NULL, 0},
+ [GF_HNDSK_GETSPEC] = {"GETSPEC", GF_HNDSK_GETSPEC, server_getspec, NULL, NULL, 0},
+ [GF_HNDSK_PING] = {"PING", GF_HNDSK_PING, server_ping, NULL, NULL, 0},
+ [GF_HNDSK_SET_LK_VER] = {"SET_LK_VER", GF_HNDSK_SET_LK_VER, server_set_lk_version, NULL, NULL },
};
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 4980424d3..9de1082dc 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -774,6 +774,7 @@ server_connection_t *
server_connection_get (xlator_t *this, const char *id)
{
server_connection_t *conn = NULL;
+ server_connection_t *trav = NULL;
server_conf_t *conf = NULL;
GF_VALIDATE_OR_GOTO ("server", this, out);
@@ -783,20 +784,29 @@ server_connection_get (xlator_t *this, const char *id)
pthread_mutex_lock (&conf->mutex);
{
+ list_for_each_entry (trav, &conf->conns, list) {
+ if (!strncmp (trav->id, id, strlen (id))) {
+ conn = trav;
+ conn->ref++;
+ goto unlock;
+ }
+ }
+
conn = (void *) GF_CALLOC (1, sizeof (*conn),
gf_server_mt_conn_t);
if (!conn)
goto unlock;
conn->id = gf_strdup (id);
+ /*'0' denotes uninitialised lock state*/
+ conn->lk_version = 0;
+ conn->ref++;
conn->fdtable = gf_fd_fdtable_alloc ();
conn->ltable = gf_lock_table_new ();
conn->this = this;
pthread_mutex_init (&conn->lock, NULL);
-
list_add (&conn->list, &conf->conns);
- conn->ref++;
}
unlock:
pthread_mutex_unlock (&conf->mutex);
@@ -982,6 +992,17 @@ out:
return ret;
}
+void
+put_server_conn_state (xlator_t *this, rpc_transport_t *xprt)
+{
+ GF_VALIDATE_OR_GOTO ("server", this, out);
+ GF_VALIDATE_OR_GOTO ("server", xprt, out);
+
+ xprt->xl_private = NULL;
+out:
+ return;
+}
+
server_connection_t *
get_server_conn_state (xlator_t *this, rpc_transport_t *xprt)
{
@@ -1497,3 +1518,26 @@ gf_server_check_setxattr_cmd (call_frame_t *frame, dict_t *dict)
return 0;
}
+
+void
+server_cancel_conn_timer (xlator_t *this, server_connection_t *conn)
+{
+ if (!this || !conn) {
+ gf_log (THIS->name, GF_LOG_ERROR, "Invalid arguments to "
+ "cancel connection timer");
+ return;
+ }
+
+ pthread_mutex_lock (&conn->lock);
+ {
+ if (!conn->timer)
+ goto unlock;
+
+ gf_timer_call_cancel (this->ctx, conn->timer);
+ conn->timer = NULL;
+ }
+unlock:
+ pthread_mutex_unlock (&conn->lock);
+
+ return;
+}
diff --git a/xlators/protocol/server/src/server-helpers.h b/xlators/protocol/server/src/server-helpers.h
index 844c98c27..99ba7e546 100644
--- a/xlators/protocol/server/src/server-helpers.h
+++ b/xlators/protocol/server/src/server-helpers.h
@@ -68,6 +68,12 @@ server_print_request (call_frame_t *frame);
call_frame_t *
get_frame_from_request (rpcsvc_request_t *req);
+void
+server_cancel_conn_timer (xlator_t *this, server_connection_t *conn);
+
+void
+put_server_conn_state (xlator_t *this, rpc_transport_t *xprt);
+
server_connection_t *
get_server_conn_state (xlator_t *this, rpc_transport_t *xptr);
diff --git a/xlators/protocol/server/src/server-mem-types.h b/xlators/protocol/server/src/server-mem-types.h
index 88bae8cb4..5438ed6db 100644
--- a/xlators/protocol/server/src/server-mem-types.h
+++ b/xlators/protocol/server/src/server-mem-types.h
@@ -33,6 +33,7 @@ enum gf_server_mem_types_ {
gf_server_mt_dirent_rsp_t,
gf_server_mt_rsp_buf_t,
gf_server_mt_volfile_ctx_t,
+ gf_server_mt_timer_data_t,
gf_server_mt_end,
};
#endif /* __SERVER_MEM_TYPES_H__ */
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index b0697bb7b..b45b77baa 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -36,6 +36,26 @@
#include "authenticate.h"
#include "rpcsvc.h"
+void
+grace_time_handler (void *data)
+{
+ server_connection_t *conn = NULL;
+ xlator_t *this = NULL;
+
+ conn = data;
+ this = conn->this;
+
+ GF_VALIDATE_OR_GOTO (THIS->name, conn, out);
+ GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+
+ gf_log (this->name, GF_LOG_INFO, "grace timer expired");
+
+ server_cancel_conn_timer (this, conn);
+ server_connection_put (this, conn);
+out:
+ return;
+}
+
struct iobuf *
gfs_serialize_reply (rpcsvc_request_t *req, void *arg, struct iovec *outmsg,
xdrproc_t xdrproc)
@@ -554,11 +574,10 @@ int
server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
void *data)
{
- xlator_t *this = NULL;
- rpc_transport_t *xprt = NULL;
- server_connection_t *conn = NULL;
- server_conf_t *conf = NULL;
-
+ xlator_t *this = NULL;
+ rpc_transport_t *xprt = NULL;
+ server_connection_t *conn = NULL;
+ server_conf_t *conf = NULL;
if (!xl || !data) {
gf_log_callingfn ("server", GF_LOG_WARNING,
@@ -589,20 +608,37 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
}
case RPCSVC_EVENT_DISCONNECT:
conn = get_server_conn_state (this, xprt);
- if (conn)
- server_connection_cleanup (this, conn);
-
- gf_log (this->name, GF_LOG_INFO,
- "disconnected connection from %s",
- xprt->peerinfo.identifier);
+ if (!conn)
+ break;
+ put_server_conn_state (this, xprt);
+ gf_log (this->name, GF_LOG_INFO, "disconnecting connection"
+ "from %s", xprt->peerinfo.identifier);
list_del (&xprt->list);
+ pthread_mutex_lock (&conn->lock);
+ {
+ if (conn->timer)
+ goto unlock;
+
+ gf_log (this->name, GF_LOG_INFO, "starting a grace "
+ "timer for %s", xprt->name);
+
+ conn->timer = gf_timer_call_after (this->ctx,
+ conf->grace_tv,
+ grace_time_handler,
+ conn);
+ }
+ unlock:
+ pthread_mutex_unlock (&conn->lock);
+
break;
case RPCSVC_EVENT_TRANSPORT_DESTROY:
- conn = get_server_conn_state (this, xprt);
- if (conn)
- server_connection_put (this, conn);
+ /*- conn obj has been disassociated from xprt on first
+ * disconnect.
+ * conn cleanup and destruction is handed over to
+ * grace_time_handler or the subsequent handler that 'owns'
+ * the conn. Nothing left to be done here. */
break;
default:
break;
@@ -668,6 +704,30 @@ _copy_auth_opt (dict_t *unused,
int
+server_init_grace_timer (xlator_t *this, dict_t *options,
+ server_conf_t *conf)
+{
+ int32_t ret = -1;
+ int32_t grace_timeout = -1;
+
+ GF_VALIDATE_OR_GOTO ("server", this, out);
+ GF_VALIDATE_OR_GOTO (this->name, options, out);
+ GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+ ret = dict_get_int32 (options, "grace-timeout", &grace_timeout);
+ if (!ret)
+ conf->grace_tv.tv_sec = grace_timeout;
+ else
+ conf->grace_tv.tv_sec = 10;
+
+ conf->grace_tv.tv_usec = 0;
+
+ ret = 0;
+out:
+ return ret;
+}
+
+int
reconfigure (xlator_t *this, dict_t *options)
{
@@ -761,6 +821,7 @@ reconfigure (xlator_t *this, dict_t *options)
"Reconfigure not found for transport" );
}
}
+ ret = server_init_grace_timer (this, options, conf);
out:
gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
@@ -797,6 +858,10 @@ init (xlator_t *this)
INIT_LIST_HEAD (&conf->xprt_list);
pthread_mutex_init (&conf->mutex, NULL);
+ ret = server_init_grace_timer (this, this->options, conf);
+ if (ret)
+ goto out;
+
ret = server_build_config (this, conf);
if (ret)
goto out;
@@ -1032,5 +1097,8 @@ struct volume_options options[] = {
.type = GF_OPTION_TYPE_PATH,
.default_value = "/tmp"
},
+ {.key = {"grace-timeout"},
+ .type = GF_OPTION_TYPE_INT,
+ },
{ .key = {NULL} },
};
diff --git a/xlators/protocol/server/src/server.h b/xlators/protocol/server/src/server.h
index 92785c5a9..091a02ccb 100644
--- a/xlators/protocol/server/src/server.h
+++ b/xlators/protocol/server/src/server.h
@@ -28,6 +28,7 @@
#include "protocol-common.h"
#include "server-mem-types.h"
#include "glusterfs3.h"
+#include "timer.h"
#define DEFAULT_BLOCK_SIZE 4194304 /* 4MB */
#define DEFAULT_VOLUME_FILE_PATH CONFDIR "/glusterfs.vol"
@@ -60,8 +61,10 @@ struct _server_connection {
pthread_mutex_t lock;
fdtable_t *fdtable;
struct _lock_table *ltable;
+ gf_timer_t *timer;
xlator_t *bound_xl;
xlator_t *this;
+ uint32_t lk_version;
};
typedef struct _server_connection server_connection_t;
@@ -92,7 +95,7 @@ struct server_conf {
gf_boolean_t trace;
char *conf_dir;
struct _volfile_ctx *volfile;
-
+ struct timeval grace_tv;
dict_t *auth_modules;
pthread_mutex_t mutex;
struct list_head conns;