summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPranith Kumar K <pranithk@gluster.com>2012-03-10 00:54:12 +0530
committerVijay Bellur <vijay@gluster.com>2012-03-13 04:58:45 -0700
commitfa5b0347193f8d1a4b917a2edb338423cb175e66 (patch)
tree4f24b9cc0bae934c71e4684ef1f1104322f3bcec
parent4a37d78da3fef69f0074cab3ff71182a68876358 (diff)
protocol/server: Remove connection from conf->conns w.o. race
1) Adding the connection to conf->conns used to happen in conf->mutex, but removing happened under conn->lock. Fixed that as below. When the connection object is created conn's ref, bind_ref count is set to '1'. For bind_ref ref/unref happens under conf->mutex whenever server_connection_get, put is called. When bind_ref goes to '0' connection object is removed from conf->conns under conf->mutex. After it is removed from the list, conn_unref is called outside the conf->mutex. conn_ref/unref still happens under conn->lock. 2) Fixed races in server_connection_cleaup in grace_timer_handler and server_setvolume. Change-Id: Ie7b63b10f658af909a11c3327066667f5b7bd114 BUG: 801675 Signed-off-by: Pranith Kumar K <pranithk@gluster.com> Reviewed-on: http://review.gluster.com/2911 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Amar Tumballi <amarts@redhat.com> Reviewed-by: Vijay Bellur <vijay@gluster.com>
-rw-r--r--xlators/protocol/server/src/server-handshake.c6
-rw-r--r--xlators/protocol/server/src/server-helpers.c35
-rw-r--r--xlators/protocol/server/src/server.c9
-rw-r--r--xlators/protocol/server/src/server.h5
4 files changed, 47 insertions, 8 deletions
diff --git a/xlators/protocol/server/src/server-handshake.c b/xlators/protocol/server/src/server-handshake.c
index d1e3659ab26..ba80185fef6 100644
--- a/xlators/protocol/server/src/server-handshake.c
+++ b/xlators/protocol/server/src/server-handshake.c
@@ -432,8 +432,8 @@ server_setvolume (rpcsvc_request_t *req)
}
cancelled = server_cancel_conn_timer (this, conn);
- if (cancelled)
- server_conn_unref (conn);
+ if (cancelled)//Do connection_put on behalf of grace-timer-handler.
+ server_connection_put (this, conn, NULL);
if (conn->lk_version != 0 &&
conn->lk_version != lk_version) {
(void) server_connection_cleanup (this, conn);
@@ -723,7 +723,7 @@ server_set_lk_version (rpcsvc_request_t *req)
conn = server_connection_get (this, args.uid);
conn->lk_version = args.lk_ver;
- server_conn_unref (conn);
+ server_connection_put (this, conn, NULL);
rsp.lk_ver = args.lk_ver;
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 28d59aa44a9..db029b2b573 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -648,7 +648,6 @@ server_conn_unref (server_connection_t *conn)
conn->ref--;
if (!conn->ref) {
- list_del_init (&conn->list);
todel = conn;
}
}
@@ -691,6 +690,7 @@ server_connection_get (xlator_t *this, const char *id)
list_for_each_entry (trav, &conf->conns, list) {
if (!strncmp (trav->id, id, strlen (id))) {
conn = trav;
+ conn->bind_ref++;
goto unlock;
}
}
@@ -706,6 +706,8 @@ server_connection_get (xlator_t *this, const char *id)
conn->fdtable = gf_fd_fdtable_alloc ();
conn->ltable = gf_lock_table_new ();
conn->this = this;
+ conn->bind_ref = 1;
+ conn->ref = 1;//when bind_ref becomes 0 it calls conn_unref
pthread_mutex_init (&conn->lock, NULL);
list_add (&conn->list, &conf->conns);
@@ -713,11 +715,38 @@ server_connection_get (xlator_t *this, const char *id)
unlock:
pthread_mutex_unlock (&conf->mutex);
out:
- if (conn)
- server_conn_ref (conn);
return conn;
}
+server_connection_t*
+server_connection_put (xlator_t *this, server_connection_t *conn,
+ gf_boolean_t *detached)
+{
+ server_conf_t *conf = NULL;
+ gf_boolean_t unref = _gf_false;
+
+ if (detached)
+ *detached = _gf_false;
+ conf = this->private;
+ pthread_mutex_lock (&conf->mutex);
+ {
+ conn->bind_ref--;
+ if (!conn->bind_ref) {
+ list_del_init (&conn->list);
+ unref = _gf_true;
+ }
+ }
+ pthread_mutex_unlock (&conf->mutex);
+ if (unref) {
+ gf_log (this->name, GF_LOG_INFO, "Shutting down connection %s",
+ conn->id);
+ if (detached)
+ *detached = _gf_true;
+ server_conn_unref (conn);
+ conn = NULL;
+ }
+ return conn;
+}
static call_frame_t *
server_alloc_frame (rpcsvc_request_t *req)
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index 7ac8590a42e..b75ad172e6f 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -42,6 +42,7 @@ grace_time_handler (void *data)
server_connection_t *conn = NULL;
xlator_t *this = NULL;
gf_boolean_t cancelled = _gf_false;
+ gf_boolean_t detached = _gf_false;
conn = data;
this = conn->this;
@@ -49,11 +50,15 @@ grace_time_handler (void *data)
GF_VALIDATE_OR_GOTO (THIS->name, conn, out);
GF_VALIDATE_OR_GOTO (THIS->name, this, out);
- gf_log (this->name, GF_LOG_INFO, "grace timer expired");
+ gf_log (this->name, GF_LOG_INFO, "grace timer expired for %s", conn->id);
cancelled = server_cancel_conn_timer (this, conn);
if (cancelled) {
- server_connection_cleanup (this, conn);
+ //conn should not be destroyed in conn_put, so take a ref.
+ server_conn_ref (conn);
+ server_connection_put (this, conn, &detached);
+ if (detached)//reconnection did not happen :-(
+ server_connection_cleanup (this, conn);
server_conn_unref (conn);
}
out:
diff --git a/xlators/protocol/server/src/server.h b/xlators/protocol/server/src/server.h
index c54454e3efe..6024d100b30 100644
--- a/xlators/protocol/server/src/server.h
+++ b/xlators/protocol/server/src/server.h
@@ -60,6 +60,7 @@ struct _server_connection {
struct list_head list;
char *id;
int ref;
+ int bind_ref;
pthread_mutex_t lock;
fdtable_t *fdtable;
struct _lock_table *ltable;
@@ -75,6 +76,10 @@ typedef struct _server_connection server_connection_t;
server_connection_t *
server_connection_get (xlator_t *this, const char *id);
+server_connection_t *
+server_connection_put (xlator_t *this, server_connection_t *conn,
+ gf_boolean_t *detached);
+
server_connection_t*
server_conn_unref (server_connection_t *conn);