protocol/server: Remove connection from conf->conns w.o. race

1) Adding the connection to conf->conns used to
happen in conf->mutex, but removing happened under conn->lock.
Fixed that as below.
When the connection object is created conn's ref, bind_ref count
is set to '1'. For bind_ref ref/unref happens under conf->mutex
whenever server_connection_get, put is called.
When bind_ref goes to '0' connection object is removed from
conf->conns under conf->mutex.  After it is removed from the list,
conn_unref is called outside the conf->mutex.
conn_ref/unref still happens under conn->lock.

2) Fixed races in server_connection_cleaup in grace_timer_handler
and server_setvolume.

Change-Id: Ie7b63b10f658af909a11c3327066667f5b7bd114
BUG: 801675
Signed-off-by: Pranith Kumar K <pranithk@gluster.com>
Reviewed-on: http://review.gluster.com/2911
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

1 files changed, 32 insertions, 3 deletions

diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 28d59aa44a9..db029b2b573 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -648,7 +648,6 @@ server_conn_unref (server_connection_t *conn)
                 conn->ref--;
 
                 if (!conn->ref) {
-                        list_del_init (&conn->list);
                         todel = conn;
                 }
         }
@@ -691,6 +690,7 @@ server_connection_get (xlator_t *this, const char *id)
                 list_for_each_entry (trav, &conf->conns, list) {
                         if (!strncmp (trav->id, id, strlen (id))) {
                                 conn = trav;
+                                conn->bind_ref++;
                                 goto unlock;
                         }
                 }
@@ -706,6 +706,8 @@ server_connection_get (xlator_t *this, const char *id)
                 conn->fdtable = gf_fd_fdtable_alloc ();
                 conn->ltable  = gf_lock_table_new ();
                 conn->this    = this;
+                conn->bind_ref = 1;
+                conn->ref     = 1;//when bind_ref becomes 0 it calls conn_unref
                 pthread_mutex_init (&conn->lock, NULL);
                 list_add (&conn->list, &conf->conns);
 
@@ -713,11 +715,38 @@ server_connection_get (xlator_t *this, const char *id)
 unlock:
         pthread_mutex_unlock (&conf->mutex);
 out:
-        if (conn)
-                server_conn_ref (conn);
         return conn;
 }
 
+server_connection_t*
+server_connection_put (xlator_t *this, server_connection_t *conn,
+                       gf_boolean_t *detached)
+{
+        server_conf_t       *conf = NULL;
+        gf_boolean_t        unref = _gf_false;
+
+        if (detached)
+                *detached = _gf_false;
+        conf = this->private;
+        pthread_mutex_lock (&conf->mutex);
+        {
+                conn->bind_ref--;
+                if (!conn->bind_ref) {
+                        list_del_init (&conn->list);
+                        unref = _gf_true;
+                }
+        }
+        pthread_mutex_unlock (&conf->mutex);
+        if (unref) {
+                gf_log (this->name, GF_LOG_INFO, "Shutting down connection %s",
+                        conn->id);
+                if (detached)
+                        *detached = _gf_true;
+                server_conn_unref (conn);
+                conn = NULL;
+        }
+        return conn;
+}
 
 static call_frame_t *
 server_alloc_frame (rpcsvc_request_t *req)