protocol/rpc/transport: bring in one more event for 'TRANSPORT-DESTROY'

needed because, a RPC disconnect doesn't mean that a RPC transport/listener
is dead. With this, the race in server protocol cleaning up the lock table /
fd table when some frames are in transit will be handled properly.

Signed-off-by: Amar Tumballi <amar@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>

BUG: 1843 ()
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1843

11 files changed, 47 insertions, 33 deletions

diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index eb2f61ba577..4d8eb22fe0c 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -661,7 +661,7 @@ cleanup_and_exit (int signum)
         glusterfs_mgmt_pmap_signout (ctx);
 
         if (ctx->mgmt)
-                rpc_clnt_destroy (ctx->mgmt);
+                rpc_clnt_unref (ctx->mgmt);
 
         gf_log ("glusterfsd", GF_LOG_NORMAL, "shutting down");
 
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 61bc5263dc7..bd878f4fc1e 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -1428,32 +1428,16 @@ rpc_clnt_ref (struct rpc_clnt *rpc)
         return rpc;
 }
 
-struct rpc_clnt *
-rpc_clnt_unref (struct rpc_clnt *rpc)
-{
-        int     count = 0;
-
-        if (!rpc)
-                return NULL;
-        pthread_mutex_lock (&rpc->lock);
-        {
-                count = --rpc->refcount;
-        }
-        pthread_mutex_unlock (&rpc->lock);
-        if (!count) {
-                rpc_clnt_destroy (rpc);
-                return NULL;
-        }
-        return rpc;
-}
 
-void
+static void
 rpc_clnt_destroy (struct rpc_clnt *rpc)
 {
         if (!rpc)
                 return;
 
-        rpc_transport_destroy (rpc->conn.trans);
+        if (rpc->conn.trans)
+                rpc_transport_unref (rpc->conn.trans);
+
         rpc_clnt_connection_cleanup (&rpc->conn);
         rpc_clnt_reconnect_cleanup (&rpc->conn);
         saved_frames_destroy (rpc->conn.saved_frames);
@@ -1469,6 +1453,25 @@ rpc_clnt_destroy (struct rpc_clnt *rpc)
         return;
 }
 
+struct rpc_clnt *
+rpc_clnt_unref (struct rpc_clnt *rpc)
+{
+        int     count = 0;
+
+        if (!rpc)
+                return NULL;
+        pthread_mutex_lock (&rpc->lock);
+        {
+                count = --rpc->refcount;
+        }
+        pthread_mutex_unlock (&rpc->lock);
+        if (!count) {
+                rpc_clnt_destroy (rpc);
+                return NULL;
+        }
+        return rpc;
+}
+
 
 void
 rpc_clnt_reconfig (struct rpc_clnt *rpc, struct rpc_clnt_config *config)
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index f5fe17f5a47..932497b1387 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -223,8 +223,6 @@ rpc_clnt_ref (struct rpc_clnt *rpc);
 struct rpc_clnt *
 rpc_clnt_unref (struct rpc_clnt *rpc);
 
-void rpc_clnt_destroy (struct rpc_clnt *rpc);
-
 void rpc_clnt_set_connected (rpc_clnt_connection_t *conn);
 
 void rpc_clnt_unset_connected (rpc_clnt_connection_t *conn);
diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c
index c69237d58c4..43b2a0c18cf 100644
--- a/rpc/rpc-lib/src/rpc-transport.c
+++ b/rpc/rpc-lib/src/rpc-transport.c
@@ -1110,8 +1110,7 @@ rpc_transport_unref (rpc_transport_t *this)
 	pthread_mutex_unlock (&this->lock);
 
 	if (refcount == 0) {
-		/* xlator_notify (this->xl, GF_EVENT_RPC_TRANSPORT_CLEANUP,
-                   this); */
+		this->notify (this, this->mydata, RPC_TRANSPORT_CLEANUP, NULL);
 		rpc_transport_destroy (this);
 	}
 
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index 2ba46fba9f7..478de9ef1c4 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -248,9 +248,6 @@ int32_t
 rpc_transport_submit_reply (rpc_transport_t *this,
                             rpc_transport_reply_t *reply);
 
-int32_t
-rpc_transport_destroy (rpc_transport_t *this);
-
 rpc_transport_t *
 rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *name);
 
diff --git a/rpc/rpc-lib/src/rpcsvc-common.h b/rpc/rpc-lib/src/rpcsvc-common.h
index 7e72bc3ae44..442049433a7 100644
--- a/rpc/rpc-lib/src/rpcsvc-common.h
+++ b/rpc/rpc-lib/src/rpcsvc-common.h
@@ -29,6 +29,7 @@
 typedef enum {
         RPCSVC_EVENT_ACCEPT,
         RPCSVC_EVENT_DISCONNECT,
+        RPCSVC_EVENT_TRANSPORT_DESTROY,
         RPCSVC_EVENT_LISTENER_DEAD,
 } rpcsvc_event_t;
 
diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index b78b1f0e2c9..e14313fe74f 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -1066,6 +1066,7 @@ rpcsvc_notify (rpc_transport_t *trans, void *mydata,
         rpc_transport_pollin_t *msg       = NULL;
         rpc_transport_t        *new_trans = NULL;
         rpcsvc_t               *svc       = NULL;
+        rpcsvc_listener_t      *listener  = NULL;
 
         svc = mydata;
         if (svc == NULL) {
@@ -1101,7 +1102,13 @@ rpcsvc_notify (rpc_transport_t *trans, void *mydata,
                 break;
 
         case RPC_TRANSPORT_CLEANUP:
-                /* FIXME: think about this later */
+                listener = rpcsvc_get_listener (svc, -1, trans);
+                if (listener == NULL) {
+                        goto out;
+                }
+
+                rpcsvc_program_notify (listener, RPCSVC_EVENT_TRANSPORT_DESTROY,
+                                       trans);
                 ret = 0;
                 break;
 
diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index ae4bd56a821..40531ad4c2a 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -1736,7 +1736,7 @@ socket_event_handler (int fd, int idx, void *data,
         if ((ret < 0) || poll_err) {
                 gf_log ("transport", GF_LOG_TRACE, "disconnecting now");
                 socket_event_poll_err (this);
-                //rpc_transport_unref (this);
+                rpc_transport_unref (this);
         }
 
 out:
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 50cb917aacc..1cc96532b04 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -2008,7 +2008,7 @@ fini (xlator_t *this)
 
         if (conf) {
                 if (conf->rpc)
-                       rpc_clnt_destroy (conf->rpc);
+                       rpc_clnt_unref (conf->rpc);
 
                 /* Saved Fds */
                 /* TODO: */
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 5ea6f561908..69fb5f6a7d6 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -102,6 +102,10 @@ free_state (server_state_t *state)
                 state->conn = NULL;
         }
 
+        if (state->xprt) {
+                rpc_transport_unref (state->xprt);
+                state->xprt = NULL;
+        }
         if (state->fd) {
                 fd_unref (state->fd);
                 state->fd = NULL;
@@ -776,7 +780,7 @@ server_alloc_frame (rpcsvc_request_t *req)
         if (conn->bound_xl)
                 state->itable = conn->bound_xl->itable;
 
-        state->xprt  = req->trans;
+        state->xprt  = rpc_transport_ref (req->trans);
         state->conn  = conn;
 
         state->resolve.fd_no = -1;
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index 2851f8cea86..4fcce6e99d2 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -424,11 +424,16 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
         case RPCSVC_EVENT_DISCONNECT:
                 conn = get_server_conn_state (this, xprt);
                 if (conn)
-                        server_connection_put (this, conn);
+                        server_connection_cleanup (this, conn);
 
                 list_del (&xprt->list);
 
                 break;
+        case RPCSVC_EVENT_TRANSPORT_DESTROY:
+                conn = get_server_conn_state (this, xprt);
+                if (conn)
+                        server_connection_put (this, conn);
+                break;
         default:
                 break;
         }