rpc: Cleanup rpc object in TRANSPORT_CLEANUP event

rpc_transport object should be alive as long as the rpc_clnt object is
alive. To ensure this, on rpc_clnt's last unref, we cleanup the
corresponding rpc_transport object and complete the rpc_clnt cleanup
later, in a bottom-up fashion.

Introduced rpc_clnt_is_disabled, to allow higher layers to differentiate
between the 'final'[1] disconnect triggered from upper layers, and a
normal disconnect. This differentiation helps in cleaning up resources,
at higher layers, in a race-free manner.

[1] - 'final' here means that the rpc and the associated connection, is
not going to be used anymore. eg - glusterd_brick_disconnect on
volume-stop.

Change-Id: I2ecf891a36e3b02cd9eacca964e659525d1bbc6e
BUG: 962619
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/5107
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

1 files changed, 38 insertions, 11 deletions

diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index c8bb8e29855..b1d004aa869 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -819,6 +819,9 @@ out:
         return;
 }
 
+static void
+rpc_clnt_destroy (struct rpc_clnt *rpc);
+
 int
 rpc_clnt_notify (rpc_transport_t *trans, void *mydata,
                  rpc_transport_event_t event, void *data, ...)
@@ -864,9 +867,7 @@ rpc_clnt_notify (rpc_transport_t *trans, void *mydata,
         }
 
         case RPC_TRANSPORT_CLEANUP:
-                /* this event should not be received on a client for, a
-                 * transport is only disconnected, but never destroyed.
-                 */
+                rpc_clnt_destroy (clnt);
                 ret = 0;
                 break;
 
@@ -1541,18 +1542,21 @@ rpc_clnt_ref (struct rpc_clnt *rpc)
 
 
 static void
-rpc_clnt_destroy (struct rpc_clnt *rpc)
+rpc_clnt_trigger_destroy (struct rpc_clnt *rpc)
 {
         if (!rpc)
                 return;
 
-        if (rpc->conn.trans) {
-                rpc_transport_unregister_notify (rpc->conn.trans);
-                rpc_transport_disconnect (rpc->conn.trans);
-                rpc_transport_unref (rpc->conn.trans);
-        }
+        rpc_clnt_disable (rpc);
+        rpc_transport_unref (rpc->conn.trans);
+}
+
+static void
+rpc_clnt_destroy (struct rpc_clnt *rpc)
+{
+        if (!rpc)
+                return;
 
-        rpc_clnt_reconnect_cleanup (&rpc->conn);
         saved_frames_destroy (rpc->conn.saved_frames);
         pthread_mutex_destroy (&rpc->lock);
         pthread_mutex_destroy (&rpc->conn.lock);
@@ -1579,13 +1583,36 @@ rpc_clnt_unref (struct rpc_clnt *rpc)
         }
         pthread_mutex_unlock (&rpc->lock);
         if (!count) {
-                rpc_clnt_destroy (rpc);
+                rpc_clnt_trigger_destroy (rpc);
                 return NULL;
         }
         return rpc;
 }
 
 
+char
+rpc_clnt_is_disabled (struct rpc_clnt *rpc)
+{
+
+        rpc_clnt_connection_t *conn = NULL;
+        char                   disabled = 0;
+
+        if (!rpc) {
+                goto out;
+        }
+
+        conn = &rpc->conn;
+
+        pthread_mutex_lock (&conn->lock);
+        {
+                disabled = rpc->disabled;
+        }
+        pthread_mutex_unlock (&conn->lock);
+
+out:
+        return disabled;
+}
+
 void
 rpc_clnt_disable (struct rpc_clnt *rpc)
 {