rpc,glusterd: Use rpc_clnt notifyfn to cleanup mydata

        Backport of http://review.gluster.org/5512

rpc:
- On a RPC_TRANSPORT_CLEANUP event, rpc_clnt_notify calls the registered
  notifyfn with a RPC_CLNT_DESTROY event. The notifyfn should properly
  cleanup the saved mydata on this event.
- Break the reconnect chain when an rpc client is disabled. This will
  prevent new disconnect events which can lead to crashes.

glusterd:
- Added support for RPC_CLNT_DESTROY in glusterd_brick_rpc_notify
- Use a common glusterd_rpc_clnt_unref() function throught glusterd in
  place of rpc_clnt_unref(). This function correctly gives up the
  big-lock before performing the unref.

Change-Id: I93230441c5089039643fc9f5632477ef1b695348
BUG: 962619
Signed-off-by: Kaushal M <kaushal@redhat.com>
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/6566
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

4 files changed, 27 insertions, 8 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 3aafa122b6a..0407741bb7b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -3713,10 +3713,12 @@ __glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
                                 "%s:%s", brickinfo->hostname, brickinfo->path);
 
                 glusterd_set_brick_status (brickinfo, GF_BRICK_STOPPED);
-                if (rpc_clnt_is_disabled (rpc))
-                        GF_FREE (brickid);
                 break;
 
+        case RPC_CLNT_DESTROY:
+                GF_FREE (mydata);
+                mydata = NULL;
+                break;
         default:
                 gf_log (this->name, GF_LOG_TRACE,
                         "got some other RPC event %d", event);
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 7911c3d210d..da29aaf0218 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -152,7 +152,7 @@ __glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
                 glusterd_store_perform_node_state_store (volinfo);
 
                 if (defrag->rpc) {
-                        rpc_clnt_unref (defrag->rpc);
+                        glusterd_rpc_clnt_unref (priv, defrag->rpc);
                         defrag->rpc = NULL;
                 }
                 if (defrag->cbk_fn)
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 03ef2ef3b18..fb1fed3a4a2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1054,7 +1054,7 @@ glusterd_friend_cleanup (glusterd_peerinfo_t *peerinfo)
 
                 peerctx = peerinfo->rpc->mydata;
                 peerinfo->rpc->mydata = NULL;
-                peerinfo->rpc = rpc_clnt_unref (peerinfo->rpc);
+                peerinfo->rpc = glusterd_rpc_clnt_unref (priv, peerinfo->rpc);
                 peerinfo->rpc = NULL;
                 if (peerctx) {
                         GF_FREE (peerctx->errstr);
@@ -1466,9 +1466,7 @@ glusterd_brick_disconnect (glusterd_brickinfo_t *brickinfo)
         brickinfo->rpc = NULL;
 
         if (rpc) {
-                synclock_unlock (&priv->big_lock);
-                rpc_clnt_unref (rpc);
-                synclock_lock (&priv->big_lock);
+                glusterd_rpc_clnt_unref (priv, rpc);
         }
 
         return 0;
@@ -3894,12 +3892,13 @@ int32_t
 glusterd_nodesvc_disconnect (char *server)
 {
         struct rpc_clnt         *rpc = NULL;
+        glusterd_conf_t         *priv = THIS->private;
 
         rpc = glusterd_nodesvc_get_rpc (server);
         (void)glusterd_nodesvc_set_rpc (server, NULL);
 
         if (rpc)
-                rpc_clnt_unref (rpc);
+                glusterd_rpc_clnt_unref (priv, rpc);
 
         return 0;
 }
@@ -9331,3 +9330,18 @@ out:
 
         return ret;
 }
+
+rpc_clnt_t *
+glusterd_rpc_clnt_unref (glusterd_conf_t *conf, rpc_clnt_t *rpc)
+{
+        rpc_clnt_t *ret = NULL;
+
+        GF_ASSERT (conf);
+        GF_ASSERT (rpc);
+        synclock_unlock (&conf->big_lock);
+        ret = rpc_clnt_unref (rpc);
+        synclock_lock (&conf->big_lock);
+
+        return ret;
+}
+
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 970b1f8a638..fd3f5772922 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -624,4 +624,7 @@ glusterd_status_has_tasks (int cmd);
 
 int
 gd_stop_rebalance_process (glusterd_volinfo_t *volinfo);
+
+rpc_clnt_t *
+glusterd_rpc_clnt_unref (glusterd_conf_t *conf, rpc_clnt_t *rpc);
 #endif