glusterd: fix double-free of rebalance process' rpc object

Change-Id: I0c79c4de47a160b1ecf3a8994eedc02e3f5002a9
BUG: 1223338
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/10872
Tested-by: NetBSD Build System
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Reviewed-by: Kaushal M <kaushal@redhat.com>

6 files changed, 72 insertions, 9 deletions

diff --git a/run-tests.sh b/run-tests.sh
index 650eb26e4cd..eb609876829 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -204,7 +204,6 @@ function is_bad_test ()
               ./tests/bugs/quota/bug-1035576.t  \
               ./tests/basic/uss.t       \
               ./tests/bugs/glusterfs/bug-867253.t       \
-              ./tests/bugs/glusterd/bug-974007.t        \
               ./tests/basic/volume-snapshot-clone.t     \
               ./tests/basic/quota-anon-fd-nfs.t \
               ./tests/bugs/snapshot/bug-1112559.t       \
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 9111c07b8fc..f126e8f0b1f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -149,10 +149,7 @@ __glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
 
                 glusterd_store_perform_node_state_store (volinfo);
 
-                if (defrag->rpc) {
-                        glusterd_rpc_clnt_unref (priv, defrag->rpc);
-                        defrag->rpc = NULL;
-                }
+                glusterd_defrag_rpc_put (defrag);
                 if (defrag->cbk_fn)
                         defrag->cbk_fn (volinfo,
                                         volinfo->rebal.defrag_status);
@@ -339,8 +336,9 @@ glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
                 goto out;
 
         //rpc obj for rebalance process already in place.
-        if (defrag->rpc) {
+        if (glusterd_defrag_rpc_get (defrag)) {
                 ret = 0;
+                glusterd_defrag_rpc_put (defrag);
                 goto out;
         }
         GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo);
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index 59b0e5f072e..231c2f720ba 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -2173,6 +2173,8 @@ glusterd_brick_op (call_frame_t *frame, xlator_t *this,
                 }
                 if (!ret)
                         pending_bricks++;
+
+                glusterd_pending_node_put_rpc (pending_node);
         }
 
         gf_log (this->name, GF_LOG_DEBUG, "Sent brick op req for operation "
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index 43539643c0b..6f36478aeca 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -1658,10 +1658,15 @@ gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
                         goto out;
 
                 brick_count++;
+                glusterd_pending_node_put_rpc (pending_node);
         }
 
+        pending_node = NULL;
         ret = 0;
 out:
+        if (pending_node)
+                glusterd_pending_node_put_rpc (pending_node);
+
         if (rsp_dict)
                 dict_unref (rsp_dict);
         gf_log (this->name, GF_LOG_DEBUG, "Sent op req to %d bricks",
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 593e32e95d0..b19813cbefb 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -3523,8 +3523,9 @@ gd_check_and_update_rebalance_info (glusterd_volinfo_t *old_volinfo,
         new = &(new_volinfo->rebal);
 
         //Disconnect from rebalance process
-        if (old->defrag && old->defrag->rpc) {
+        if (glusterd_defrag_rpc_get (old->defrag)) {
                 rpc_transport_disconnect (old->defrag->rpc->conn.trans);
+                glusterd_defrag_rpc_put (old->defrag);
         }
 
         if (!gf_uuid_is_null (old->rebalance_id) &&
@@ -3806,6 +3807,39 @@ out:
 }
 
 struct rpc_clnt*
+glusterd_defrag_rpc_get (glusterd_defrag_info_t *defrag)
+{
+        struct rpc_clnt *rpc = NULL;
+
+        if (!defrag)
+                return NULL;
+
+        LOCK (&defrag->lock);
+        {
+                rpc = rpc_clnt_ref (defrag->rpc);
+        }
+        UNLOCK (&defrag->lock);
+        return rpc;
+}
+
+struct rpc_clnt*
+glusterd_defrag_rpc_put (glusterd_defrag_info_t *defrag)
+{
+        struct rpc_clnt *rpc = NULL;
+
+        if (!defrag)
+                return NULL;
+
+        LOCK (&defrag->lock);
+        {
+                rpc = rpc_clnt_unref (defrag->rpc);
+                defrag->rpc = rpc;
+        }
+        UNLOCK (&defrag->lock);
+        return rpc;
+}
+
+struct rpc_clnt*
 glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node)
 {
         struct rpc_clnt *rpc = NULL;
@@ -3827,8 +3861,8 @@ glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node)
                 rpc = svc->conn.rpc;
         } else if (pending_node->type == GD_NODE_REBALANCE) {
                 volinfo = pending_node->node;
-                if (volinfo->rebal.defrag)
-                        rpc = volinfo->rebal.defrag->rpc;
+                rpc = glusterd_defrag_rpc_get (volinfo->rebal.defrag);
+
         } else if (pending_node->type == GD_NODE_SNAPD) {
                 volinfo = pending_node->node;
                 rpc = volinfo->snapd.svc.conn.rpc;
@@ -3840,6 +3874,23 @@ out:
         return rpc;
 }
 
+void
+glusterd_pending_node_put_rpc (glusterd_pending_node_t *pending_node)
+{
+        glusterd_volinfo_t      *volinfo = NULL;
+
+        switch (pending_node->type) {
+        case GD_NODE_REBALANCE:
+                volinfo = pending_node->node;
+                glusterd_defrag_rpc_put (volinfo->rebal.defrag);
+                break;
+
+        default:
+                break;
+        }
+
+}
+
 int32_t
 glusterd_unlink_file (char *sockfpath)
 {
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 8975e53f91b..3906449e902 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -180,6 +180,9 @@ glusterd_set_socket_filepath (char *sock_filepath, char *sockpath, size_t len);
 struct rpc_clnt*
 glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node);
 
+void
+glusterd_pending_node_put_rpc (glusterd_pending_node_t *pending_node);
+
 int
 glusterd_remote_hostname_get (rpcsvc_request_t *req,
                               char *remote_host, int len);
@@ -654,4 +657,9 @@ int
 glusterd_disallow_op_for_tier (glusterd_volinfo_t *volinfo, glusterd_op_t op,
                                int cmd);
 
+struct rpc_clnt*
+glusterd_defrag_rpc_get (glusterd_defrag_info_t *defrag);
+
+struct rpc_clnt*
+glusterd_defrag_rpc_put (glusterd_defrag_info_t *defrag);
 #endif