glusterd: Use GF_ATOMIC to update 'blockers' counter at glusterd_conf

Problem: 
Currently in glusterd code uses sync_lock/sync_unlock to update blockers
counter which could add delays to the overall transaction phase
escpecially when there's a batch of volume stop operations processed by
glusterd in brick multiplexing mode.

Solution: Use GF_ATOMIC to update blocker counter to ensure unnecessary
context switching can be avoided.

Change-Id: Ie13177dfee2af66687ae7cf5c67405c152853990
Fixes: bz#1631128
Signed-off-by: Mohit Agrawal <moagrawal@redhat.com>

4 files changed, 11 insertions, 12 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 0944cc648bf..ce44c4679b1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -6018,9 +6018,12 @@ glusterd_op_stage_validate(glusterd_op_t op, dict_t *dict, char **op_errstr,
 static void
 glusterd_wait_for_blockers(glusterd_conf_t *priv)
 {
-    while (priv->blockers) {
+    uint64_t blockers = GF_ATOMIC_GET(priv->blockers);
+
+    while (blockers) {
         synclock_unlock(&priv->big_lock);
         sleep(1);
+        blockers = GF_ATOMIC_GET(priv->blockers);
         synclock_lock(&priv->big_lock);
     }
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 1214b6cbb19..7d611b11781 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -5426,9 +5426,7 @@ my_callback(struct rpc_req *req, struct iovec *iov, int count, void *v_frame)
     call_frame_t *frame = v_frame;
     glusterd_conf_t *conf = frame->this->private;
 
-    synclock_lock(&conf->big_lock);
-    --(conf->blockers);
-    synclock_unlock(&conf->big_lock);
+    GF_ATOMIC_DEC(conf->blockers);
 
     STACK_DESTROY(frame->root);
     return 0;
@@ -5524,9 +5522,7 @@ attach_brick_callback(struct rpc_req *req, struct iovec *iov, int count,
         }
     }
 out:
-    synclock_lock(&conf->big_lock);
-    --(conf->blockers);
-    synclock_unlock(&conf->big_lock);
+    GF_ATOMIC_DEC(conf->blockers);
     STACK_DESTROY(frame->root);
     return 0;
 }
@@ -5613,7 +5609,7 @@ send_attach_req(xlator_t *this, struct rpc_clnt *rpc, char *path,
         cbkfn = attach_brick_callback;
     }
     /* Send the msg */
-    ++(conf->blockers);
+    GF_ATOMIC_INC(conf->blockers);
     ret = rpc_clnt_submit(rpc, &gd_brick_prog, op, cbkfn, &iov, 1, NULL, 0,
                           iobref, frame, NULL, 0, NULL, 0, NULL);
     return ret;
@@ -6347,7 +6343,7 @@ glusterd_restart_bricks(void *opaque)
     }
     conf->restart_bricks = _gf_true;
 
-    ++(conf->blockers);
+    GF_ATOMIC_INC(conf->blockers);
     ret = glusterd_get_quorum_cluster_counts(this, &active_count,
                                              &quorum_count);
     if (ret)
@@ -6456,7 +6452,7 @@ glusterd_restart_bricks(void *opaque)
     ret = 0;
 
 out:
-    --(conf->blockers);
+    GF_ATOMIC_DEC(conf->blockers);
     conf->restart_done = _gf_true;
     conf->restart_bricks = _gf_false;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 187038fc029..3da579464bb 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1976,7 +1976,7 @@ init(xlator_t *this)
         }
     }
 
-    conf->blockers = 0;
+    GF_ATOMIC_INIT(conf->blockers, 0);
     /* If the peer count is less than 2 then this would be the best time to
      * spawn process/bricks that may need (re)starting since last time
      * (this) glusterd was up. */
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 412ba7415f0..3b681ef9544 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -212,7 +212,7 @@ typedef struct {
     int ping_timeout;
     uint32_t generation;
     int32_t workers;
-    uint32_t blockers;
+    gf_atomic_t blockers;
     uint32_t mgmt_v3_lock_timeout;
     gf_boolean_t restart_bricks;
 } glusterd_conf_t;