glusterd: Refresh glusterd-syncop fixes from master

Following commits were cherry-picked from master,
044f8ce syncop: Remove task from synclock's waitq before 'wake'
cb6aeed glusterd: Give up big lock before performing any RPC
46572fe Revert "glusterd: Fix spurious wakeups in glusterd syncops"
5021e04 synctask: implement barriers around yield, not the other way
4843937 glusterd: Syncop callbks should take big lock too

Change-Id: I5ae71ab98f9a336dc9bbf0e7b2ec50a6ed42b0f5
BUG: 948686
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/4938
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>
Reviewed-on: http://review.gluster.org/5021
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

6 files changed, 252 insertions, 137 deletions

diff --git a/libglusterfs/src/syncop.c b/libglusterfs/src/syncop.c
index 87f39841739..dbc52259fe7 100644
--- a/libglusterfs/src/syncop.c
+++ b/libglusterfs/src/syncop.c
@@ -80,24 +80,15 @@ __wait (struct synctask *task)
 
 
 void
-synctask_waitfor (struct synctask *task, int waitfor)
+synctask_yield (struct synctask *task)
 {
-	struct syncenv *env = NULL;
         xlator_t *oldTHIS = THIS;
 
-	env = task->env;
-
 #if defined(__NetBSD__) && defined(_UC_TLSBASE)
 	/* Preserve pthread private pointer through swapcontex() */
 	task->proc->sched.uc_flags &= ~_UC_TLSBASE;
 #endif
 
-	pthread_mutex_lock (&env->mutex);
-	{
-		task->waitfor = waitfor;
-	}
-	pthread_mutex_unlock (&env->mutex);
-
         if (swapcontext (&task->ctx, &task->proc->sched) < 0) {
                 gf_log ("syncop", GF_LOG_ERROR,
                         "swapcontext failed (%s)", strerror (errno));
@@ -108,13 +99,6 @@ synctask_waitfor (struct synctask *task, int waitfor)
 
 
 void
-synctask_yield (struct synctask *task)
-{
-	synctask_waitfor (task, 1);
-}
-
-
-void
 synctask_yawn (struct synctask *task)
 {
 	struct syncenv *env = NULL;
@@ -124,7 +108,6 @@ synctask_yawn (struct synctask *task)
 	pthread_mutex_lock (&env->mutex);
 	{
 		task->woken = 0;
-		task->waitfor = 0;
 	}
 	pthread_mutex_unlock (&env->mutex);
 }
@@ -139,9 +122,9 @@ synctask_wake (struct synctask *task)
 
         pthread_mutex_lock (&env->mutex);
         {
-                task->woken++;
+                task->woken = 1;
 
-                if (task->slept && task->woken >= task->waitfor)
+                if (task->slept)
                         __run (task);
 
 		pthread_cond_broadcast (&env->cond);
@@ -352,7 +335,6 @@ syncenv_task (struct syncproc *proc)
 
                 task->woken = 0;
                 task->slept = 0;
-                task->waitfor = 0;
 
                 task->proc = proc;
         }
@@ -390,7 +372,7 @@ synctask_switchto (struct synctask *task)
 
         pthread_mutex_lock (&env->mutex);
         {
-                if (task->woken >= task->waitfor) {
+                if (task->woken) {
                         __run (task);
                 } else {
                         task->slept = 1;
@@ -547,12 +529,11 @@ __synclock_lock (struct synclock *lock)
 		if (task) {
 			/* called within a synctask */
 			list_add_tail (&task->waitq, &lock->waitq);
-			{
-				pthread_mutex_unlock (&lock->guard);
-				synctask_yield (task);
-				pthread_mutex_lock (&lock->guard);
-			}
-			list_del_init (&task->waitq);
+                        pthread_mutex_unlock (&lock->guard);
+                        synctask_yield (task);
+                        /* task is removed from waitq in unlock,
+                         * under lock->guard.*/
+                        pthread_mutex_lock (&lock->guard);
 		} else {
 			/* called by a non-synctask */
 			pthread_cond_wait (&lock->cond, &lock->guard);
@@ -634,6 +615,7 @@ __synclock_unlock (synclock_t *lock)
 	pthread_cond_signal (&lock->cond);
 	if (!list_empty (&lock->waitq)) {
 		task = list_entry (lock->waitq.next, struct synctask, waitq);
+                list_del_init (&task->waitq);
 		synctask_wake (task);
 	}
 
@@ -655,6 +637,122 @@ synclock_unlock (synclock_t *lock)
 	return ret;
 }
 
+/* Barriers */
+
+int
+syncbarrier_init (struct syncbarrier *barrier)
+{
+	if (!barrier) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	pthread_cond_init (&barrier->cond, 0);
+	barrier->count = 0;
+	INIT_LIST_HEAD (&barrier->waitq);
+
+	return pthread_mutex_init (&barrier->guard, 0);
+}
+
+
+int
+syncbarrier_destroy (struct syncbarrier *barrier)
+{
+	if (!barrier) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	pthread_cond_destroy (&barrier->cond);
+	return pthread_mutex_destroy (&barrier->guard);
+}
+
+
+static int
+__syncbarrier_wait (struct syncbarrier *barrier, int waitfor)
+{
+	struct synctask *task = NULL;
+
+	if (!barrier) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	task = synctask_get ();
+
+	while (barrier->count < waitfor) {
+		if (task) {
+			/* called within a synctask */
+			list_add_tail (&task->waitq, &barrier->waitq);
+			{
+				pthread_mutex_unlock (&barrier->guard);
+				synctask_yield (task);
+				pthread_mutex_lock (&barrier->guard);
+			}
+			list_del_init (&task->waitq);
+		} else {
+			/* called by a non-synctask */
+			pthread_cond_wait (&barrier->cond, &barrier->guard);
+		}
+	}
+
+	barrier->count = 0;
+
+	return 0;
+}
+
+
+int
+syncbarrier_wait (struct syncbarrier *barrier, int waitfor)
+{
+	int ret = 0;
+
+	pthread_mutex_lock (&barrier->guard);
+	{
+		ret = __syncbarrier_wait (barrier, waitfor);
+	}
+	pthread_mutex_unlock (&barrier->guard);
+
+	return ret;
+}
+
+
+static int
+__syncbarrier_wake (struct syncbarrier *barrier)
+{
+	struct synctask *task = NULL;
+
+	if (!barrier) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	barrier->count++;
+
+	pthread_cond_signal (&barrier->cond);
+	if (!list_empty (&barrier->waitq)) {
+		task = list_entry (barrier->waitq.next, struct synctask, waitq);
+		synctask_wake (task);
+	}
+
+	return 0;
+}
+
+
+int
+syncbarrier_wake (struct syncbarrier *barrier)
+{
+	int ret = 0;
+
+	pthread_mutex_lock (&barrier->guard);
+	{
+		ret = __syncbarrier_wake (barrier);
+	}
+	pthread_mutex_unlock (&barrier->guard);
+
+	return ret;
+}
+
 
 /* FOPS */
 
diff --git a/libglusterfs/src/syncop.h b/libglusterfs/src/syncop.h
index b1a7229b3ee..f6eb423a4b6 100644
--- a/libglusterfs/src/syncop.h
+++ b/libglusterfs/src/syncop.h
@@ -57,7 +57,6 @@ struct synctask {
         void               *stack;
         int                 woken;
         int                 slept;
-        int                 waitfor;
         int                 ret;
 
         uid_t               uid;
@@ -107,6 +106,16 @@ struct synclock {
 };
 typedef struct synclock synclock_t;
 
+
+struct syncbarrier {
+	pthread_mutex_t     guard; /* guard the remaining members, pair @cond */
+	pthread_cond_t      cond;  /* waiting non-synctasks */
+	struct list_head    waitq; /* waiting synctasks */
+	int                 count; /* count the number of wakes */
+};
+typedef struct syncbarrier syncbarrier_t;
+
+
 struct syncargs {
         int                 op_ret;
         int                 op_errno;
@@ -127,11 +136,13 @@ struct syncargs {
         dict_t             *dict;
         pthread_mutex_t     lock_dict;
 
+	syncbarrier_t       barrier;
+
         /* do not touch */
         struct synctask    *task;
         pthread_mutex_t     mutex;
         pthread_cond_t      cond;
-        int                 wakecnt;
+	int                 done;
 };
 
 
@@ -142,7 +153,7 @@ struct syncargs {
         } else {                                                \
                 pthread_mutex_init (&args->mutex, NULL);        \
                 pthread_cond_init (&args->cond, NULL);          \
-                args->wakecnt = 0;                              \
+                args->done = 0;					\
         }                                                       \
         } while (0)
 
@@ -153,7 +164,7 @@ struct syncargs {
         } else {                                                \
                 pthread_mutex_lock (&args->mutex);              \
                 {                                               \
-                        args->wakecnt++;                        \
+                        args->done = 1;				\
                         pthread_cond_signal (&args->cond);      \
                 }                                               \
                 pthread_mutex_unlock (&args->mutex);            \
@@ -161,24 +172,21 @@ struct syncargs {
         } while (0)
 
 
-#define __waitfor(args, cnt) do {                                       \
-        if (args->task) {                                               \
-                synctask_waitfor (args->task, cnt);                     \
-        } else {                                                        \
-                pthread_mutex_lock (&args->mutex);                      \
-                {                                                       \
-                        while (args->wakecnt < cnt)                     \
-                                pthread_cond_wait (&args->cond,         \
-                                                   &args->mutex);       \
-                }                                                       \
-                pthread_mutex_unlock (&args->mutex);                    \
-                pthread_mutex_destroy (&args->mutex);                   \
-                pthread_cond_destroy (&args->cond);                     \
-        }                                                               \
-        } while (0)
-
-
-#define __yield(args) __waitfor(args, 1)
+#define __yield(args) do {						\
+	if (args->task) {				                \
+		synctask_yield (args->task);				\
+	} else {							\
+		pthread_mutex_lock (&args->mutex);			\
+		{							\
+			while (!args->done)				\
+				pthread_cond_wait (&args->cond,		\
+						   &args->mutex);	\
+		}							\
+		pthread_mutex_unlock (&args->mutex);			\
+		pthread_mutex_destroy (&args->mutex);			\
+		pthread_cond_destroy (&args->cond);			\
+	}								\
+	} while (0)
 
 
 #define SYNCOP(subvol, stb, cbk, op, params ...) do {                   \
@@ -223,9 +231,9 @@ void synctask_yield (struct synctask *task);
 void synctask_yawn (struct synctask *task);
 void synctask_waitfor (struct synctask *task, int count);
 
-#define synctask_barrier_init(args) __yawn (args)
-#define synctask_barrier_wait(args, n) __waitfor (args, n)
-#define synctask_barrier_wake(args) __wake (args)
+#define synctask_barrier_init(args) syncbarrier_init (&args->barrier)
+#define synctask_barrier_wait(args, n) syncbarrier_wait (&args->barrier, n)
+#define synctask_barrier_wake(args) syncbarrier_wake (&args->barrier)
 
 int synctask_setid (struct synctask *task, uid_t uid, gid_t gid);
 #define SYNCTASK_SETID(uid, gid) synctask_setid (synctask_get(), uid, gid);
@@ -237,6 +245,12 @@ int synclock_lock (synclock_t *lock);
 int synclock_trylock (synclock_t *lock);
 int synclock_unlock (synclock_t *lock);
 
+
+int syncbarrier_init (syncbarrier_t *barrier);
+int syncbarrier_wait (syncbarrier_t *barrier, int waitfor);
+int syncbarrier_wake (syncbarrier_t *barrier);
+int syncbarrier_destroy (syncbarrier_t *barrier);
+
 int syncop_lookup (xlator_t *subvol, loc_t *loc, dict_t *xattr_req,
                    /* out */
                    struct iatt *iatt, dict_t **xattr_rsp, struct iatt *parent);
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index a27d0479193..e6c681df8f3 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -1384,14 +1384,13 @@ out:
 
 
 int
-rpc_clnt_submit2 (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
-                  int procnum, fop_cbk_fn_t cbkfn,
-                  struct iovec *proghdr, int proghdrcount,
-                  struct iovec *progpayload, int progpayloadcount,
-                  struct iobref *iobref, void *frame, struct iovec *rsphdr,
-                  int rsphdr_count, struct iovec *rsp_payload,
-                  int rsp_payload_count, struct iobref *rsp_iobref,
-                  gf_boolean_t *lost)
+rpc_clnt_submit (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
+                 int procnum, fop_cbk_fn_t cbkfn,
+                 struct iovec *proghdr, int proghdrcount,
+                 struct iovec *progpayload, int progpayloadcount,
+                 struct iobref *iobref, void *frame, struct iovec *rsphdr,
+                 int rsphdr_count, struct iovec *rsp_payload,
+                 int rsp_payload_count, struct iobref *rsp_iobref)
 {
         rpc_clnt_connection_t *conn        = NULL;
         struct iobuf          *request_iob = NULL;
@@ -1402,7 +1401,6 @@ rpc_clnt_submit2 (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
         int                    proglen     = 0;
         char                   new_iobref  = 0;
         uint64_t               callid      = 0;
-        gf_boolean_t           cbk_lost    = _gf_true;
 
         if (!rpc || !prog || !frame) {
                 goto out;
@@ -1491,7 +1489,6 @@ rpc_clnt_submit2 (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
 
                 if ((ret >= 0) && frame) {
                         /* Save the frame in queue */
-                        cbk_lost = _gf_false;
                         __save_frame (rpc, frame, rpcreq);
 
                         gf_log ("rpc-clnt", GF_LOG_TRACE, "submitted request "
@@ -1522,31 +1519,12 @@ out:
                 if (rpcreq) {
                         rpcreq->rpc_status = -1;
                         cbkfn (rpcreq, NULL, 0, frame);
-                        cbk_lost = _gf_false;
                         mem_put (rpcreq);
                 }
         }
-        if (lost)
-                *lost = cbk_lost;
         return ret;
 }
 
-int
-rpc_clnt_submit (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
-                 int procnum, fop_cbk_fn_t cbkfn,
-                 struct iovec *proghdr, int proghdrcount,
-                 struct iovec *progpayload, int progpayloadcount,
-                 struct iobref *iobref, void *frame, struct iovec *rsphdr,
-                 int rsphdr_count, struct iovec *rsp_payload,
-                 int rsp_payload_count, struct iobref *rsp_iobref)
-{
-        return rpc_clnt_submit2 (rpc, prog, procnum, cbkfn, proghdr,
-                                 proghdrcount, progpayload, progpayloadcount,
-                                 iobref, frame, rsphdr, rsphdr_count,
-                                 rsp_payload, rsp_payload_count, rsp_iobref,
-                                 NULL);
-}
-
 
 struct rpc_clnt *
 rpc_clnt_ref (struct rpc_clnt *rpc)
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index 34612fcc452..0da1655590a 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -216,15 +216,6 @@ int rpc_clnt_submit (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
                      int rsphdr_count, struct iovec *rsp_payload,
                      int rsp_payload_count, struct iobref *rsp_iobref);
 
-int rpc_clnt_submit2 (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
-                     int procnum, fop_cbk_fn_t cbkfn,
-                     struct iovec *proghdr, int proghdrcount,
-                     struct iovec *progpayload, int progpayloadcount,
-                     struct iobref *iobref, void *frame, struct iovec *rsphdr,
-                     int rsphdr_count, struct iovec *rsp_payload,
-                     int rsp_payload_count, struct iobref *rsp_iobref,
-                     gf_boolean_t *cbk_lost);
-
 struct rpc_clnt *
 rpc_clnt_ref (struct rpc_clnt *rpc);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index cebf1977881..766a24a8b66 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -26,6 +26,8 @@ gd_synctask_barrier_wait (struct syncargs *args, int count)
         synclock_unlock (&conf->big_lock);
         synctask_barrier_wait (args, count);
         synclock_lock (&conf->big_lock);
+
+	syncbarrier_destroy (&args->barrier);
 }
 
 static void
@@ -82,8 +84,7 @@ gd_brick_op_req_free (gd1_mgmt_brick_op_req *req)
 int
 gd_syncop_submit_request (struct rpc_clnt *rpc, void *req,
                           void *cookie, rpc_clnt_prog_t *prog,
-                          int procnum, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc,
-                          gf_boolean_t *cbk_lost)
+                          int procnum, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
         int            ret      = -1;
         struct iobuf  *iobuf    = NULL;
@@ -126,9 +127,9 @@ gd_syncop_submit_request (struct rpc_clnt *rpc, void *req,
         frame->local = cookie;
 
         /* Send the msg */
-        ret = rpc_clnt_submit2 (rpc, prog, procnum, cbkfn, &iov, count, NULL,
-                                0, iobref, frame, NULL, 0, NULL, 0, NULL,
-                                cbk_lost);
+        ret = rpc_clnt_submit (rpc, prog, procnum, cbkfn,
+                               &iov, count, NULL, 0, iobref,
+                               frame, NULL, 0, NULL, 0, NULL);
 
         /* TODO: do we need to start ping also? */
 
@@ -208,8 +209,8 @@ out:
 }
 
 int32_t
-gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe)
+_gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
 {
         int                         ret         = -1;
         struct syncargs             *args       = NULL;
@@ -239,6 +240,13 @@ out:
         return 0;
 }
 
+int32_t
+gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_mgmt_lock_cbk);
+}
 
 int
 gd_syncop_mgmt_lock (struct rpc_clnt *rpc, struct syncargs *args,
@@ -246,22 +254,21 @@ gd_syncop_mgmt_lock (struct rpc_clnt *rpc, struct syncargs *args,
 {
         int                       ret = -1;
         gd1_mgmt_cluster_lock_req req  = {{0},};
-        gf_boolean_t              cbk_lost = _gf_true;
+        glusterd_conf_t           *conf = THIS->private;
 
         uuid_copy (req.uuid, my_uuid);
+        synclock_unlock (&conf->big_lock);
         ret = gd_syncop_submit_request (rpc, &req, args, &gd_mgmt_prog,
                                         GLUSTERD_MGMT_CLUSTER_LOCK,
                                         gd_syncop_mgmt_lock_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req,
-                                        &cbk_lost);
-        if (cbk_lost)
-                synctask_barrier_wake(args);
+                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
+        synclock_lock (&conf->big_lock);
         return ret;
 }
 
 int32_t
-gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                           int count, void *myframe)
+_gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                            int count, void *myframe)
 {
         int                         ret         = -1;
         struct syncargs             *args       = NULL;
@@ -291,6 +298,14 @@ out:
         return 0;
 }
 
+int32_t
+gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                           int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_mgmt_unlock_cbk);
+}
+
 
 int
 gd_syncop_mgmt_unlock (struct rpc_clnt *rpc, struct syncargs *args,
@@ -298,22 +313,21 @@ gd_syncop_mgmt_unlock (struct rpc_clnt *rpc, struct syncargs *args,
 {
         int                         ret     = -1;
         gd1_mgmt_cluster_unlock_req req     = {{0},};
-        gf_boolean_t                cbk_lost = _gf_true;
+        glusterd_conf_t             *conf   = THIS->private;
 
         uuid_copy (req.uuid, my_uuid);
+        synclock_unlock (&conf->big_lock);
         ret = gd_syncop_submit_request (rpc, &req, args, &gd_mgmt_prog,
                                         GLUSTERD_MGMT_CLUSTER_UNLOCK,
                                         gd_syncop_mgmt_unlock_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req,
-                                        &cbk_lost);
-        if (cbk_lost)
-                synctask_barrier_wake(args);
+                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
+        synclock_lock (&conf->big_lock);
         return ret;
 }
 
 int32_t
-gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+_gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
 {
         int                         ret         = -1;
         gd1_mgmt_stage_op_rsp       rsp         = {{0},};
@@ -377,6 +391,14 @@ out:
         return 0;
 }
 
+int32_t
+gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_stage_op_cbk);
+}
+
 
 int
 gd_syncop_mgmt_stage_op (struct rpc_clnt *rpc, struct syncargs *args,
@@ -384,8 +406,8 @@ gd_syncop_mgmt_stage_op (struct rpc_clnt *rpc, struct syncargs *args,
                          dict_t *dict_out, dict_t *op_ctx)
 {
         gd1_mgmt_stage_op_req *req  = NULL;
-        int                   ret  = -1;
-        gf_boolean_t          cbk_lost = _gf_true;
+        glusterd_conf_t       *conf = THIS->private;
+        int                   ret   = -1;
 
         req = GF_CALLOC (1, sizeof (*req), gf_gld_mt_mop_stage_req_t);
         if (!req)
@@ -399,22 +421,20 @@ gd_syncop_mgmt_stage_op (struct rpc_clnt *rpc, struct syncargs *args,
         if (ret)
                 goto out;
 
+        synclock_unlock (&conf->big_lock);
         ret = gd_syncop_submit_request (rpc, req, args, &gd_mgmt_prog,
                                         GLUSTERD_MGMT_STAGE_OP,
                                         gd_syncop_stage_op_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_stage_op_req,
-                                        &cbk_lost);
+                                        (xdrproc_t) xdr_gd1_mgmt_stage_op_req);
+        synclock_lock (&conf->big_lock);
 out:
         gd_stage_op_req_free (req);
-        if (cbk_lost)
-                synctask_barrier_wake(args);
-
         return ret;
 
 }
 
 int32_t
-gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+_gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
                         int count, void *myframe)
 {
         struct syncargs        *args  = NULL;
@@ -469,6 +489,14 @@ out:
         return 0;
 }
 
+int32_t
+gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_brick_op_cbk);
+}
+
 int
 gd_syncop_mgmt_brick_op (struct rpc_clnt *rpc, glusterd_pending_node_t *pnode,
                          int op, dict_t *dict_out, dict_t *op_ctx,
@@ -532,8 +560,8 @@ out:
 }
 
 int32_t
-gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe)
+_gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
 {
         int                         ret         = -1;
         gd1_mgmt_commit_op_rsp      rsp         = {{0},};
@@ -595,15 +623,23 @@ out:
         return 0;
 }
 
+int32_t
+gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_commit_op_cbk);
+}
+
 
 int
 gd_syncop_mgmt_commit_op (struct rpc_clnt *rpc, struct syncargs *args,
                           uuid_t my_uuid, uuid_t recv_uuid,
                           int op, dict_t *dict_out, dict_t *op_ctx)
 {
+        glusterd_conf_t        *conf = THIS->private;
         gd1_mgmt_commit_op_req *req  = NULL;
         int                    ret  = -1;
-        gf_boolean_t           cbk_lost = _gf_true;
 
         req = GF_CALLOC (1, sizeof (*req), gf_gld_mt_mop_commit_req_t);
         if (!req)
@@ -617,16 +653,14 @@ gd_syncop_mgmt_commit_op (struct rpc_clnt *rpc, struct syncargs *args,
         if (ret)
                 goto out;
 
+        synclock_unlock (&conf->big_lock);
         ret = gd_syncop_submit_request (rpc, req, args, &gd_mgmt_prog,
                                         GLUSTERD_MGMT_COMMIT_OP ,
                                         gd_syncop_commit_op_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_commit_op_req,
-                                        &cbk_lost);
+                                        (xdrproc_t) xdr_gd1_mgmt_commit_op_req);
+        synclock_lock (&conf->big_lock);
 out:
         gd_commit_op_req_free (req);
-        if (cbk_lost)
-                synctask_barrier_wake(args);
-
         return ret;
 }
 
@@ -942,12 +976,8 @@ gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, char **op
                                 "due to rpc failure.");
                         goto out;
                 }
-                /*This is to ensure that the brick_op_cbk is able to take
-                 * the big lock*/
-                synclock_unlock (&conf->big_lock);
                 ret = gd_syncop_mgmt_brick_op (rpc, pending_node, op, req_dict,
                                                op_ctx, op_errstr);
-                synclock_lock (&conf->big_lock);
                 if (ret)
                         goto out;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.h b/xlators/mgmt/glusterd/src/glusterd-syncop.h
index 212e495d52d..9318862e508 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.h
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.h
@@ -18,23 +18,27 @@
 #define GD_SYNCOP(rpc, stb, cbk, req, prog, procnum, xdrproc) do {      \
                 int ret = 0;                                            \
                 struct  synctask        *task = NULL;                   \
-                gf_boolean_t            cbk_lost = _gf_true;            \
+                glusterd_conf_t         *conf= THIS->private;           \
+                                                                        \
                 task = synctask_get ();                                 \
                 stb->task = task;                                       \
                                                                         \
+                /*This is to ensure that the brick_op_cbk is able to    \
+                 * take the big lock*/                                  \
+                synclock_unlock (&conf->big_lock);                      \
                 ret = gd_syncop_submit_request (rpc, req, stb,          \
                                                 prog, procnum, cbk,     \
-                                                (xdrproc_t)xdrproc,     \
-                                                &cbk_lost);             \
-                if (!cbk_lost)                                          \
+                                                (xdrproc_t)xdrproc);    \
+                if (!ret)                                               \
                         synctask_yield (stb->task);                     \
+                synclock_lock (&conf->big_lock);                        \
         } while (0)
 
 
 int gd_syncop_submit_request (struct rpc_clnt *rpc, void *req,
                                void *cookie, rpc_clnt_prog_t *prog,
                                int procnum, fop_cbk_fn_t cbkfn,
-                               xdrproc_t xdrproc, gf_boolean_t *cbk_lost);
+                               xdrproc_t xdrproc);
 
 
 int gd_syncop_mgmt_lock (struct rpc_clnt *rpc, struct syncargs *arg,