glusterd: Refresh glusterd-syncop fixes from master

Following commits were cherry-picked from master,
044f8ce syncop: Remove task from synclock's waitq before 'wake'
cb6aeed glusterd: Give up big lock before performing any RPC
46572fe Revert "glusterd: Fix spurious wakeups in glusterd syncops"
5021e04 synctask: implement barriers around yield, not the other way
4843937 glusterd: Syncop callbks should take big lock too

Change-Id: I5ae71ab98f9a336dc9bbf0e7b2ec50a6ed42b0f5
BUG: 948686
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/4938
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>
Reviewed-on: http://review.gluster.org/5021
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

2 files changed, 165 insertions, 53 deletions

diff --git a/libglusterfs/src/syncop.c b/libglusterfs/src/syncop.c
index 87f39841739..dbc52259fe7 100644
--- a/libglusterfs/src/syncop.c
+++ b/libglusterfs/src/syncop.c
@@ -80,24 +80,15 @@ __wait (struct synctask *task)
 
 
 void
-synctask_waitfor (struct synctask *task, int waitfor)
+synctask_yield (struct synctask *task)
 {
-	struct syncenv *env = NULL;
         xlator_t *oldTHIS = THIS;
 
-	env = task->env;
-
 #if defined(__NetBSD__) && defined(_UC_TLSBASE)
 	/* Preserve pthread private pointer through swapcontex() */
 	task->proc->sched.uc_flags &= ~_UC_TLSBASE;
 #endif
 
-	pthread_mutex_lock (&env->mutex);
-	{
-		task->waitfor = waitfor;
-	}
-	pthread_mutex_unlock (&env->mutex);
-
         if (swapcontext (&task->ctx, &task->proc->sched) < 0) {
                 gf_log ("syncop", GF_LOG_ERROR,
                         "swapcontext failed (%s)", strerror (errno));
@@ -108,13 +99,6 @@ synctask_waitfor (struct synctask *task, int waitfor)
 
 
 void
-synctask_yield (struct synctask *task)
-{
-	synctask_waitfor (task, 1);
-}
-
-
-void
 synctask_yawn (struct synctask *task)
 {
 	struct syncenv *env = NULL;
@@ -124,7 +108,6 @@ synctask_yawn (struct synctask *task)
 	pthread_mutex_lock (&env->mutex);
 	{
 		task->woken = 0;
-		task->waitfor = 0;
 	}
 	pthread_mutex_unlock (&env->mutex);
 }
@@ -139,9 +122,9 @@ synctask_wake (struct synctask *task)
 
         pthread_mutex_lock (&env->mutex);
         {
-                task->woken++;
+                task->woken = 1;
 
-                if (task->slept && task->woken >= task->waitfor)
+                if (task->slept)
                         __run (task);
 
 		pthread_cond_broadcast (&env->cond);
@@ -352,7 +335,6 @@ syncenv_task (struct syncproc *proc)
 
                 task->woken = 0;
                 task->slept = 0;
-                task->waitfor = 0;
 
                 task->proc = proc;
         }
@@ -390,7 +372,7 @@ synctask_switchto (struct synctask *task)
 
         pthread_mutex_lock (&env->mutex);
         {
-                if (task->woken >= task->waitfor) {
+                if (task->woken) {
                         __run (task);
                 } else {
                         task->slept = 1;
@@ -547,12 +529,11 @@ __synclock_lock (struct synclock *lock)
 		if (task) {
 			/* called within a synctask */
 			list_add_tail (&task->waitq, &lock->waitq);
-			{
-				pthread_mutex_unlock (&lock->guard);
-				synctask_yield (task);
-				pthread_mutex_lock (&lock->guard);
-			}
-			list_del_init (&task->waitq);
+                        pthread_mutex_unlock (&lock->guard);
+                        synctask_yield (task);
+                        /* task is removed from waitq in unlock,
+                         * under lock->guard.*/
+                        pthread_mutex_lock (&lock->guard);
 		} else {
 			/* called by a non-synctask */
 			pthread_cond_wait (&lock->cond, &lock->guard);
@@ -634,6 +615,7 @@ __synclock_unlock (synclock_t *lock)
 	pthread_cond_signal (&lock->cond);
 	if (!list_empty (&lock->waitq)) {
 		task = list_entry (lock->waitq.next, struct synctask, waitq);
+                list_del_init (&task->waitq);
 		synctask_wake (task);
 	}
 
@@ -655,6 +637,122 @@ synclock_unlock (synclock_t *lock)
 	return ret;
 }
 
+/* Barriers */
+
+int
+syncbarrier_init (struct syncbarrier *barrier)
+{
+	if (!barrier) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	pthread_cond_init (&barrier->cond, 0);
+	barrier->count = 0;
+	INIT_LIST_HEAD (&barrier->waitq);
+
+	return pthread_mutex_init (&barrier->guard, 0);
+}
+
+
+int
+syncbarrier_destroy (struct syncbarrier *barrier)
+{
+	if (!barrier) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	pthread_cond_destroy (&barrier->cond);
+	return pthread_mutex_destroy (&barrier->guard);
+}
+
+
+static int
+__syncbarrier_wait (struct syncbarrier *barrier, int waitfor)
+{
+	struct synctask *task = NULL;
+
+	if (!barrier) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	task = synctask_get ();
+
+	while (barrier->count < waitfor) {
+		if (task) {
+			/* called within a synctask */
+			list_add_tail (&task->waitq, &barrier->waitq);
+			{
+				pthread_mutex_unlock (&barrier->guard);
+				synctask_yield (task);
+				pthread_mutex_lock (&barrier->guard);
+			}
+			list_del_init (&task->waitq);
+		} else {
+			/* called by a non-synctask */
+			pthread_cond_wait (&barrier->cond, &barrier->guard);
+		}
+	}
+
+	barrier->count = 0;
+
+	return 0;
+}
+
+
+int
+syncbarrier_wait (struct syncbarrier *barrier, int waitfor)
+{
+	int ret = 0;
+
+	pthread_mutex_lock (&barrier->guard);
+	{
+		ret = __syncbarrier_wait (barrier, waitfor);
+	}
+	pthread_mutex_unlock (&barrier->guard);
+
+	return ret;
+}
+
+
+static int
+__syncbarrier_wake (struct syncbarrier *barrier)
+{
+	struct synctask *task = NULL;
+
+	if (!barrier) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	barrier->count++;
+
+	pthread_cond_signal (&barrier->cond);
+	if (!list_empty (&barrier->waitq)) {
+		task = list_entry (barrier->waitq.next, struct synctask, waitq);
+		synctask_wake (task);
+	}
+
+	return 0;
+}
+
+
+int
+syncbarrier_wake (struct syncbarrier *barrier)
+{
+	int ret = 0;
+
+	pthread_mutex_lock (&barrier->guard);
+	{
+		ret = __syncbarrier_wake (barrier);
+	}
+	pthread_mutex_unlock (&barrier->guard);
+
+	return ret;
+}
+
 
 /* FOPS */
 
diff --git a/libglusterfs/src/syncop.h b/libglusterfs/src/syncop.h
index b1a7229b3ee..f6eb423a4b6 100644
--- a/libglusterfs/src/syncop.h
+++ b/libglusterfs/src/syncop.h
@@ -57,7 +57,6 @@ struct synctask {
         void               *stack;
         int                 woken;
         int                 slept;
-        int                 waitfor;
         int                 ret;
 
         uid_t               uid;
@@ -107,6 +106,16 @@ struct synclock {
 };
 typedef struct synclock synclock_t;
 
+
+struct syncbarrier {
+	pthread_mutex_t     guard; /* guard the remaining members, pair @cond */
+	pthread_cond_t      cond;  /* waiting non-synctasks */
+	struct list_head    waitq; /* waiting synctasks */
+	int                 count; /* count the number of wakes */
+};
+typedef struct syncbarrier syncbarrier_t;
+
+
 struct syncargs {
         int                 op_ret;
         int                 op_errno;
@@ -127,11 +136,13 @@ struct syncargs {
         dict_t             *dict;
         pthread_mutex_t     lock_dict;
 
+	syncbarrier_t       barrier;
+
         /* do not touch */
         struct synctask    *task;
         pthread_mutex_t     mutex;
         pthread_cond_t      cond;
-        int                 wakecnt;
+	int                 done;
 };
 
 
@@ -142,7 +153,7 @@ struct syncargs {
         } else {                                                \
                 pthread_mutex_init (&args->mutex, NULL);        \
                 pthread_cond_init (&args->cond, NULL);          \
-                args->wakecnt = 0;                              \
+                args->done = 0;					\
         }                                                       \
         } while (0)
 
@@ -153,7 +164,7 @@ struct syncargs {
         } else {                                                \
                 pthread_mutex_lock (&args->mutex);              \
                 {                                               \
-                        args->wakecnt++;                        \
+                        args->done = 1;				\
                         pthread_cond_signal (&args->cond);      \
                 }                                               \
                 pthread_mutex_unlock (&args->mutex);            \
@@ -161,24 +172,21 @@ struct syncargs {
         } while (0)
 
 
-#define __waitfor(args, cnt) do {                                       \
-        if (args->task) {                                               \
-                synctask_waitfor (args->task, cnt);                     \
-        } else {                                                        \
-                pthread_mutex_lock (&args->mutex);                      \
-                {                                                       \
-                        while (args->wakecnt < cnt)                     \
-                                pthread_cond_wait (&args->cond,         \
-                                                   &args->mutex);       \
-                }                                                       \
-                pthread_mutex_unlock (&args->mutex);                    \
-                pthread_mutex_destroy (&args->mutex);                   \
-                pthread_cond_destroy (&args->cond);                     \
-        }                                                               \
-        } while (0)
-
-
-#define __yield(args) __waitfor(args, 1)
+#define __yield(args) do {						\
+	if (args->task) {				                \
+		synctask_yield (args->task);				\
+	} else {							\
+		pthread_mutex_lock (&args->mutex);			\
+		{							\
+			while (!args->done)				\
+				pthread_cond_wait (&args->cond,		\
+						   &args->mutex);	\
+		}							\
+		pthread_mutex_unlock (&args->mutex);			\
+		pthread_mutex_destroy (&args->mutex);			\
+		pthread_cond_destroy (&args->cond);			\
+	}								\
+	} while (0)
 
 
 #define SYNCOP(subvol, stb, cbk, op, params ...) do {                   \
@@ -223,9 +231,9 @@ void synctask_yield (struct synctask *task);
 void synctask_yawn (struct synctask *task);
 void synctask_waitfor (struct synctask *task, int count);
 
-#define synctask_barrier_init(args) __yawn (args)
-#define synctask_barrier_wait(args, n) __waitfor (args, n)
-#define synctask_barrier_wake(args) __wake (args)
+#define synctask_barrier_init(args) syncbarrier_init (&args->barrier)
+#define synctask_barrier_wait(args, n) syncbarrier_wait (&args->barrier, n)
+#define synctask_barrier_wake(args) syncbarrier_wake (&args->barrier)
 
 int synctask_setid (struct synctask *task, uid_t uid, gid_t gid);
 #define SYNCTASK_SETID(uid, gid) synctask_setid (synctask_get(), uid, gid);
@@ -237,6 +245,12 @@ int synclock_lock (synclock_t *lock);
 int synclock_trylock (synclock_t *lock);
 int synclock_unlock (synclock_t *lock);
 
+
+int syncbarrier_init (syncbarrier_t *barrier);
+int syncbarrier_wait (syncbarrier_t *barrier, int waitfor);
+int syncbarrier_wake (syncbarrier_t *barrier);
+int syncbarrier_destroy (syncbarrier_t *barrier);
+
 int syncop_lookup (xlator_t *subvol, loc_t *loc, dict_t *xattr_req,
                    /* out */
                    struct iatt *iatt, dict_t **xattr_rsp, struct iatt *parent);