syncop: don't wake task in synctask_wake unless really needed

Problem:

In EC and AFR, we launch synctasks during self-heal.

(i) These tasks usually stackwind a FOP to all its children and call
synctask_yield() which does a swapcontext to synctask_switchto() and puts the
task in syncenv's waitq by calling __wait(task). This happends as long as the
FOP ckbs from all children haven't been received.

(ii) For each FOP cbk, we call synctask_wake() which again does a swapcontext
to synctask_switchto() which now puts the task in syncenv's runq by calling
__run(task). When the task runs and the conext switches back to the FOP path,
it puts the task in waitq because we haven't heard from all children as
explained in (i).

Thus we are unnecessarily using the swapcontext syscalls to just toggle
the task back and forth between the waitq and runq.

Fix:
Store the stackwind count in new variable 'syncbarrier->waitfor' before
winding the fop. In each cbk when we call synctask_wake(),  perform an actual
wake only if the cbk count == stackwind count.

Change-Id: Id62d3b6ffed5a8c50f8b79267fb34e9470ba5ed5
BUG: 1434274
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Signed-off-by: Ashish Pandey <aspandey@redhat.com>
Reviewed-on: https://review.gluster.org/16931
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Niels de Vos <ndevos@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>

4 files changed, 28 insertions, 9 deletions

diff --git a/libglusterfs/src/cluster-syncop.c b/libglusterfs/src/cluster-syncop.c
index 98a46c85e4b..36945d69379 100644
--- a/libglusterfs/src/cluster-syncop.c
+++ b/libglusterfs/src/cluster-syncop.c
@@ -31,13 +31,18 @@
         if (syncbarrier_init (&__local.barrier))                        \
                 break;                                                  \
         frame->local = &__local;                                        \
+        for (__i = 0; __i < numsubvols; __i++) {                        \
+                if (on[__i]) {                                          \
+                        __count++;                                      \
+                }                                                       \
+        }                                                               \
+        __local.barrier.waitfor = __count;                              \
         for (__i = 0; __i < numsubvols; __i++) {		        \
                 if (!on[__i])                                           \
                         continue;				        \
                 STACK_WIND_COOKIE (frame, cluster_##fop##_cbk,          \
                                    (void *)(long) __i, subvols[__i],    \
                                    subvols[__i]->fops->fop, args);      \
-                __count++;						\
         }								\
         syncbarrier_wait (&__local.barrier, __count);			\
         syncbarrier_destroy (&__local.barrier);                         \
diff --git a/libglusterfs/src/syncop.c b/libglusterfs/src/syncop.c
index 3fa798a4342..246229f1cc2 100644
--- a/libglusterfs/src/syncop.c
+++ b/libglusterfs/src/syncop.c
@@ -1086,6 +1086,7 @@ syncbarrier_init (struct syncbarrier *barrier)
 
 	pthread_cond_init (&barrier->cond, 0);
 	barrier->count = 0;
+        barrier->waitfor = 0;
 	INIT_LIST_HEAD (&barrier->waitq);
 
 	return pthread_mutex_init (&barrier->guard, 0);
@@ -1162,6 +1163,8 @@ __syncbarrier_wake (struct syncbarrier *barrier)
 	}
 
 	barrier->count++;
+        if (barrier->waitfor && (barrier->count < barrier->waitfor))
+                return 0;
 
 	pthread_cond_signal (&barrier->cond);
 	if (!list_empty (&barrier->waitq)) {
@@ -1169,6 +1172,7 @@ __syncbarrier_wake (struct syncbarrier *barrier)
                 list_del_init (&task->waitq);
 		synctask_wake (task);
 	}
+        barrier->waitfor = 0;
 
 	return 0;
 }
diff --git a/libglusterfs/src/syncop.h b/libglusterfs/src/syncop.h
index 0d0da58f4cf..a9cdee1fa00 100644
--- a/libglusterfs/src/syncop.h
+++ b/libglusterfs/src/syncop.h
@@ -138,6 +138,8 @@ struct syncbarrier {
 	pthread_cond_t      cond;  /* waiting non-synctasks */
 	struct list_head    waitq; /* waiting synctasks */
 	int                 count; /* count the number of wakes */
+        int                 waitfor; /* no. of wakes until which task can be in
+                                        waitq before being woken up. */
 };
 typedef struct syncbarrier syncbarrier_t;
 
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index 0a3d6482ca3..735e520070e 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -19,16 +19,23 @@
 #define AFR_ONALL(frame, rfn, fop, args ...) do {			\
 	afr_local_t *__local = frame->local;				\
 	afr_private_t *__priv = frame->this->private;			\
-	int __i = 0, __count = 0;					\
+        int __i = 0, __count = 0;                                       \
+        unsigned char *__child_up = NULL;                               \
+                                                                        \
+        __child_up = alloca0 (__priv->child_count);                     \
+        memcpy (__child_up, __priv->child_up,                           \
+                sizeof (*__child_up) * __priv->child_count);            \
+        __count = AFR_COUNT (__child_up, __priv->child_count);          \
 									\
-	afr_local_replies_wipe (__local, __priv);				\
+        __local->barrier.waitfor = __count;                             \
+	afr_local_replies_wipe (__local, __priv);			\
 									\
 	for (__i = 0; __i < __priv->child_count; __i++) {		\
-		if (!__priv->child_up[__i]) continue;			\
+		if (!__child_up[__i])                                   \
+                        continue;			                \
 		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
 				   __priv->children[__i],		\
 				   __priv->children[__i]->fops->fop, args); \
-		__count++;						\
 	}								\
 	syncbarrier_wait (&__local->barrier, __count);			\
 	} while (0)
@@ -40,16 +47,17 @@
 #define AFR_ONLIST(list, frame, rfn, fop, args ...) do {		\
 	afr_local_t *__local = frame->local;				\
 	afr_private_t *__priv = frame->this->private;			\
-	int __i = 0, __count = 0;					\
+	int __i = 0;                                                    \
+        int __count = AFR_COUNT (list, __priv->child_count);            \
 									\
-	afr_local_replies_wipe (__local, __priv);				\
+        __local->barrier.waitfor = __count;                             \
+	afr_local_replies_wipe (__local, __priv);			\
 									\
 	for (__i = 0; __i < __priv->child_count; __i++) {		\
 		if (!list[__i]) continue;				\
 		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
 				   __priv->children[__i],		\
 				   __priv->children[__i]->fops->fop, args); \
-		__count++;						\
 	}								\
 	syncbarrier_wait (&__local->barrier, __count);			\
 	} while (0)
@@ -60,7 +68,7 @@
 	afr_private_t *__priv = frame->this->private;			\
 	int __i = 0;							\
 									\
-	afr_local_replies_wipe (__local, __priv);				\
+	afr_local_replies_wipe (__local, __priv);			\
 									\
 	for (__i = 0; __i < __priv->child_count; __i++) {		\
 		if (!__priv->child_up[__i]) continue;			\