io-threads: re-port changes since 3.6 on top of FB version

Includes io-threads parts of the following patches:

  9e3fea1 performance/io-threads: Exit all threads on PARENT_DOWN
  2cfb7bc performance/io-threads: Exit threads in fini() as well

Change-Id: Id7cc7720e75414fb8a3ac2db68a5fe63c459ffe2
Signed-off-by: Jeff Darcy <jdarcy@fb.com>

3 files changed, 63 insertions, 22 deletions

diff --git a/xlators/performance/io-threads/src/Makefile.am b/xlators/performance/io-threads/src/Makefile.am
index bdbd0290fd6..5f5118b4e0f 100644
--- a/xlators/performance/io-threads/src/Makefile.am
+++ b/xlators/performance/io-threads/src/Makefile.am
@@ -1,12 +1,12 @@
 xlator_LTLIBRARIES = io-threads.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-io_threads_la_LDFLAGS = -module -avoid-version
+io_threads_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 io_threads_la_SOURCES = io-threads.c
 io_threads_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = io-threads.h iot-mem-types.h
+noinst_HEADERS = io-threads.h iot-mem-types.h io-threads-messages.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
               -DGLUSTERD_WORKDIR=$(GLUSTERD_WORKDIR)
diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c
index c0ebdd33caf..ac4bb66b868 100644
--- a/xlators/performance/io-threads/src/io-threads.c
+++ b/xlators/performance/io-threads/src/io-threads.c
@@ -8,11 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "call-stub.h"
 #include "defaults.h"
 #include "glusterfs.h"
@@ -715,7 +710,6 @@ iot_worker (void *data)
         struct timespec   sleep_till = {0, };
         int               ret = 0;
         int               pri = -1;
-        char              timeout = 0;
         char              bye = 0;
 
         conf = data;
@@ -732,6 +726,11 @@ iot_worker (void *data)
                         }
 
                         while (conf->queue_size == 0) {
+                                if (conf->down) {
+                                        bye = _gf_true;/*Avoid sleep*/
+                                        break;
+                                }
+
                                 clock_gettime (CLOCK_REALTIME_COARSE, &sleep_till);
                                 sleep_till.tv_sec += conf->idle_time;
 
@@ -741,25 +740,30 @@ iot_worker (void *data)
                                                               &sleep_till);
                                 conf->sleep_count--;
 
-                                if (ret == ETIMEDOUT) {
-                                        timeout = 1;
+                                if (conf->down || ret == ETIMEDOUT) {
+                                        bye = _gf_true;
                                         break;
                                 }
                         }
 
-                        if (timeout) {
-                                if (conf->curr_count > IOT_MIN_THREADS) {
+                        if (bye) {
+                                if (conf->down ||
+                                    conf->curr_count > IOT_MIN_THREADS) {
                                         conf->curr_count--;
-                                        bye = 1;
+                                        if (conf->curr_count == 0)
+                                           pthread_cond_broadcast (&conf->cond);
                                         gf_log (conf->this->name, GF_LOG_DEBUG,
-                                                "timeout, terminated. conf->curr_count=%d",
+                                                "terminated. "
+                                                "conf->curr_count=%d",
                                                 conf->curr_count);
                                 } else {
-                                        timeout = 0;
+                                        bye = _gf_false;
                                 }
                         }
 
-                        stub = __iot_dequeue (conf, &pri);
+                        if (!bye) {
+                                stub = __iot_dequeue (conf, &pri);
+                        }
                 }
                 pthread_mutex_unlock (&conf->mutex);
 
@@ -771,6 +775,7 @@ iot_worker (void *data)
                         } else {
                                 call_resume (stub);
                         }
+                        stub = NULL;
                 }
 
                 if (bye) {
@@ -1784,12 +1789,14 @@ init (xlator_t *this)
                         "pthread_cond_init failed (%d)", ret);
                 goto out;
         }
+        conf->cond_inited = _gf_true;
 
         if ((ret = pthread_mutex_init (&conf->mutex, NULL)) != 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "pthread_mutex_init failed (%d)", ret);
                 goto out;
         }
+        conf->mutex_inited = _gf_true;
 
         set_stack_size (conf);
 
@@ -1874,12 +1881,49 @@ out:
 	return ret;
 }
 
+static void
+iot_exit_threads (iot_conf_t *conf)
+{
+        pthread_mutex_lock (&conf->mutex);
+        {
+                conf->down = _gf_true;
+                /*Let all the threads know that xl is going down*/
+                pthread_cond_broadcast (&conf->cond);
+                while (conf->curr_count)/*Wait for threads to exit*/
+                        pthread_cond_wait (&conf->cond, &conf->mutex);
+        }
+        pthread_mutex_unlock (&conf->mutex);
+}
+
+int
+notify (xlator_t *this, int32_t event, void *data, ...)
+{
+        iot_conf_t *conf = this->private;
+
+        if (GF_EVENT_PARENT_DOWN == event)
+                iot_exit_threads (conf);
+        default_notify (this, event, data);
+
+        return 0;
+}
 
 void
 fini (xlator_t *this)
 {
 	iot_conf_t *conf = this->private;
 
+        if (!conf)
+                return;
+
+        if (conf->mutex_inited && conf->cond_inited)
+                iot_exit_threads (conf);
+
+        if (conf->cond_inited)
+                pthread_cond_destroy (&conf->cond);
+
+        if (conf->mutex_inited)
+                pthread_mutex_destroy (&conf->mutex);
+
         stop_iot_watchdog (this);
         stop_iot_reinit_ns_conf_thread (this);
 
diff --git a/xlators/performance/io-threads/src/io-threads.h b/xlators/performance/io-threads/src/io-threads.h
index 5c3403a7153..b6d44621067 100644
--- a/xlators/performance/io-threads/src/io-threads.h
+++ b/xlators/performance/io-threads/src/io-threads.h
@@ -11,12 +11,6 @@
 #ifndef __IOT_H
 #define __IOT_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
 #include "compat-errno.h"
 #include "glusterfs.h"
 #include "logging.h"
@@ -108,6 +102,9 @@ struct iot_conf {
         gf_boolean_t        queue_marked[IOT_PRI_MAX];
 
 	gf_boolean_t        cleanup_disconnected_reqs;
+        gf_boolean_t         down; /*PARENT_DOWN event is notified*/
+        gf_boolean_t         mutex_inited;
+        gf_boolean_t         cond_inited;
 };
 
 typedef struct iot_conf iot_conf_t;