glusterfs: Not able to mount running volume after enable brick mux and stopped any volume

Problem: After enabled brick mux if any volume has down and then try ot run mount
         with running volume , mount command is hung.

Solution: After enable brick mux server has shared one data structure server_conf
          for all associated subvolumes.After down any subvolume in some
          ungraceful manner (remove brick directory) posix xlator sends
          GF_EVENT_CHILD_DOWN event to parent xlatros and server notify
          updates the child_up to false in server_conf.When client is trying
          to communicate with server through mount it checks conf->child_up
          and it is FALSE so it throws message "translator are not yet ready".
          From this patch updated structure server_conf to save child_up status
          for xlator wise. Another improtant correction from this patch is
          cleanup threads from server side xlators after stop the volume.

BUG: 1453977
Change-Id: Ic54da3f01881b7c9429ce92cc569236eb1d43e0d
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>
Reviewed-on: https://review.gluster.org/17356
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra Talur <rtalur@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>

6 files changed, 96 insertions, 2 deletions

diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub.c b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
index 895e7690a5e..b8a34422522 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub.c
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
@@ -10,6 +10,7 @@
 
 #include <ctype.h>
 #include <sys/uio.h>
+#include <signal.h>
 
 #include "glusterfs.h"
 #include "xlator.h"
@@ -212,6 +213,35 @@ out:
         return ret;
 }
 
+
+int
+notify (xlator_t *this, int event, void *data, ...)
+{
+        br_stub_private_t *priv = NULL;
+
+        if (!this)
+                return 0;
+
+        priv = this->private;
+        if (!priv)
+                return 0;
+
+        switch (event) {
+        case GF_EVENT_CLEANUP:
+                if (priv->signth) {
+                        (void) gf_thread_cleanup_xint (priv->signth);
+                        priv->signth = 0;
+                }
+                if (priv->container.thread) {
+                        (void) gf_thread_cleanup_xint (priv->container.thread);
+                        priv->container.thread = 0;
+                }
+                break;
+        }
+        default_notify (this, event, data);
+        return 0;
+}
+
 void
 fini (xlator_t *this)
 {
@@ -832,6 +862,7 @@ br_stub_signth (void *arg)
 
         THIS = this;
         while (1) {
+
                 pthread_mutex_lock (&priv->lock);
                 {
                         while (list_empty (&priv->squeue))
diff --git a/xlators/features/changelog/src/changelog-ev-handle.c b/xlators/features/changelog/src/changelog-ev-handle.c
index caa5bbbadcd..38e127b7eb5 100644
--- a/xlators/features/changelog/src/changelog-ev-handle.c
+++ b/xlators/features/changelog/src/changelog-ev-handle.c
@@ -342,6 +342,7 @@ changelog_ev_dispatch (void *data)
 
         while (1) {
                 /* TODO: change this to be pthread cond based.. later */
+
                 tv.tv_sec = 1;
                 tv.tv_usec = 0;
                 select (0, NULL, NULL, NULL, &tv);
@@ -363,6 +364,7 @@ changelog_ev_dispatch (void *data)
                         gf_msg (this->name, GF_LOG_WARNING, 0,
                                 CHANGELOG_MSG_PUT_BUFFER_FAILED,
                                 "failed to put buffer after consumption");
+
         }
 
         return NULL;
diff --git a/xlators/features/changelog/src/changelog.c b/xlators/features/changelog/src/changelog.c
index 697e8ca7b11..20af188d0d4 100644
--- a/xlators/features/changelog/src/changelog.c
+++ b/xlators/features/changelog/src/changelog.c
@@ -21,6 +21,7 @@
 #include "changelog-messages.h"
 
 #include <pthread.h>
+#include <signal.h>
 
 #include "changelog-rpc.h"
 #include "errno.h"
@@ -2098,6 +2099,7 @@ notify (xlator_t *this, int event, void *data, ...)
         int                     ret             = 0;
         int                     ret1            = 0;
         struct list_head        queue           = {0, };
+        int                     i               = 0;
 
         INIT_LIST_HEAD (&queue);
 
@@ -2105,6 +2107,20 @@ notify (xlator_t *this, int event, void *data, ...)
         if (!priv)
                 goto out;
 
+        if (event == GF_EVENT_CLEANUP) {
+                if (priv->connector) {
+                        (void) gf_thread_cleanup_xint (priv->connector);
+                        priv->connector = 0;
+                }
+
+                for (; i < NR_DISPATCHERS; i++) {
+                        if (priv->ev_dispatcher[i]) {
+                                (void) gf_thread_cleanup_xint (priv->ev_dispatcher[i]);
+                                priv->ev_dispatcher[i] = 0;
+                        }
+               }
+        }
+
         if (event == GF_EVENT_TRANSLATOR_OP) {
 
                 dict = data;
diff --git a/xlators/features/changetimerecorder/src/changetimerecorder.c b/xlators/features/changetimerecorder/src/changetimerecorder.c
index 574b627a825..1811cec420f 100644
--- a/xlators/features/changetimerecorder/src/changetimerecorder.c
+++ b/xlators/features/changetimerecorder/src/changetimerecorder.c
@@ -2297,6 +2297,32 @@ out:
         return 0;
 }
 
+int
+notify (xlator_t *this, int event, void *data, ...)
+{
+
+       gf_ctr_private_t *priv = NULL;
+       int               ret  = 0;
+
+       priv = this->private;
+
+       if (!priv)
+               goto out;
+
+       if (event == GF_EVENT_CLEANUP) {
+               if (fini_db (priv->_db_conn)) {
+                       gf_msg (this->name, GF_LOG_WARNING, 0,
+                                CTR_MSG_CLOSE_DB_CONN_FAILED, "Failed closing "
+                                "db connection");
+               }
+       } else  {
+               ret = default_notify (this, event, data);
+       }
+out:
+      return ret;
+
+}
+
 int32_t
 mem_acct_init (xlator_t *this)
 {
diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c
index 975d5f998bd..f0462dcb7eb 100644
--- a/xlators/features/index/src/index.c
+++ b/xlators/features/index/src/index.c
@@ -15,6 +15,7 @@
 #include "common-utils.h"
 #include "index-messages.h"
 #include <ftw.h>
+#include <signal.h>
 
 #define XATTROP_SUBDIR "xattrop"
 #define DIRTY_SUBDIR "dirty"
@@ -2264,7 +2265,6 @@ init (xlator_t *this)
         int ret = -1;
         int64_t count = -1;
         index_priv_t *priv = NULL;
-        pthread_t thread;
         pthread_attr_t  w_attr;
         gf_boolean_t    mutex_inited = _gf_false;
         gf_boolean_t    cond_inited  = _gf_false;
@@ -2381,7 +2381,7 @@ init (xlator_t *this)
         count = index_fetch_link_count (this, XATTROP);
         index_set_link_count (priv, count, XATTROP);
 
-        ret = gf_thread_create (&thread, &w_attr, index_worker, this);
+        ret = gf_thread_create (&priv->thread, &w_attr, index_worker, this);
         if (ret) {
                 gf_msg (this->name, GF_LOG_WARNING, ret,
                         INDEX_MSG_WORKER_THREAD_CREATE_FAILED,
@@ -2494,6 +2494,24 @@ int
 notify (xlator_t *this, int event, void *data, ...)
 {
         int     ret = 0;
+        index_priv_t *priv = NULL;
+
+        if (!this)
+                return 0;
+
+        priv = this->private;
+        if (!priv)
+                return 0;
+
+        switch (event) {
+        case GF_EVENT_CLEANUP:
+                if (priv->thread) {
+                        (void) gf_thread_cleanup_xint (priv->thread);
+                        priv->thread = 0;
+                }
+                break;
+        }
+
         ret = default_notify (this, event, data);
         return ret;
 }
diff --git a/xlators/features/index/src/index.h b/xlators/features/index/src/index.h
index 5fb5a65cd8e..f622ceced5b 100644
--- a/xlators/features/index/src/index.h
+++ b/xlators/features/index/src/index.h
@@ -60,6 +60,7 @@ typedef struct index_priv {
         dict_t  *pending_watchlist;
         dict_t  *complete_watchlist;
         int64_t  pending_count;
+        pthread_t thread;
 } index_priv_t;
 
 typedef struct index_local {