glusterd: On gaining quorum spawn_daemons in new thread

During startup, if a glusterd has peers, it waits till quorum is
obtained to spawn bricks and other services. If peers are not present,
the daemons are started during glusterd' startup itself.

The spawning of daemons as a quorum action was done without using a
seperate thread, unlike the spawn on startup. Since, quotad was launched
using the blocking runner_run api, this leads to the thread being
blocked. The calling thread is almost always the epoll thread and this
leads to a deadlock. The runner_run call blocks the epoll thread waiting
for quotad to start, as a result glusterd cannot serve any requests. But
the startup of quotad is blocked as it cannot fetch the volfile from
glusterd.

The fix for this is to launch the spawn daemons task in a seperate
thread. This will free up the epoll thread and prevents the above
deadlock from happening.

Change-Id: Ife47b3591223cdfdfb2b4ea8dcd73e63f18e8749
BUG: 1095585
Signed-off-by: Kaushal M <kaushal@redhat.com>
Reviewed-on: http://review.gluster.org/7703
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Tested-by: Krishnan Parthasarathi <kparthas@redhat.com>

4 files changed, 44 insertions, 28 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
index 6dc35109d85..b14b7d729d0 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -1039,6 +1039,13 @@ glusterd_friend_sm ()
         gf_boolean_t                     is_await_conn = _gf_false;
         gf_boolean_t                     quorum_action = _gf_false;
         glusterd_friend_sm_state_t       old_state = GD_FRIEND_STATE_DEFAULT;
+        xlator_t                        *this = NULL;
+        glusterd_conf_t                 *priv = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
 
         while (!list_empty (&gd_friend_sm_queue)) {
                 list_for_each_entry_safe (event, tmp, &gd_friend_sm_queue, list) {
@@ -1137,7 +1144,9 @@ out:
              * the functions spawn process(es) only if they are not started yet.
              *
              * */
-                glusterd_spawn_daemons (NULL);
+                synclock_unlock (&priv->big_lock);
+                glusterd_launch_synctask (glusterd_spawn_daemons, NULL);
+                synclock_lock (&priv->big_lock);
                 glusterd_do_quorum_action ();
         }
         return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 1134d29dc5b..bb44b255dd4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -5800,6 +5800,9 @@ glusterd_nodesvc_start (char *server, gf_boolean_t wait)
         glusterd_nodesvc_set_socket_filepath (rundir, MY_UUID,
                                               sockfpath, sizeof (sockfpath));
 
+        if (gf_is_service_running(pidfile, NULL))
+                goto connect;
+
         runinit (&runner);
 
         if (priv->valgrind) {
@@ -5848,7 +5851,7 @@ glusterd_nodesvc_start (char *server, gf_boolean_t wait)
                 }
                 synclock_lock (&priv->big_lock);
         }
-
+connect:
         if (ret == 0) {
                 glusterd_nodesvc_connect (server, sockfpath);
         }
@@ -12737,3 +12740,30 @@ glusterd_snap_quorum_check (dict_t *dict, gf_boolean_t snap_volume, char **op_er
 out:
         return ret;
 }
+
+static int
+gd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
+{
+        glusterd_conf_t     *priv = THIS->private;
+        synclock_unlock (&priv->big_lock);
+        return ret;
+}
+
+void
+glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
+{
+        xlator_t        *this = NULL;
+        glusterd_conf_t *priv = NULL;
+        int             ret   = -1;
+
+        this = THIS;
+        priv = this->private;
+
+        synclock_lock (&priv->big_lock);
+        ret = synctask_new (this->ctx->env, fn, gd_default_synctask_cbk, NULL,
+                            opaque);
+        if (ret)
+                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn bricks"
+                        " and other volume related services");
+}
+
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 122969ec252..a0fdb196418 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -818,4 +818,7 @@ int32_t
 glusterd_snap_brick_create (glusterd_volinfo_t *snap_volinfo,
                             glusterd_brickinfo_t *brickinfo,
                             int32_t brick_count);
+
+void
+glusterd_launch_synctask (synctask_fn_t fn, void *opaque);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 52498c7fee2..d89c1ef7bcf 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -953,32 +953,6 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
 }
 
 
-static int
-gd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
-{
-        glusterd_conf_t     *priv = THIS->private;
-        synclock_unlock (&priv->big_lock);
-        return ret;
-}
-
-static void
-glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
-{
-        xlator_t        *this = NULL;
-        glusterd_conf_t *priv = NULL;
-        int             ret   = -1;
-
-        this = THIS;
-        priv = this->private;
-
-        synclock_lock (&priv->big_lock);
-        ret = synctask_new (this->ctx->env, fn, gd_default_synctask_cbk, NULL,
-                            opaque);
-        if (ret)
-                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn bricks"
-                        " and other volume related services");
-}
-
 int
 glusterd_uds_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
                             void *data)