glusterd: Run post hooks on a different thread

This change ensures post hooks can 'wait' if need be
and _not_ prevent glusterd from being able to run other operations
meanwhile.

Also ensures that post hook scripts are 'serialized'
between transactions. ie, post hook scripts of txn1 are completed
before post hook scripts of txn2 are started, where txn1 happens before
txn2.

Change-Id: Iaeb676737d8c67e7151127c8d1fd8c2891e10aee
BUG: 806996
Signed-off-by: Krishnan Parthasarathi <kp@gluster.com>
Reviewed-on: http://review.gluster.com/3450
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Tested-by: Amar Tumballi <amarts@redhat.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

6 files changed, 238 insertions, 6 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.c b/xlators/mgmt/glusterd/src/glusterd-hooks.c
index b0c4c2638..ab8e07005 100644
--- a/xlators/mgmt/glusterd/src/glusterd-hooks.c
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.c
@@ -358,3 +358,171 @@ out:
         return ret;
 }
 
+int
+glusterd_hooks_post_stub_enqueue (char *scriptdir, glusterd_op_t op,
+                                  dict_t *op_ctx)
+{
+        int                     ret     = -1;
+        glusterd_hooks_stub_t   *stub   = NULL;
+        glusterd_hooks_private_t *hooks_priv = NULL;
+        glusterd_conf_t         *conf   = NULL;
+
+        conf = THIS->private;
+        hooks_priv = conf->hooks_priv;
+
+        ret = glusterd_hooks_stub_init (&stub, scriptdir, op, op_ctx);
+        if (ret)
+                goto out;
+
+        pthread_mutex_lock (&hooks_priv->mutex);
+        {
+                hooks_priv->waitcount++;
+                list_add_tail (&stub->all_hooks, &hooks_priv->list);
+                pthread_cond_signal (&hooks_priv->cond);
+        }
+        pthread_mutex_unlock (&hooks_priv->mutex);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_hooks_stub_init (glusterd_hooks_stub_t **stub, char *scriptdir,
+                          glusterd_op_t op, dict_t *op_ctx)
+{
+        int                     ret             = -1;
+        glusterd_hooks_stub_t   *hooks_stub     = NULL;
+
+        GF_ASSERT (stub);
+        if (!stub)
+                goto out;
+
+        hooks_stub = GF_CALLOC (1, sizeof (*hooks_stub),
+                                gf_gld_mt_hooks_stub_t);
+        if (!hooks_stub)
+                goto out;
+
+        INIT_LIST_HEAD (&hooks_stub->all_hooks);
+        hooks_stub->op = op;
+        hooks_stub->scriptdir = gf_strdup (scriptdir);
+        if (!hooks_stub->scriptdir)
+                goto out;
+
+        hooks_stub->op_ctx = dict_copy_with_ref (op_ctx, hooks_stub->op_ctx);
+        if (!hooks_stub->op_ctx)
+                goto out;
+
+        *stub = hooks_stub;
+        ret = 0;
+out:
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to initialize "
+                        "post hooks stub");
+                glusterd_hooks_stub_cleanup (hooks_stub);
+        }
+
+        return ret;
+}
+
+void
+glusterd_hooks_stub_cleanup (glusterd_hooks_stub_t *stub)
+{
+        if (!stub) {
+                gf_log_callingfn (THIS->name, GF_LOG_WARNING,
+                                  "hooks_stub is NULL");
+                return;
+        }
+
+        if (stub->op_ctx)
+                dict_unref (stub->op_ctx);
+
+        if (stub->scriptdir)
+                GF_FREE (stub->scriptdir);
+
+        if (stub)
+                GF_FREE (stub);
+}
+
+static void*
+hooks_worker (void *args)
+{
+        glusterd_conf_t *conf = NULL;
+        glusterd_hooks_private_t *hooks_priv = NULL;
+        glusterd_hooks_stub_t *stub = NULL;
+
+        THIS = args;
+        conf = THIS->private;
+        hooks_priv = conf->hooks_priv;
+
+        for (;;) {
+                pthread_mutex_lock (&hooks_priv->mutex);
+                {
+                        while (list_empty (&hooks_priv->list)) {
+                                pthread_cond_wait (&hooks_priv->cond,
+                                                   &hooks_priv->mutex);
+                        }
+                        stub = list_entry (hooks_priv->list.next,
+                                           glusterd_hooks_stub_t,
+                                           all_hooks);
+                        list_del_init (&stub->all_hooks);
+                        hooks_priv->waitcount--;
+
+                }
+                pthread_mutex_unlock (&hooks_priv->mutex);
+
+                glusterd_hooks_run_hooks (stub->scriptdir, stub->op,
+                                          stub->op_ctx, GD_COMMIT_HOOK_POST);
+                glusterd_hooks_stub_cleanup (stub);
+        }
+
+        return NULL;
+}
+
+int
+glusterd_hooks_priv_init (glusterd_hooks_private_t **new)
+{
+        int                      ret            = -1;
+        glusterd_hooks_private_t *hooks_priv    = NULL;
+
+        if (!new)
+                goto out;
+
+        hooks_priv = GF_CALLOC (1, sizeof (*hooks_priv),
+                                gf_gld_mt_hooks_priv_t);
+        if (!hooks_priv)
+                goto out;
+
+        pthread_mutex_init (&hooks_priv->mutex, NULL);
+        pthread_cond_init (&hooks_priv->cond, NULL);
+        INIT_LIST_HEAD (&hooks_priv->list);
+        hooks_priv->waitcount = 0;
+
+        *new = hooks_priv;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_hooks_spawn_worker (xlator_t *this)
+{
+        int                       ret          = -1;
+        glusterd_conf_t           *conf        = NULL;
+        glusterd_hooks_private_t  *hooks_priv  = NULL;
+
+
+        ret = glusterd_hooks_priv_init (&hooks_priv);
+        if (ret)
+                goto out;
+
+        conf = this->private;
+        conf->hooks_priv = hooks_priv;
+        ret = pthread_create (&hooks_priv->worker, NULL, hooks_worker,
+                              (void *)this);
+        if (ret)
+                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn post "
+                        "hooks worker thread");
+out:
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.h b/xlators/mgmt/glusterd/src/glusterd-hooks.h
index 02d9e4331..120fd5a47 100644
--- a/xlators/mgmt/glusterd/src/glusterd-hooks.h
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.h
@@ -38,6 +38,23 @@ typedef enum glusterd_commit_hook_type {
         GD_COMMIT_HOOK_MAX
 } glusterd_commit_hook_type_t;
 
+typedef struct hooks_private {
+        struct list_head        list;
+        int                     waitcount; //debug purposes
+        pthread_mutex_t         mutex;
+        pthread_cond_t          cond;
+        pthread_t               worker;
+} glusterd_hooks_private_t;
+
+typedef struct hooks_stub {
+        struct list_head        all_hooks;
+        char                    *scriptdir;
+        glusterd_op_t           op;
+        dict_t                  *op_ctx;
+
+} glusterd_hooks_stub_t;
+
+
 int
 glusterd_hooks_create_hooks_directory (char *basedir);
 
@@ -47,4 +64,18 @@ glusterd_hooks_get_hooks_cmd_subdir (glusterd_op_t op);
 int
 glusterd_hooks_run_hooks (char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
                           glusterd_commit_hook_type_t type);
+int
+glusterd_hooks_spawn_worker (xlator_t *this);
+
+int
+glusterd_hooks_stub_init (glusterd_hooks_stub_t **stub, char *scriptdir,
+                          glusterd_op_t op, dict_t *op_ctx);
+void
+glusterd_hooks_stub_cleanup (glusterd_hooks_stub_t *stub);
+
+int
+glusterd_hooks_post_stub_enqueue (char *scriptdir, glusterd_op_t op,
+                                      dict_t *op_ctx);
+int
+glusterd_hooks_priv_init (glusterd_hooks_private_t **new);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
index 0ef904c7c..9031f83a5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
@@ -73,7 +73,9 @@ typedef enum gf_gld_mem_types_ {
         gf_gld_mt_georep_meet_spec              = gf_common_mt_end + 47,
         gf_gld_mt_nodesrv_t                     = gf_common_mt_end + 48,
         gf_gld_mt_charptr                       = gf_common_mt_end + 49,
-        gf_gld_mt_end                           = gf_common_mt_end + 50,
+        gf_gld_mt_hooks_stub_t                  = gf_common_mt_end + 50,
+        gf_gld_mt_hooks_priv_t                  = gf_common_mt_end + 51,
+        gf_gld_mt_end                           = gf_common_mt_end + 52,
 } gf_gld_mem_types_t;
 #endif
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index a507c76d7..f727a76eb 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -2271,12 +2271,22 @@ glusterd_op_commit_hook (glusterd_op_t op, dict_t *op_ctx,  glusterd_commit_hook
         char            scriptdir[PATH_MAX]     = {0, };
         char            type_subdir[256]        = {0, };
         char            *cmd_subdir             = NULL;
+        int             ret                     = -1;
 
         priv = THIS->private;
-        if (type == GD_COMMIT_HOOK_PRE)
-                strcpy (type_subdir, "pre");
-        else if (type == GD_COMMIT_HOOK_POST)
-                strcpy (type_subdir, "post");
+        switch (type) {
+                case GD_COMMIT_HOOK_NONE:
+                case GD_COMMIT_HOOK_MAX:
+                        /*Won't be called*/
+                        break;
+
+                case GD_COMMIT_HOOK_PRE:
+                        strcpy (type_subdir, "pre");
+                        break;
+                case GD_COMMIT_HOOK_POST:
+                        strcpy (type_subdir, "post");
+                        break;
+        }
 
         cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir (op);
         if (strlen (cmd_subdir) == 0)
@@ -2286,7 +2296,23 @@ glusterd_op_commit_hook (glusterd_op_t op, dict_t *op_ctx,  glusterd_commit_hook
         snprintf (scriptdir, sizeof (scriptdir), "%s/%s/%s",
                   hookdir, cmd_subdir, type_subdir);
 
-        return glusterd_hooks_run_hooks (scriptdir, op, op_ctx, type);
+        switch (type) {
+                case GD_COMMIT_HOOK_NONE:
+                case GD_COMMIT_HOOK_MAX:
+                        /*Won't be called*/
+                        break;
+
+                case GD_COMMIT_HOOK_PRE:
+                        ret = glusterd_hooks_run_hooks (scriptdir, op, op_ctx,
+                                                        type);
+                        break;
+                case GD_COMMIT_HOOK_POST:
+                        ret = glusterd_hooks_post_stub_enqueue (scriptdir, op,
+                                                                op_ctx);
+                        break;
+        }
+
+        return ret;
 }
 
 static int
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 58890985f..5f7298c34 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1022,6 +1022,10 @@ init (xlator_t *this)
         if (ret)
                 goto out;
 
+        ret = glusterd_hooks_spawn_worker (this);
+        if (ret)
+                goto out;
+
         glusterd_restart_rebalance (conf);
         ret = 0;
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 1ba922a80..1397d9224 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -125,6 +125,7 @@ typedef struct {
         gf_boolean_t      valgrind;
 #endif
         pthread_t       brick_thread;
+        void           *hooks_priv;
         xlator_t       *xl;  /* Should be set to 'THIS' before creating thread */
 } glusterd_conf_t;