feature/changelog: Add timeout to changelog barrier.

This patch introduces configurable 'changelog-barrier-timeout'
in changelog. The default changelog-barrier-timeout is set to
120 sec which is same as barrier-timeout of barrier translator.
On this timeout, all the barriered fops are unbarriered. It is
safe to have this internal timeout for changelog barrier to
prevent it from indefinite barrierring of fops if barrier
disable notification does not come from snapshot for some
reason.

Change-Id: Iad07807935f88222cdefcd46312b4b6579186dce
BUG: 1094226
Signed-off-by: Kotresh H R <khiremat@redhat.com>
Reviewed-on: http://review.gluster.org/7669
Reviewed-by: Kaushal M <kaushal@redhat.com>
Reviewed-by: Venky Shankar <vshankar@redhat.com>
Tested-by: Venky Shankar <vshankar@redhat.com>

6 files changed, 108 insertions, 7 deletions

diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index 21f8e3cb26e..2c092745a0a 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -88,6 +88,9 @@ void trap (void);
 #define GF_DAY_IN_SECONDS (24*60*60)
 #define GF_WEEK_IN_SECONDS (7*24*60*60)
 
+/* Default timeout for both barrier and changelog translator */
+#define BARRIER_TIMEOUT "120"
+
 enum _gf_boolean
 {
 	_gf_false = 0,
diff --git a/xlators/features/barrier/src/barrier.c b/xlators/features/barrier/src/barrier.c
index 74973499d56..f2d8d9632ec 100644
--- a/xlators/features/barrier/src/barrier.c
+++ b/xlators/features/barrier/src/barrier.c
@@ -654,7 +654,7 @@ struct volume_options options[] = {
         },
         { .key = {"barrier-timeout"},
           .type = GF_OPTION_TYPE_TIME,
-          .default_value = "120",
+          .default_value = BARRIER_TIMEOUT,
           .description = "After 'timeout' seconds since the time 'barrier' "
                          "option was set to \"on\", acknowledgements to file "
                          "operations are no longer blocked and previously "
diff --git a/xlators/features/changelog/src/changelog-barrier.c b/xlators/features/changelog/src/changelog-barrier.c
index c20eed85b1c..2c237708cf4 100644
--- a/xlators/features/changelog/src/changelog-barrier.c
+++ b/xlators/features/changelog/src/changelog-barrier.c
@@ -52,9 +52,42 @@ chlog_barrier_dequeue_all (xlator_t *this, struct list_head *queue)
 {
         call_stub_t            *stub    = NULL;
 
+        gf_log (this->name, GF_LOG_INFO,
+                "Dequeuing all the changelog barriered fops");
+
         while ((stub = __chlog_barrier_dequeue (this, queue)))
                 call_resume (stub);
 
+        gf_log (this->name, GF_LOG_INFO,
+                "Dequeuing changelog barriered fops is finished");
+        return;
+}
+
+/* Function called on changelog barrier timeout */
+void
+chlog_barrier_timeout (void *data)
+{
+        xlator_t               *this    = NULL;
+        changelog_priv_t       *priv    = NULL;
+        struct list_head        queue   = {0,};
+
+        this = data;
+        THIS = this;
+        priv = this->private;
+
+        INIT_LIST_HEAD (&queue);
+
+        gf_log (this->name, GF_LOG_ERROR,
+                "Disabling changelog barrier because of the timeout.");
+
+        LOCK (&priv->lock);
+        {
+                __chlog_barrier_disable (this, &queue);
+        }
+        UNLOCK (&priv->lock);
+
+        chlog_barrier_dequeue_all (this, &queue);
+
         return;
 }
 
@@ -63,8 +96,35 @@ void
 __chlog_barrier_disable (xlator_t *this, struct list_head *queue)
 {
         changelog_priv_t  *priv   = this->private;
+        int                ret    = 0;
+        GF_ASSERT (priv);
+
+        if (priv->timer) {
+                ret = gf_timer_call_cancel (this->ctx, priv->timer);
+                priv->timer = NULL;
+        }
 
         list_splice_init (&priv->queue, queue);
         priv->queue_size = 0;
         priv->barrier_enabled = _gf_false;
 }
+
+/* Enable chagelog barrier enable with timer */
+int
+__chlog_barrier_enable (xlator_t *this, changelog_priv_t *priv)
+{
+        int             ret     = -1;
+
+        priv->timer = gf_timer_call_after (this->ctx, priv->timeout,
+                                           chlog_barrier_timeout, (void *)this);
+        if (!priv->timer) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Couldn't add changelog barrier timeout event.");
+                goto out;
+        }
+
+        priv->barrier_enabled = _gf_true;
+        ret = 0;
+out:
+        return ret;
+}
diff --git a/xlators/features/changelog/src/changelog-helpers.h b/xlators/features/changelog/src/changelog-helpers.h
index 987af190b9c..03108d36f7d 100644
--- a/xlators/features/changelog/src/changelog-helpers.h
+++ b/xlators/features/changelog/src/changelog-helpers.h
@@ -280,6 +280,9 @@ struct changelog_priv {
         gf_boolean_t      barrier_enabled;
         struct list_head  queue;
         uint32_t          queue_size;
+        gf_timer_t       *timer;
+        struct timespec   timeout;
+
 };
 
 struct changelog_local {
@@ -433,6 +436,8 @@ void __chlog_barrier_enqueue (xlator_t *this, call_stub_t *stub);
 void __chlog_barrier_disable (xlator_t *this, struct list_head *queue);
 void chlog_barrier_dequeue_all (xlator_t *this, struct list_head *queue);
 call_stub_t *__chlog_barrier_dequeue (xlator_t *this, struct list_head *queue);
+int __chlog_barrier_enable (xlator_t *this, changelog_priv_t *priv);
+
 
 /* macros */
 
@@ -563,6 +568,6 @@ call_stub_t *__chlog_barrier_dequeue (xlator_t *this, struct list_head *queue);
                         goto label;                                            \
                 }                                                              \
         } while (0)
-/*  End: Geo-Rep snapshot dependency changes */
+/* End: Geo-Rep snapshot dependency changes */
 
 #endif /* _CHANGELOG_HELPERS_H */
diff --git a/xlators/features/changelog/src/changelog.c b/xlators/features/changelog/src/changelog.c
index 0b982148f44..4ed7cc7f0e9 100644
--- a/xlators/features/changelog/src/changelog.c
+++ b/xlators/features/changelog/src/changelog.c
@@ -1281,6 +1281,16 @@ changelog_assign_encoding (changelog_priv_t *priv, char *enc)
         }
 }
 
+static void
+changelog_assign_barrier_timeout(changelog_priv_t *priv, uint32_t timeout)
+{
+       LOCK (&priv->lock);
+       {
+               priv->timeout.tv_sec = timeout;
+       }
+       UNLOCK (&priv->lock);
+}
+
 /* cleanup any helper threads that are running */
 static void
 changelog_cleanup_helper_threads (xlator_t *this, changelog_priv_t *priv)
@@ -1444,9 +1454,7 @@ notify (xlator_t *this, int event, void *data, ...)
         if (event == GF_EVENT_TRANSLATOR_OP) {
 
                 dict = data;
-                /*TODO: Also barrier option is persistent. Need to
-                 *      decide on the brick crash scenarios.
-                 */
+
                 barrier = dict_get_str_boolean (dict, "barrier", DICT_DEFAULT);
 
                 switch (barrier) {
@@ -1540,9 +1548,14 @@ notify (xlator_t *this, int event, void *data, ...)
                         /* Start changelog barrier */
                         LOCK (&priv->lock);
                         {
-                                priv->barrier_enabled = _gf_true;
+                                ret = __chlog_barrier_enable (this, priv);
                         }
                         UNLOCK (&priv->lock);
+                        if (ret == -1) {
+                                changelog_barrier_cleanup (this, priv, &queue);
+                                goto out;
+                        }
+
                         gf_log(this->name, GF_LOG_DEBUG,
                                            "Enabled changelog barrier");
 
@@ -1786,6 +1799,7 @@ reconfigure (xlator_t *this, dict_t *options)
         changelog_log_data_t    cld            = {0,};
         char    htime_dir[PATH_MAX]            = {0,};
         struct timeval          tv             = {0,};
+        uint32_t                timeout        = 0;
 
         priv = this->private;
         if (!priv)
@@ -1840,6 +1854,9 @@ reconfigure (xlator_t *this, dict_t *options)
                           priv->rollover_time, options, int32, out);
         GF_OPTION_RECONF ("fsync-interval",
                           priv->fsync_interval, options, int32, out);
+        GF_OPTION_RECONF ("changelog-barrier-timeout",
+                          timeout, options, time, out);
+        changelog_assign_barrier_timeout (priv, timeout);
 
         if (active_now || active_earlier) {
                 ret = changelog_fill_rollover_data (&cld, !active_now);
@@ -1898,6 +1915,7 @@ init (xlator_t *this)
         changelog_priv_t        *priv                   = NULL;
         gf_boolean_t            cond_lock_init          = _gf_false;
         char                    htime_dir[PATH_MAX]     = {0,};
+        uint32_t                timeout                 = 0;
 
         GF_VALIDATE_OR_GOTO ("changelog", this, out);
 
@@ -1977,6 +1995,8 @@ init (xlator_t *this)
         GF_OPTION_INIT ("rollover-time", priv->rollover_time, int32, out);
 
         GF_OPTION_INIT ("fsync-interval", priv->fsync_interval, int32, out);
+        GF_OPTION_INIT ("changelog-barrier-timeout", timeout, time, out);
+        priv->timeout.tv_sec = timeout;
 
         changelog_encode_change(priv);
 
@@ -2127,6 +2147,14 @@ struct volume_options options[] = {
          .description = "do not open CHANGELOG file with O_SYNC mode."
                         " instead perform fsync() at specified intervals"
         },
+        { .key = {"changelog-barrier-timeout"},
+          .type = GF_OPTION_TYPE_TIME,
+          .default_value = BARRIER_TIMEOUT,
+          .description = "After 'timeout' seconds since the time 'barrier' "
+                         "option was set to \"on\", unlink/rmdir/rename  "
+                         "operations are no longer blocked and previously "
+                         "blocked fops are allowed to go through"
+        },
         {.key = {NULL}
         },
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 75ec0026ff9..f94c5bca40c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -1602,6 +1602,11 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .type        = NO_DOC,
           .op_version  = 3
         },
+        { .key         = "changelog.changelog-barrier-timeout",
+          .voltype     = "features/changelog",
+          .value       = BARRIER_TIMEOUT,
+          .op_version  = GD_OP_VERSION_3_6_0,
+        },
         { .key         = "features.barrier",
           .voltype     = "features/barrier",
           .value       = "disable",
@@ -1609,7 +1614,7 @@ struct volopt_map_entry glusterd_volopt_map[] = {
         },
         { .key         = "features.barrier-timeout",
           .voltype     = "features/barrier",
-          .value       = "120",
+          .value       = BARRIER_TIMEOUT,
           .op_version  = GD_OP_VERSION_3_6_0,
         },
         { .key         = "cluster.op-version",