features/bitrot: reimplement scrubbing frequency

This patch reimplments existing scrub-frequency mechanism used
to schedule scrubber runs. Existing mechanism uses periodic
sleeps (waking up periodically on minimum granularity) and
performing a number of tracking checks based on counters and
sleep times. This patch does away with all the nifty counters
and uses timer-wheel to schedule scrub runs.

Scheduling changes are peformed by merely calculating the new
expiry time and calling mod_timer() [mod_timer_pending() in
some cases] making the code more debuggable and easier to
follow. This also introduces "hourly" scrubbing tunable as an
aid for testing scrubbing during development/testing cycle.

One could also implement on-demand scrubbing with ease: by
invoking mod_timer() with an expiry of one (1) second, thereby
scheduling a scrub run the very next second.

Change-Id: I6c7c5f0c6c9f886bf574d88c04cde14b76e60a8b
BUG: 1224596
Signed-off-by: Venky Shankar <vshankar@redhat.com>
Reviewed-on: http://review.gluster.org/10893
Reviewed-by: Gaurav Kumar Garg <ggarg@redhat.com>
Tested-by: NetBSD Build System
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

8 files changed, 322 insertions, 183 deletions

diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index d4c846acd0a..2390822e5df 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -4967,7 +4967,8 @@ cli_cmd_bitrot_parse (const char **words, int wordcount, dict_t **options)
                                                      "scrub", NULL};
         char               *scrub_throt_values[]  = {"lazy", "normal",
                                                      "aggressive", NULL};
-        char               *scrub_freq_values[]   = {"daily", "weekly",
+        char               *scrub_freq_values[]   = {"hourly",
+                                                     "daily", "weekly",
                                                      "biweekly", "monthly",
                                                       NULL};
         char               *scrub_values[]        = {"pause", "resume",
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 4cb82a70719..da6603cc1f6 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -2691,7 +2691,7 @@ struct cli_cmd volume_cmds[] = {
         },
         {"volume bitrot <VOLNAME> {enable|disable} |\n"
          "volume bitrot <volname> scrub-throttle {lazy|normal|aggressive} |\n"
-         "volume bitrot <volname> scrub-frequency {daily|weekly|biweekly"
+         "volume bitrot <volname> scrub-frequency {hourly|daily|weekly|biweekly"
          "|monthly} |\n"
          "volume bitrot <volname> scrub {pause|resume}",
          cli_cmd_bitrot_cbk,
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
index 390148fdb06..26ad97a16e8 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
@@ -508,198 +508,217 @@ br_fsscanner_handle_entry (xlator_t *subvol,
         return -1;
 }
 
-/*scrubber frequency tunable value in second (day * hour * minut * second)*/
-#define DAILY      (1*24*60*60)
-#define WEEKLY     (7*24*60*60)
-#define BIWEEKLY   (14*24*60*60)
-#define MONTHLY    (30*24*60*60)
-
-struct timeval
-br_scrubber_calc_freq (xlator_t *this)
+static inline void
+br_fsscanner_log_time (xlator_t *this, br_child_t *child, const char *sfx)
 {
-        br_private_t    *priv                = NULL;
-        struct          timeval   scrub_sec  = {0,};
+        struct timeval tv = {0,};
+        char timestr[1024] = {0,};
 
-        priv = this->private;
+        gettimeofday (&tv, NULL);
+        gf_time_fmt (timestr, sizeof (timestr), tv.tv_sec, gf_timefmt_FT);
 
-        /*By default scrubber frequency will be biweekly*/
-        if (!strncmp (priv->scrub_freq, "daily", strlen("daily"))) {
-                scrub_sec.tv_sec = DAILY;
-        } else if (!strncmp (priv->scrub_freq, "weekly", strlen("weekly"))) {
-                scrub_sec.tv_sec = WEEKLY;
-        } else if (!strncmp (priv->scrub_freq, "monthly", strlen("monthly"))) {
-                scrub_sec.tv_sec = MONTHLY;
-        } else if (!strncmp (priv->scrub_freq, "biweekly",
-                             strlen("biweekly"))) {
-                scrub_sec.tv_sec = BIWEEKLY;
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid scrub-frequency %s"
-                        "value.", priv->scrub_freq);
-                scrub_sec.tv_sec = -1;
-        }
+        gf_log (this->name, GF_LOG_INFO,
+                "Scrubbing \"%s\" %s at %s", child->brick_path, sfx, timestr);
+}
 
-        return scrub_sec;
+static void
+br_fsscanner_wait_until_kicked (struct br_scanfs *fsscan)
+{
+        pthread_mutex_lock (&fsscan->wakelock);
+        {
+                while (!fsscan->kick)
+                        pthread_cond_wait (&fsscan->wakecond,
+                                           &fsscan->wakelock);
+                fsscan->kick = _gf_false;
+        }
+        pthread_mutex_unlock (&fsscan->wakelock);
 }
 
-#define SCRUBBER_SLEEP(freq_diff, elapse_time) do {                  \
-                                                                     \
-                if (freq_diff < 0) {                                 \
-                        return 0;                                    \
-                } else if (freq_diff <= DAILY) {                     \
-                        gf_log (this->name, GF_LOG_INFO,             \
-                                "Scrubber is sleeping for %ld "      \
-                                "sec", freq_diff);                   \
-                        sleep (freq_diff);                           \
-                        return 0;                                    \
-                } else {                                             \
-                        gf_log (this->name, GF_LOG_INFO,             \
-                                "Scrubber is sleeping for %ld "      \
-                                "sec", freq_diff);                   \
-                        sleep (DAILY);                               \
-                        elapse_time += DAILY;                        \
-                }                                                    \
-        } while (0)
-
-static int
-br_scrubber_sleep_check (struct timeval *begin, struct timeval *end,
-                         xlator_t *this)
+void *
+br_fsscanner (void *arg)
 {
-        br_private_t    *priv                   = NULL;
-        struct           timeval   elapse_time  = {0,};
-        struct           timeval   freq_diff    = {0,};
-        struct           timeval   scrub_sec    = {0,};
-        struct           timeval   temp         = {0,};
+        loc_t               loc     = {0,};
+        br_child_t         *child   = NULL;
+        xlator_t           *this    = NULL;
+        br_private_t       *priv    = NULL;
+        struct br_scanfs   *fsscan  = NULL;
+        struct br_scrubber *fsscrub = NULL;
 
+        child = arg;
+        this = child->this;
         priv = this->private;
 
-        scrub_sec = br_scrubber_calc_freq (this);
-        if (scrub_sec.tv_sec == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to calculate scrub "
-                        "frequency %s value", priv->scrub_freq);
-                return -1;
-        }
+        fsscan = &child->fsscan;
+        fsscrub = &priv->fsscrub;
 
-        if ((end->tv_sec - begin->tv_sec) < scrub_sec.tv_sec) {
-                /* Sleep, if scrubber have completed its job before schedule
-                 * scrub frequency based on current scrub frequency value */
-                do {
-                        scrub_sec = br_scrubber_calc_freq (this);
-                        freq_diff.tv_sec = scrub_sec.tv_sec - (end->tv_sec -
-                                           begin->tv_sec) -
-                                           elapse_time.tv_sec;
-                        SCRUBBER_SLEEP(freq_diff.tv_sec, elapse_time.tv_sec);
-                } while (1);
-
-
-        } else {
-                /* Sleep, if scrubber have completed its job after schedule
-                 * scrub frequency based on current scrub frequency value */
-                temp.tv_sec = (end->tv_sec - begin->tv_sec) % scrub_sec.tv_sec;
-                if (temp.tv_sec != 0) {
-                        do {
-                                scrub_sec = br_scrubber_calc_freq (this);
-                                freq_diff.tv_sec = scrub_sec.tv_sec
-                                                   - temp.tv_sec
-                                                   - elapse_time.tv_sec;
-                                SCRUBBER_SLEEP(freq_diff.tv_sec,
-                                               elapse_time.tv_sec);
-                        } while (1);
+        THIS = this;
+        loc.inode = child->table->root;
+
+        while (1) {
+                br_fsscanner_wait_until_kicked (fsscan);
+                {
+                        /* log start time */
+                        br_fsscanner_log_time (this, child, "started");
+
+                        /* scrub */
+                        (void) syncop_ftw (child->xl,
+                                           &loc, GF_CLIENT_PID_SCRUB,
+                                           child, br_fsscanner_handle_entry);
+                        if (!list_empty (&fsscan->queued))
+                                wait_for_scrubbing (this, fsscan);
+
+                        /* log finish time */
+                        br_fsscanner_log_time (this, child, "finished");
                 }
+                br_fsscan_reschedule (this, child, fsscan, fsscrub, _gf_false);
         }
 
-        return 0;
+        return NULL;
 }
 
-void *
-br_fsscanner (void *arg)
+void
+br_kickstart_scanner (struct gf_tw_timer_list *timer,
+                      void *data, unsigned long calltime)
 {
-        int32_t          ret                          = -1;
-        loc_t            loc                          = {0,};
-        char             timestr[1024]                = {0,};
-        xlator_t         *this                        = NULL;
-        br_child_t       *child                       = NULL;
-        struct br_scanfs *fsscan                      = NULL;
-        br_private_t     *priv                        = NULL;
-        struct           timeval       elapse_time    = {0,};
-        struct           timeval       scrub_sec      = {0,};
-        struct           timeval       freq_diff      = {0,};
+        xlator_t *this = NULL;
+        br_child_t *child = data;
+        struct br_scanfs *fsscan = NULL;
 
-        child = arg;
-        this = child->this;
+        THIS = this = child->this;
         fsscan = &child->fsscan;
 
-        THIS = this;
+        /* kickstart scanning.. */
+        pthread_mutex_lock (&fsscan->wakelock);
+        {
+                fsscan->kick = _gf_true;
+                pthread_cond_signal (&fsscan->wakecond);
+        }
+        pthread_mutex_unlock (&fsscan->wakelock);
+
+        return;
+
+}
+
+static inline uint32_t
+br_fsscan_calculate_delta (uint32_t boot, uint32_t now, uint32_t times)
+{
+        uint32_t secs = 0;
+        uint32_t diff = 0;
+
+        diff = (now - boot);
+        secs = times * ((diff / times) + 1);
+
+        return (secs - diff);
+}
+
+#define BR_SCRUB_HOURLY     (60 * 60)
+#define BR_SCRUB_DAILY      (1 * 24 * 60 * 60)
+#define BR_SCRUB_WEEKLY     (7 * 24 * 60 * 60)
+#define BR_SCRUB_BIWEEKLY   (14 * 24 * 60 * 60)
+#define BR_SCRUB_MONTHLY    (30 * 24 * 60 * 60)
+
+static unsigned int
+br_fsscan_calculate_timeout (uint32_t boot, uint32_t now, scrub_freq_t freq)
+{
+        uint32_t timo = 0;
+
+        switch (freq) {
+        case BR_FSSCRUB_FREQ_HOURLY:
+                timo = br_fsscan_calculate_delta (boot, now, BR_SCRUB_HOURLY);
+                break;
+        case BR_FSSCRUB_FREQ_DAILY:
+                timo = br_fsscan_calculate_delta (boot, now, BR_SCRUB_DAILY);
+                break;
+        case BR_FSSCRUB_FREQ_WEEKLY:
+                timo = br_fsscan_calculate_delta (boot, now, BR_SCRUB_WEEKLY);
+                break;
+        case BR_FSSCRUB_FREQ_BIWEEKLY:
+                timo = br_fsscan_calculate_delta (boot, now, BR_SCRUB_BIWEEKLY);
+                break;
+        case BR_FSSCRUB_FREQ_MONTHLY:
+                timo = br_fsscan_calculate_delta (boot, now, BR_SCRUB_MONTHLY);
+        }
+
+        return timo;
+}
+
+int32_t
+br_fsscan_schedule (xlator_t *this, br_child_t *child,
+                    struct br_scanfs *fsscan, struct br_scrubber *fsscrub)
+{
+        uint32_t timo = 0;
+        br_private_t *priv = NULL;
+        struct timeval tv = {0,};
+        char timestr[1024] = {0,};
+        struct gf_tw_timer_list *timer = NULL;
 
         priv = this->private;
 
-        loc.inode = child->table->root;
+        (void) gettimeofday (&tv, NULL);
+        fsscan->boot = tv.tv_sec;
 
-        /* Scrubber should start scrubbing the filesystem *after* the
-         * schedueled scrub-frequency has expired.*/
-        do {
-                /*Calculate current scrub frequency value in second*/
-                scrub_sec = br_scrubber_calc_freq (this);
-                if (scrub_sec.tv_sec == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to calculate "
-                                "scrub frequency %s value", priv->scrub_freq);
-                        return NULL;
-                }
+        timo = br_fsscan_calculate_timeout (fsscan->boot,
+                                            fsscan->boot, fsscrub->frequency);
 
-                freq_diff.tv_sec = scrub_sec.tv_sec - elapse_time.tv_sec;
+        fsscan->timer = GF_CALLOC (1, sizeof (*fsscan->timer),
+                                   gf_br_stub_mt_br_scanner_freq_t);
+        if (!fsscan->timer)
+                goto error_return;
 
-                if (freq_diff.tv_sec < 0) {
-                        break;
-                } else if (freq_diff.tv_sec == DAILY) {
-                        sleep (DAILY);
-                        break;
-                } else {
-                        sleep (DAILY);
-                        elapse_time.tv_sec += DAILY;
-                }
+        timer = fsscan->timer;
+        INIT_LIST_HEAD (&timer->entry);
 
-        } while (1);
+        timer->data = child;
+        timer->expires = timo;
+        timer->function = br_kickstart_scanner;
+        gf_tw_add_timer (priv->timer_wheel, timer);
 
-        while (1) {
-                /* log scrub start time */
-                gettimeofday (&priv->tv_before_scrub, NULL);
-                gf_time_fmt (timestr, sizeof timestr,
-                             priv->tv_before_scrub.tv_sec, gf_timefmt_FT);
-                gf_log (this->name, GF_LOG_INFO,
-                        "Scrubbing \"%s\" started at %s",
-                        child->brick_path, timestr);
-
-                /* scrub */
-                (void) syncop_ftw (child->xl, &loc,
-                                   GF_CLIENT_PID_SCRUB,
-                                   child, br_fsscanner_handle_entry);
-                if (!list_empty (&fsscan->queued))
-                        wait_for_scrubbing (this, fsscan);
-
-                gettimeofday (&priv->tv_after_scrub, NULL);
-                /* log scrub finish time */
-                gf_time_fmt (timestr, sizeof timestr,
-                             priv->tv_after_scrub.tv_sec, gf_timefmt_FT);
+        gf_time_fmt (timestr, sizeof (timestr),
+                     (fsscan->boot + timo), gf_timefmt_FT);
+        gf_log (this->name, GF_LOG_INFO, "Scrubbing for %s scheduled to "
+                "run at %s", child->brick_path, timestr);
+
+        return 0;
+
+ error_return:
+        return -1;
+}
+
+int32_t
+br_fsscan_reschedule (xlator_t *this,
+                      br_child_t *child, struct br_scanfs *fsscan,
+                      struct br_scrubber *fsscrub, gf_boolean_t pendingcheck)
+{
+        int32_t ret = 0;
+        uint32_t timo = 0;
+        char timestr[1024] = {0,};
+        struct timeval now = {0,};
+        br_private_t *priv = NULL;
+
+        priv = this->private;
+
+        (void) gettimeofday (&now, NULL);
+        timo = br_fsscan_calculate_timeout (fsscan->boot,
+                                            now.tv_sec, fsscrub->frequency);
+
+        gf_time_fmt (timestr, sizeof (timestr),
+                     (now.tv_sec + timo), gf_timefmt_FT);
+
+        if (pendingcheck)
+                ret = gf_tw_mod_timer_pending (priv->timer_wheel,
+                                               fsscan->timer, timo);
+        else
+                ret = gf_tw_mod_timer (priv->timer_wheel, fsscan->timer, timo);
+
+        if (!ret && pendingcheck)
                 gf_log (this->name, GF_LOG_INFO,
-                        "Scrubbing \"%s\" finished at %s",
-                        child->brick_path, timestr);
-
-                /* Scrubber should sleep if it have completed scrubbing
-                 * of filesystem before the scheduled scrub-frequency*/
-                ret = br_scrubber_sleep_check (&priv->tv_before_scrub,
-                                               &priv->tv_after_scrub,
-                                               this);
-                if (!ret) {
-                        gf_log (this->name, GF_LOG_DEBUG, "scrubber is crawling"
-                                " file system with scrubber frequency %s",
-                               priv->scrub_freq);
-                } else {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to perform "
-                                "scrubber sleep check for scrubber frequency");
-                        return NULL;
-                }
-        }
+                        "Scrubber for %s is currently running and would be "
+                        "rescheduled after completion", child->brick_path);
+        else
+                gf_log (this->name, GF_LOG_INFO, "Scrubbing for %s rescheduled "
+                        "to run at %s", child->brick_path, timestr);
 
-        return NULL;
+        return 0;
 }
 
 #define BR_SCRUB_THREAD_SCALE_LAZY       0
@@ -1092,10 +1111,34 @@ static int32_t
 br_scrubber_handle_freq (xlator_t *this, br_private_t *priv, dict_t *options)
 {
         int32_t ret  = -1;
+        char *tmp = NULL;
+        scrub_freq_t frequency = BR_FSSCRUB_FREQ_HOURLY;
+        struct br_scrubber *fsscrub = NULL;
 
-        ret = br_scrubber_fetch_option (this, "scrub-freq", options,
-                                        &priv->scrub_freq);
-        return ret;
+        fsscrub = &priv->fsscrub;
+
+        ret = br_scrubber_fetch_option (this, "scrub-freq", options, &tmp);
+        if (ret)
+                goto error_return;
+
+        if (strcasecmp (tmp, "hourly") == 0) {
+                frequency = BR_FSSCRUB_FREQ_HOURLY;
+        } else if (strcasecmp (tmp, "daily") == 0) {
+                frequency = BR_FSSCRUB_FREQ_DAILY;
+        } else if (strcasecmp (tmp, "weekly") == 0) {
+                frequency = BR_FSSCRUB_FREQ_WEEKLY;
+        } else if (strcasecmp (tmp, "biweekly") == 0) {
+                frequency = BR_FSSCRUB_FREQ_BIWEEKLY;
+        } else if (strcasecmp (tmp, "monthly") == 0) {
+                frequency = BR_FSSCRUB_FREQ_MONTHLY;
+        } else
+                goto error_return;
+
+        fsscrub->frequency = frequency;
+        return 0;
+
+ error_return:
+        return -1;
 }
 
 int32_t
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
index 4f00020d66a..6c4254a397a 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
@@ -16,6 +16,12 @@
 
 void *br_fsscanner (void *);
 
+int32_t br_fsscan_schedule (xlator_t *, br_child_t *,
+                            struct br_scanfs *, struct br_scrubber *);
+int32_t br_fsscan_reschedule (xlator_t *this,
+                              br_child_t *child, struct br_scanfs *,
+                              struct br_scrubber *, gf_boolean_t);
+
 int32_t br_scrubber_handle_options (xlator_t *, br_private_t *, dict_t *);
 
 int32_t br_scrubber_init (xlator_t *, br_private_t *);
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c
index 2652f02b4ea..e7cfe89e1dd 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.c
@@ -1135,6 +1135,11 @@ br_enact_scrubber (xlator_t *this, br_child_t *child)
         INIT_LIST_HEAD (&fsscan->queued);
         INIT_LIST_HEAD (&fsscan->ready);
 
+        /* init scheduler related variables */
+        fsscan->kick = _gf_false;
+        pthread_mutex_init (&fsscan->wakelock, NULL);
+        pthread_cond_init (&fsscan->wakecond, NULL);
+
         ret = gf_thread_create (&child->thread, NULL, br_fsscanner, child);
         if (ret != 0) {
                 gf_log (this->name, GF_LOG_ALERT, "failed to spawn bitrot "
@@ -1142,6 +1147,10 @@ br_enact_scrubber (xlator_t *this, br_child_t *child)
                 goto error_return;
         }
 
+        ret = br_fsscan_schedule (this, child, fsscan, fsscrub);
+        if (ret)
+                goto error_return;
+
         /**
          * Everything has been setup.. add this subvolume to scrubbers
          * list.
@@ -1407,13 +1416,6 @@ br_init_signer (xlator_t *this, br_private_t *priv)
         if (ret)
                 goto out;
 
-        priv->timer_wheel = glusterfs_global_timer_wheel (this);
-        if (!priv->timer_wheel) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "global timer wheel unavailable");
-                goto out;
-        }
-
         pthread_cond_init (&priv->object_cond, NULL);
 
         priv->obj_queue = GF_CALLOC (1, sizeof (*priv->obj_queue),
@@ -1568,6 +1570,13 @@ init (xlator_t *this)
                 INIT_LIST_HEAD (&priv->children[i].list);
         INIT_LIST_HEAD (&priv->bricks);
 
+        priv->timer_wheel = glusterfs_global_timer_wheel (this);
+        if (!priv->timer_wheel) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "global timer wheel unavailable");
+                goto cleanup_mutex;
+        }
+
 	this->private = priv;
 
         if (!priv->iamscrubber) {
@@ -1633,12 +1642,55 @@ fini (xlator_t *this)
 int
 reconfigure (xlator_t *this, dict_t *options)
 {
-        br_private_t *priv = this->private;
+        int i = 0;
+        int32_t ret = -1;
+        br_child_t *child = NULL;
+        br_private_t *priv = NULL;
+        struct br_scanfs *fsscan = NULL;
+        struct br_scrubber *fsscrub = NULL;
+
+        priv = this->private;
 
         if (!priv->iamscrubber)
                 return 0;
 
-        return br_scrubber_handle_options (this, priv, options);
+        ret = br_scrubber_handle_options (this, priv, options);
+        if (ret)
+                goto err;
+
+        fsscrub = &priv->fsscrub;
+
+        /* reschedule all _up_ subvolume(s) */
+        pthread_mutex_lock (&priv->lock);
+        {
+                for (; i < priv->child_count; i++) {
+                        child = &priv->children[i];
+                        if (!child->child_up) {
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "Brick %s is offline, skipping "
+                                        "rescheduling (scrub would auto- "
+                                        "schedule when brick is back online).",
+                                        child->brick_path);
+                                continue;
+                        }
+
+                        fsscan = &child->fsscan;
+                        ret = br_fsscan_reschedule (this, child,
+                                                    fsscan, fsscrub, _gf_true);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Could not "
+                                        "reschedule scrubber for brick: %s. "
+                                        "Scubbing will continue according to "
+                                        "old frequency.", child->brick_path);
+                        }
+                }
+        }
+        pthread_mutex_unlock (&priv->lock);
+
+        return 0;
+
+ err:
+        return -1;
 }
 
 struct xlator_fops fops;
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.h b/xlators/features/bit-rot/src/bitd/bit-rot.h
index 6543be763d6..ec943e9131f 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.h
@@ -46,6 +46,14 @@ typedef enum scrub_throttle {
         BR_SCRUB_THROTTLE_STALLED    = 3,
 } scrub_throttle_t;
 
+typedef enum scrub_freq {
+        BR_FSSCRUB_FREQ_HOURLY = 1,
+        BR_FSSCRUB_FREQ_DAILY,
+        BR_FSSCRUB_FREQ_WEEKLY,
+        BR_FSSCRUB_FREQ_BIWEEKLY,
+        BR_FSSCRUB_FREQ_MONTHLY,
+} scrub_freq_t;
+
 #define signature_size(hl) (sizeof (br_isignature_t) + hl + 1)
 
 struct br_scanfs {
@@ -57,6 +65,15 @@ struct br_scanfs {
         unsigned int     entries;
         struct list_head queued;
         struct list_head ready;
+
+        /* scheduler */
+        uint32_t boot;
+        gf_boolean_t kick;
+
+        pthread_mutex_t wakelock;
+        pthread_cond_t  wakecond;
+
+        struct gf_tw_timer_list *timer;
 };
 
 struct br_child {
@@ -98,13 +115,21 @@ struct br_scrubber {
 
         scrub_throttle_t throttle;
 
+        /**
+         * frequency of scanning for this subvolume. this should
+         * normally be per-child, but since all childs follow the
+         * same frequency for a volume, this option ends up here
+         * instead of br_child_t.
+         */
+        scrub_freq_t frequency;
+
         pthread_mutex_t mutex;
         pthread_cond_t  cond;
 
         unsigned int nr_scrubbers;
         struct list_head scrubbers;
 
-        /*
+        /**
          * list of "rotatable" subvolume(s) undergoing scrubbing
          */
         struct list_head scrublist;
@@ -139,11 +164,6 @@ struct br_private {
 
         gf_boolean_t iamscrubber;         /* function as a fs scrubber */
         struct br_scrubber fsscrub;       /* scrubbers for this subvolume */
-
-        char    *scrub_freq;              /* Scrubber frequency*/
-
-        struct  timeval  tv_before_scrub; /* time before starting scrubbing*/
-        struct  timeval  tv_after_scrub;  /* time after  scrubbing completion*/
 };
 
 typedef struct br_private br_private_t;
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
index 9f6da89032f..504b8ab3635 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
@@ -29,6 +29,7 @@ enum br_mem_types {
         gf_br_mt_br_scrubber_t,
         gf_br_mt_br_fsscan_entry_t,
         gf_br_stub_mt_br_stub_fd_t,
+        gf_br_stub_mt_br_scanner_freq_t,
         gf_br_stub_mt_end,
 };
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c
index 987acb60955..8d1133f9f55 100644
--- a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c
@@ -88,7 +88,23 @@ out:
 int
 glusterd_scrubsvc_start (glusterd_svc_t *svc, int flags)
 {
-        return glusterd_svc_start (svc, flags, NULL);
+        int ret = -1;
+        dict_t *cmdict = NULL;
+
+        cmdict = dict_new ();
+        if (!cmdict)
+                goto error_return;
+
+        ret = dict_set_str (cmdict, "cmdarg0", "--global-timer-wheel");
+        if (ret)
+                goto dealloc_dict;
+
+        ret = glusterd_svc_start (svc, flags, cmdict);
+
+ dealloc_dict:
+        dict_unref (cmdict);
+ error_return:
+        return ret;
 }
 
 int