features/bitrot: Introduce scrubber monitor thread

The patch does following changes.

 1. Introduce scrubber monitor thread.
 2. Move scrub status related APIs to separate file
    and make part of libbitrot library.

Problem:
     Earlier, each child of the scrubber was maintaining
the state machine and hence there was no way to track
the start and end time of scrubbing as each brick has
it's own start and end time. Also each brick was maintaining
it's own timer wheel instance. It was also not possible
to get scrubbed files count per session as we could not
get last child which finishes scrubbing to reset it to
zero.

Solution:
    Introduce scrubber monitor thread. It does following.

 1. Maintains the scrubber state machine. Earlier each
    child had it's own state machine. Now, only monitor
    maintains on behalf of all it's children.
 2. Maintains the timer wheel instance. Earlier each
    child had it's own timer wheel instance. Now, only
    monitor maintains on behalf of all it's children.

As a result, we can track the scrub statistics easily
and correctly.

Change-Id: Ic6e34ffa57984bd7a5ee81f4e263342bc1d9b302
BUG: 1329211
Signed-off-by: Kotresh HR <khiremat@redhat.com>
Reviewed-on: http://review.gluster.org/14044
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Venky Shankar <vshankar@redhat.com>

10 files changed, 690 insertions, 317 deletions

diff --git a/xlators/features/bit-rot/src/bitd/Makefile.am b/xlators/features/bit-rot/src/bitd/Makefile.am
index 6557943ac69..a915f2d34b8 100644
--- a/xlators/features/bit-rot/src/bitd/Makefile.am
+++ b/xlators/features/bit-rot/src/bitd/Makefile.am
@@ -9,11 +9,13 @@ AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
                  -I$(CONTRIBDIR)/timer-wheel \
                  -I$(top_srcdir)/xlators/features/bit-rot/src/stub
 
-bit_rot_la_SOURCES = bit-rot.c bit-rot-scrub.c bit-rot-tbf.c bit-rot-ssm.c
+bit_rot_la_SOURCES = bit-rot.c bit-rot-scrub.c bit-rot-tbf.c bit-rot-ssm.c \
+		     bit-rot-scrub-status.c
 bit_rot_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
                     $(top_builddir)/xlators/features/changelog/lib/src/libgfchangelog.la
 
-noinst_HEADERS = bit-rot.h bit-rot-scrub.h bit-rot-tbf.h bit-rot-bitd-messages.h bit-rot-ssm.h
+noinst_HEADERS = bit-rot.h bit-rot-scrub.h bit-rot-tbf.h bit-rot-bitd-messages.h bit-rot-ssm.h \
+		 bit-rot-scrub-status.h
 
 AM_CFLAGS = -Wall -DBR_RATE_LIMIT_SIGNER $(GF_CFLAGS)
 
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h b/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h
index c0b83c681d7..c6b6a4afa05 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h
@@ -40,7 +40,7 @@
  */
 
 #define GLFS_BITROT_BITD_BASE                   GLFS_MSGID_COMP_BITROT_BITD
-#define GLFS_BITROT_BITD_NUM_MESSAGES           53
+#define GLFS_BITROT_BITD_NUM_MESSAGES           55
 #define GLFS_MSGID_END                          (GLFS_BITROT_BITD_BASE + \
                                            GLFS_BITROT_BITD_NUM_MESSAGES + 1)
 /* Messaged with message IDs */
@@ -427,6 +427,22 @@
  *
  */
 /*------------*/
+#define BRB_MSG_SSM_FAILED                 (GLFS_BITROT_BITD_BASE + 54)
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+/*------------*/
+#define BRB_MSG_SCRUB_WAIT_FAILED          (GLFS_BITROT_BITD_BASE + 55)
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+/*------------*/
 
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 #endif /* !_BITROT_BITD_MESSAGES_H_ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c
new file mode 100644
index 00000000000..0afd7ea05b1
--- /dev/null
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c
@@ -0,0 +1,73 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <string.h>
+
+#include "bit-rot-scrub-status.h"
+
+void
+br_inc_unsigned_file_count (br_scrub_stats_t *scrub_stat)
+{
+        if (!scrub_stat)
+                return;
+
+        pthread_mutex_lock (&scrub_stat->lock);
+        {
+                scrub_stat->unsigned_files++;
+        }
+        pthread_mutex_unlock (&scrub_stat->lock);
+}
+
+void
+br_inc_scrubbed_file (br_scrub_stats_t *scrub_stat)
+{
+        if (!scrub_stat)
+                return;
+
+        pthread_mutex_lock (&scrub_stat->lock);
+        {
+                scrub_stat->scrubbed_files++;
+        }
+        pthread_mutex_unlock (&scrub_stat->lock);
+}
+
+void
+br_update_scrub_start_time (br_scrub_stats_t *scrub_stat, struct timeval *tv)
+{
+        if (!scrub_stat)
+                return;
+
+        pthread_mutex_lock (&scrub_stat->lock);
+        {
+                scrub_stat->scrub_start_tv.tv_sec = tv->tv_sec;
+        }
+        pthread_mutex_unlock (&scrub_stat->lock);
+}
+
+void
+br_update_scrub_finish_time (br_scrub_stats_t *scrub_stat, char *timestr,
+                             struct timeval *tv)
+{
+        if (!scrub_stat)
+                return;
+
+        pthread_mutex_lock (&scrub_stat->lock);
+        {
+                scrub_stat->scrub_end_tv.tv_sec = tv->tv_sec;
+
+                scrub_stat->scrub_duration =
+                                 scrub_stat->scrub_end_tv.tv_sec -
+                                 scrub_stat->scrub_start_tv.tv_sec;
+
+                strncpy (scrub_stat->last_scrub_time, timestr,
+                         sizeof (scrub_stat->last_scrub_time));
+        }
+        pthread_mutex_unlock (&scrub_stat->lock);
+}
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h
new file mode 100644
index 00000000000..694ba0acbe3
--- /dev/null
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h
@@ -0,0 +1,46 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __BIT_ROT_SCRUB_STATUS_H__
+#define __BIT_ROT_SCRUB_STATUS_H__
+
+#include <stdint.h>
+#include <sys/time.h>
+#include <pthread.h>
+
+struct br_scrub_stats {
+        uint64_t       scrubbed_files;       /* Total number of scrubbed file */
+
+        uint64_t       unsigned_files;       /* Total number of unsigned file */
+
+        uint64_t       scrub_duration;            /* Duration of last scrub */
+
+        char           last_scrub_time[1024];    /*last scrub completion time */
+
+        struct         timeval scrub_start_tv;   /* Scrubbing starting time*/
+
+        struct         timeval scrub_end_tv;     /* Scrubbing finishing time */
+
+        pthread_mutex_t  lock;
+};
+
+typedef struct br_scrub_stats br_scrub_stats_t;
+
+void
+br_inc_unsigned_file_count (br_scrub_stats_t *scrub_stat);
+void
+br_inc_scrubbed_file (br_scrub_stats_t *scrub_stat);
+void
+br_update_scrub_start_time (br_scrub_stats_t *scrub_stat, struct timeval *tv);
+void
+br_update_scrub_finish_time (br_scrub_stats_t *scrub_stat, char *timestr,
+                             struct timeval *tv);
+
+#endif /* __BIT_ROT_SCRUB_STATUS_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
index d95008ca9a2..c0421d0ac02 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
@@ -19,6 +19,7 @@
 #include "bit-rot-scrub.h"
 #include <pthread.h>
 #include "bit-rot-bitd-messages.h"
+#include "bit-rot-scrub-status.h"
 
 struct br_scrubbers {
         pthread_t scrubthread;
@@ -74,20 +75,6 @@ bitd_fetch_signature (xlator_t *this, br_child_t *child,
 
 }
 
-static void
-br_inc_unsigned_file_count (xlator_t *this)
-{
-        br_private_t   *priv = NULL;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->scrub_stat.lock);
-        {
-                priv->scrub_stat.unsigned_files++;
-        }
-        pthread_mutex_unlock (&priv->scrub_stat.lock);
-}
-
 /**
  * POST COMPUTE CHECK
  *
@@ -101,7 +88,8 @@ int32_t
 bitd_scrub_post_compute_check (xlator_t *this,
                                br_child_t *child,
                                fd_t *fd, unsigned long version,
-                               br_isignature_out_t **signature)
+                               br_isignature_out_t **signature,
+                               br_scrub_stats_t *scrub_stat)
 {
         int32_t              ret     = 0;
         size_t               signlen = 0;
@@ -109,8 +97,10 @@ bitd_scrub_post_compute_check (xlator_t *this,
         br_isignature_out_t *signptr = NULL;
 
         ret = bitd_fetch_signature (this, child, fd, &xattr, &signptr);
-        if (ret < 0)
+        if (ret < 0) {
+                br_inc_unsigned_file_count (scrub_stat);
                 goto out;
+        }
 
         /**
          * Either the object got dirtied during the time the signature was
@@ -121,7 +111,7 @@ bitd_scrub_post_compute_check (xlator_t *this,
          * The log entry looks pretty ugly, but helps in debugging..
          */
         if (signptr->stale || (signptr->version != version)) {
-                br_inc_unsigned_file_count (this);
+                br_inc_unsigned_file_count (scrub_stat);
                 gf_msg_debug (this->name, 0, "<STAGE: POST> Object [GFID: %s] "
                               "either has a stale signature OR underwent "
                               "signing during checksumming {Stale: %d | "
@@ -149,15 +139,18 @@ bitd_scrub_post_compute_check (xlator_t *this,
 static int32_t
 bitd_signature_staleness (xlator_t *this,
                           br_child_t *child, fd_t *fd,
-                          int *stale, unsigned long *version)
+                          int *stale, unsigned long *version,
+                          br_scrub_stats_t *scrub_stat)
 {
         int32_t ret = -1;
         dict_t *xattr = NULL;
         br_isignature_out_t *signptr = NULL;
 
         ret = bitd_fetch_signature (this, child, fd, &xattr, &signptr);
-        if (ret < 0)
+        if (ret < 0) {
+                br_inc_unsigned_file_count (scrub_stat);
                 goto out;
+        }
 
         /**
          * save verison for validation in post compute stage
@@ -182,7 +175,8 @@ bitd_signature_staleness (xlator_t *this,
  */
 int32_t
 bitd_scrub_pre_compute_check (xlator_t *this, br_child_t *child,
-                              fd_t *fd, unsigned long *version)
+                              fd_t *fd, unsigned long *version,
+                              br_scrub_stats_t *scrub_stat)
 {
         int     stale = 0;
         int32_t ret   = -1;
@@ -194,9 +188,10 @@ bitd_scrub_pre_compute_check (xlator_t *this, br_child_t *child,
                 goto out;
         }
 
-        ret = bitd_signature_staleness (this, child, fd, &stale, version);
+        ret = bitd_signature_staleness (this, child, fd, &stale, version,
+                                        scrub_stat);
         if (!ret && stale) {
-                br_inc_unsigned_file_count (this);
+                br_inc_unsigned_file_count (scrub_stat);
                 gf_msg_debug (this->name, 0, "<STAGE: PRE> Object [GFID: %s] "
                               "has stale signature",
                               uuid_utoa (fd->inode->gfid));
@@ -272,16 +267,6 @@ bitd_compare_ckum (xlator_t *this,
         return ret;
 }
 
-static void
-br_inc_scrubbed_file (br_private_t *priv)
-{
-        pthread_mutex_lock (&priv->scrub_stat.lock);
-        {
-                priv->scrub_stat.scrubbed_files++;
-        }
-        pthread_mutex_unlock (&priv->scrub_stat.lock);
-}
-
 /**
  * "The Scrubber"
  *
@@ -374,7 +359,8 @@ br_scrubber_scrub_begin (xlator_t *this, struct br_fsscan_entry *fsentry)
          *  - presence of bad object
          *  - signature staleness
          */
-        ret = bitd_scrub_pre_compute_check (this, child, fd, &signedversion);
+        ret = bitd_scrub_pre_compute_check (this, child, fd, &signedversion,
+                                            &priv->scrub_stat);
         if (ret)
                 goto unrefd; /* skip this object */
 
@@ -397,8 +383,8 @@ br_scrubber_scrub_begin (xlator_t *this, struct br_fsscan_entry *fsentry)
          * perform post compute checks as an object's signature may have
          * become stale while scrubber calculated checksum.
          */
-        ret = bitd_scrub_post_compute_check (this, child,
-                                             fd, signedversion, &sign);
+        ret = bitd_scrub_post_compute_check (this, child, fd, signedversion,
+                                             &sign, &priv->scrub_stat);
         if (ret)
                 goto free_md;
 
@@ -406,7 +392,7 @@ br_scrubber_scrub_begin (xlator_t *this, struct br_fsscan_entry *fsentry)
                                  linked_inode, entry, fd, child, &loc);
 
         /* Increment of total number of scrubbed file counter */
-        br_inc_scrubbed_file (priv);
+        br_inc_scrubbed_file (&priv->scrub_stat);
 
         GF_FREE (sign); /* alloced on post-compute */
 
@@ -560,171 +546,215 @@ br_fsscanner_handle_entry (xlator_t *subvol,
 }
 
 int32_t
-br_fsscan_deactivate (xlator_t *this, br_child_t *child)
+br_fsscan_deactivate (xlator_t *this)
 {
         int ret = 0;
         br_private_t *priv = NULL;
         br_scrub_state_t nstate = 0;
-        struct br_scanfs *fsscan = NULL;
+        struct br_monitor *scrub_monitor = NULL;
 
         priv = this->private;
-        fsscan = &child->fsscan;
+        scrub_monitor = &priv->scrub_monitor;
 
-        ret = gf_tw_del_timer (priv->timer_wheel, fsscan->timer);
+        ret = gf_tw_del_timer (priv->timer_wheel, scrub_monitor->timer);
         if (ret == 0) {
                 nstate = BR_SCRUB_STATE_STALLED;
                 gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Brick [%s] is under active scrubbing. Pausing scrub..",
-                        child->brick_path);
+                        "Volume is under active scrubbing. Pausing scrub..");
         } else {
                 nstate = BR_SCRUB_STATE_PAUSED;
                 gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Scrubber paused [Brick: %s]", child->brick_path);
+                        "Scrubber paused");
         }
 
-        _br_child_set_scrub_state (child, nstate);
+        _br_monitor_set_scrub_state (scrub_monitor, nstate);
 
         return 0;
 }
+
 static void
-br_update_scrub_start_time (xlator_t *this, struct timeval *tv)
+br_scrubber_log_time (xlator_t *this, const char *sfx)
 {
-        br_private_t     *priv = NULL;
-        static int       child;
+        char           timestr[1024] = {0,};
+        struct         timeval tv    = {0,};
+        br_private_t  *priv          = NULL;
 
         priv = this->private;
 
+        gettimeofday (&tv, NULL);
+        gf_time_fmt (timestr, sizeof (timestr), tv.tv_sec, gf_timefmt_FT);
 
-        /* Setting scrubber starting time for first child only */
-        if (child == 0) {
-                pthread_mutex_lock (&priv->scrub_stat.lock);
-                {
-                        priv->scrub_stat.scrub_start_tv.tv_sec = tv->tv_sec;
-                }
-                pthread_mutex_unlock (&priv->scrub_stat.lock);
+        if (strcasecmp (sfx, "started") == 0) {
+                br_update_scrub_start_time (&priv->scrub_stat, &tv);
+                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_START,
+                        "Scrubbing %s at %s", sfx, timestr);
+        } else {
+                br_update_scrub_finish_time (&priv->scrub_stat, timestr, &tv);
+                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_FINISH,
+                        "Scrubbing %s at %s", sfx, timestr);
         }
+}
 
-        if (++child == priv->up_children) {
-                child = 0;
+static void
+br_fsscanner_log_time (xlator_t *this, br_child_t *child, const char *sfx)
+{
+        char           timestr[1024] = {0,};
+        struct         timeval tv    = {0,};
+
+        gettimeofday (&tv, NULL);
+        gf_time_fmt (timestr, sizeof (timestr), tv.tv_sec, gf_timefmt_FT);
+
+        if (strcasecmp (sfx, "started") == 0) {
+                gf_msg_debug (this->name, 0, "Scrubbing \"%s\" %s at %s",
+                              child->brick_path, sfx, timestr);
+        } else {
+                gf_msg_debug (this->name, 0, "Scrubbing \"%s\" %s at %s",
+                              child->brick_path, sfx, timestr);
         }
 }
 
+void
+br_child_set_scrub_state (br_child_t *child, gf_boolean_t state)
+{
+        child->active_scrubbing = state;
+}
+
 static void
-br_update_scrub_finish_time (xlator_t *this, char *timestr, struct timeval *tv)
+br_fsscanner_wait_until_kicked (xlator_t *this, br_child_t *child)
 {
-        br_private_t     *priv = NULL;
-        static int       child;
+        br_private_t      *priv          = NULL;
+        struct br_monitor *scrub_monitor = NULL;
 
         priv = this->private;
+        scrub_monitor = &priv->scrub_monitor;
 
-        /*Setting scrubber finishing time at time time of last child operation*/
-        if (++child == priv->up_children) {
-                pthread_mutex_lock (&priv->scrub_stat.lock);
-                {
-                        priv->scrub_stat.scrub_end_tv.tv_sec = tv->tv_sec;
-
-                        priv->scrub_stat.scrub_duration =
-                                         priv->scrub_stat.scrub_end_tv.tv_sec -
-                                         priv->scrub_stat.scrub_start_tv.tv_sec;
-
-                        strncpy (priv->scrub_stat.last_scrub_time, timestr,
-                                 sizeof (priv->scrub_stat.last_scrub_time));
+        pthread_cleanup_push (_br_lock_cleaner, &scrub_monitor->wakelock);
+        pthread_mutex_lock (&scrub_monitor->wakelock);
+        {
+                while (!scrub_monitor->kick)
+                        pthread_cond_wait (&scrub_monitor->wakecond,
+                                           &scrub_monitor->wakelock);
 
-                        child = 0;
+                /* Child lock is to synchronize with disconnect events */
+                pthread_cleanup_push (_br_lock_cleaner, &child->lock);
+                LOCK (&child->lock);
+                {
+                        scrub_monitor->active_child_count++;
+                        br_child_set_scrub_state (child, _gf_true);
                 }
-                pthread_mutex_unlock (&priv->scrub_stat.lock);
+                UNLOCK (&child->lock);
+                pthread_cleanup_pop (0);
         }
+        pthread_mutex_unlock (&scrub_monitor->wakelock);
+        pthread_cleanup_pop (0);
 }
 
 static void
-br_fsscanner_log_time (xlator_t *this, br_child_t *child, const char *sfx)
+br_scrubber_entry_control (xlator_t *this)
 {
-        char           timestr[1024] = {0,};
-        struct         timeval tv    = {0,};
+        br_private_t      *priv          = NULL;
+        struct br_monitor *scrub_monitor = NULL;
 
-        gettimeofday (&tv, NULL);
-        gf_time_fmt (timestr, sizeof (timestr), tv.tv_sec, gf_timefmt_FT);
+        priv = this->private;
+        scrub_monitor = &priv->scrub_monitor;
 
-        if (strcasecmp (sfx, "started") == 0) {
-                br_update_scrub_start_time (this, &tv);
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_START,
-                        "Scrubbing \"%s\" %s at %s", child->brick_path, sfx,
-                        timestr);
-        } else {
-                br_update_scrub_finish_time (this, timestr, &tv);
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_FINISH,
-                        "Scrubbing \"%s\" %s at %s", child->brick_path, sfx,
-                        timestr);
+        LOCK (&scrub_monitor->lock);
+        {
+                /* Move the state to BR_SCRUB_STATE_ACTIVE */
+                if (scrub_monitor->state == BR_SCRUB_STATE_PENDING)
+                        scrub_monitor->state = BR_SCRUB_STATE_ACTIVE;
+                br_scrubber_log_time (this, "started");
         }
+        UNLOCK (&scrub_monitor->lock);
 }
 
 static void
-br_fsscanner_wait_until_kicked (xlator_t *this, struct br_scanfs *fsscan)
+br_scrubber_exit_control (xlator_t *this)
 {
-        static int            i;
-        br_private_t         *priv    = NULL;
+        br_private_t      *priv          = NULL;
+        struct br_monitor *scrub_monitor = NULL;
 
         priv = this->private;
+        scrub_monitor = &priv->scrub_monitor;
 
-        pthread_cleanup_push (_br_lock_cleaner, &fsscan->wakelock);
-        pthread_mutex_lock (&fsscan->wakelock);
+        LOCK (&scrub_monitor->lock);
         {
-                while (!fsscan->kick)
-                        pthread_cond_wait (&fsscan->wakecond,
-                                           &fsscan->wakelock);
-
-                /* resetting total number of scrubbed file when scrubbing
-                 * done for all of its children */
-                if (i == priv->up_children) {
-                        pthread_mutex_lock (&priv->scrub_stat.lock);
-                        {
-                                priv->scrub_stat.scrubbed_files = 0;
-                                priv->scrub_stat.unsigned_files = 0;
-                                i = 0;
-                        }
-                        pthread_mutex_unlock (&priv->scrub_stat.lock);
-                }
-                ++i;
+                br_scrubber_log_time (this, "finished");
 
-                fsscan->kick = _gf_false;
+                if (scrub_monitor->state == BR_SCRUB_STATE_ACTIVE) {
+                        (void) br_fsscan_activate (this);
+                } else {
+                        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+                                "Volume waiting to get rescheduled..");
+                }
         }
-        pthread_mutex_unlock (&fsscan->wakelock);
-        pthread_cleanup_pop (0);
+        UNLOCK (&scrub_monitor->lock);
 }
 
 static void
 br_fsscanner_entry_control (xlator_t *this, br_child_t *child)
 {
-        struct br_scanfs *fsscan = &child->fsscan;
-
-        LOCK (&child->lock);
-        {
-                if (fsscan->state == BR_SCRUB_STATE_PENDING)
-                        fsscan->state = BR_SCRUB_STATE_ACTIVE;
                 br_fsscanner_log_time (this, child, "started");
-        }
-        UNLOCK (&child->lock);
 }
 
 static void
 br_fsscanner_exit_control (xlator_t *this, br_child_t *child)
 {
-        struct br_scanfs *fsscan = &child->fsscan;
+        br_private_t *priv = NULL;
+        struct br_monitor *scrub_monitor = NULL;
 
-        LOCK (&child->lock);
+        priv = this->private;
+        scrub_monitor = &priv->scrub_monitor;
+
+        if (!_br_is_child_connected (child)) {
+                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_SCRUB_INFO,
+                        "Brick [%s] disconnected while scrubbing. Scrubbing "
+                        "might be incomplete", child->brick_path);
+        }
+
+        br_fsscanner_log_time (this, child, "finished");
+
+        pthread_cleanup_push (_br_lock_cleaner, &scrub_monitor->wakelock);
+        pthread_mutex_lock (&scrub_monitor->wakelock);
         {
-                fsscan->over = _gf_true;
-                br_fsscanner_log_time (this, child, "finished");
+                scrub_monitor->active_child_count--;
+                pthread_cleanup_push (_br_lock_cleaner, &child->lock);
+                LOCK (&child->lock);
+                {
+                        br_child_set_scrub_state (child, _gf_false);
+                }
+                UNLOCK (&child->lock);
+                pthread_cleanup_pop (0);
 
-                if (fsscan->state == BR_SCRUB_STATE_ACTIVE) {
-                        (void) br_fsscan_activate (this, child);
+                if (scrub_monitor->active_child_count == 0) {
+                        /* The last child has finished scrubbing.
+                         * Set the kick to false and  wake up other
+                         * children who are waiting for the last
+                         * child to complete scrubbing.
+                         */
+                        scrub_monitor->kick = _gf_false;
+                        pthread_cond_broadcast (&scrub_monitor->wakecond);
+
+                        /* Signal monitor thread waiting for the all
+                         * the children to finish scrubbing.
+                         */
+                        pthread_cleanup_push (_br_lock_cleaner,
+                                              &scrub_monitor->donelock);
+                        pthread_mutex_lock (&scrub_monitor->donelock);
+                        {
+                                scrub_monitor->done = _gf_true;
+                                pthread_cond_signal (&scrub_monitor->donecond);
+                        }
+                        pthread_mutex_unlock (&scrub_monitor->donelock);
+                        pthread_cleanup_pop (0);
                 } else {
-                        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                                "Brick [%s] waiting to get rescheduled..",
-                                child->brick_path);
+                        while (scrub_monitor->active_child_count)
+                                pthread_cond_wait (&scrub_monitor->wakecond,
+                                                   &scrub_monitor->wakelock);
                 }
         }
-        UNLOCK (&child->lock);
+        pthread_mutex_unlock (&scrub_monitor->wakelock);
+        pthread_cleanup_pop (0);
 }
 
 void *
@@ -743,7 +773,7 @@ br_fsscanner (void *arg)
         loc.inode = child->table->root;
 
         while (1) {
-                br_fsscanner_wait_until_kicked (this, fsscan);
+                br_fsscanner_wait_until_kicked (this, child);
                 {
                         /* precursor for scrub */
                         br_fsscanner_entry_control (this, child);
@@ -775,22 +805,29 @@ br_kickstart_scanner (struct gf_tw_timer_list *timer,
                       void *data, unsigned long calltime)
 {
         xlator_t *this = NULL;
-        br_child_t *child = data;
-        struct br_scanfs *fsscan = NULL;
+        struct br_monitor *scrub_monitor = data;
+        br_private_t *priv = NULL;
 
-        THIS = this = child->this;
-        fsscan = &child->fsscan;
+        THIS = this = scrub_monitor->this;
+        priv = this->private;
+
+        /* Reset scrub statistics */
+        priv->scrub_stat.scrubbed_files = 0;
+        priv->scrub_stat.unsigned_files = 0;
+
+        /* Moves state from PENDING to ACTIVE */
+        (void) br_scrubber_entry_control (this);
 
         /* kickstart scanning.. */
-        pthread_mutex_lock (&fsscan->wakelock);
+        pthread_mutex_lock (&scrub_monitor->wakelock);
         {
-                fsscan->kick = _gf_true;
-                pthread_cond_signal (&fsscan->wakecond);
+                scrub_monitor->kick = _gf_true;
+                GF_ASSERT (scrub_monitor->active_child_count == 0);
+                pthread_cond_broadcast (&scrub_monitor->wakecond);
         }
-        pthread_mutex_unlock (&fsscan->wakelock);
+        pthread_mutex_unlock (&scrub_monitor->wakelock);
 
         return;
-
 }
 
 static uint32_t
@@ -834,22 +871,22 @@ br_fsscan_calculate_timeout (scrub_freq_t freq)
 }
 
 int32_t
-br_fsscan_schedule (xlator_t *this, br_child_t *child)
+br_fsscan_schedule (xlator_t *this)
 {
         uint32_t timo = 0;
         br_private_t *priv = NULL;
         struct timeval tv = {0,};
         char timestr[1024] = {0,};
-        struct br_scanfs *fsscan = NULL;
         struct br_scrubber *fsscrub = NULL;
         struct gf_tw_timer_list *timer = NULL;
+        struct br_monitor *scrub_monitor = NULL;
 
         priv = this->private;
-        fsscan = &child->fsscan;
         fsscrub = &priv->fsscrub;
+        scrub_monitor = &priv->scrub_monitor;
 
         (void) gettimeofday (&tv, NULL);
-        fsscan->boot = tv.tv_sec;
+        scrub_monitor->boot = tv.tv_sec;
 
         timo = br_fsscan_calculate_timeout (fsscrub->frequency);
         if (timo == 0) {
@@ -858,25 +895,25 @@ br_fsscan_schedule (xlator_t *this, br_child_t *child)
                 goto error_return;
         }
 
-        fsscan->timer = GF_CALLOC (1, sizeof (*fsscan->timer),
+        scrub_monitor->timer = GF_CALLOC (1, sizeof (*scrub_monitor->timer),
                                    gf_br_stub_mt_br_scanner_freq_t);
-        if (!fsscan->timer)
+        if (!scrub_monitor->timer)
                 goto error_return;
 
-        timer = fsscan->timer;
+        timer = scrub_monitor->timer;
         INIT_LIST_HEAD (&timer->entry);
 
-        timer->data = child;
+        timer->data = scrub_monitor;
         timer->expires = timo;
         timer->function = br_kickstart_scanner;
 
         gf_tw_add_timer (priv->timer_wheel, timer);
-        _br_child_set_scrub_state (child, BR_SCRUB_STATE_PENDING);
+        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PENDING);
 
         gf_time_fmt (timestr, sizeof (timestr),
-                     (fsscan->boot + timo), gf_timefmt_FT);
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, "Scrubbing for "
-                "%s scheduled to run at %s", child->brick_path, timestr);
+                     (scrub_monitor->boot + timo), gf_timefmt_FT);
+        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, "Scrubbing is "
+                "scheduled to run at %s", timestr);
 
         return 0;
 
@@ -885,18 +922,18 @@ br_fsscan_schedule (xlator_t *this, br_child_t *child)
 }
 
 int32_t
-br_fsscan_activate (xlator_t *this, br_child_t *child)
+br_fsscan_activate (xlator_t *this)
 {
         uint32_t            timo    = 0;
         char timestr[1024]          = {0,};
         struct timeval      now     = {0,};
         br_private_t       *priv    = NULL;
-        struct br_scanfs   *fsscan  = NULL;
         struct br_scrubber *fsscrub = NULL;
+        struct br_monitor  *scrub_monitor = NULL;
 
         priv = this->private;
-        fsscan = &child->fsscan;
         fsscrub = &priv->fsscrub;
+        scrub_monitor = &priv->scrub_monitor;
 
         (void) gettimeofday (&now, NULL);
         timo = br_fsscan_calculate_timeout (fsscrub->frequency);
@@ -906,32 +943,37 @@ br_fsscan_activate (xlator_t *this, br_child_t *child)
                 return -1;
         }
 
-        fsscan->over = _gf_false;
+        pthread_mutex_lock (&scrub_monitor->donelock);
+        {
+                scrub_monitor->done = _gf_false;
+        }
+        pthread_mutex_unlock (&scrub_monitor->donelock);
+
         gf_time_fmt (timestr, sizeof (timestr),
                      (now.tv_sec + timo), gf_timefmt_FT);
-        (void) gf_tw_mod_timer (priv->timer_wheel, fsscan->timer, timo);
+        (void) gf_tw_mod_timer (priv->timer_wheel, scrub_monitor->timer, timo);
 
-        _br_child_set_scrub_state (child, BR_SCRUB_STATE_PENDING);
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, "Scrubbing for "
-                "%s rescheduled to run at %s", child->brick_path, timestr);
+        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PENDING);
+        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, "Scrubbing is "
+                "rescheduled to run at %s", timestr);
 
         return 0;
 }
 
 int32_t
-br_fsscan_reschedule (xlator_t *this, br_child_t *child)
+br_fsscan_reschedule (xlator_t *this)
 {
         int32_t             ret     = 0;
         uint32_t            timo    = 0;
         char timestr[1024]          = {0,};
         struct timeval      now     = {0,};
         br_private_t       *priv    = NULL;
-        struct br_scanfs   *fsscan  = NULL;
         struct br_scrubber *fsscrub = NULL;
+        struct br_monitor  *scrub_monitor = NULL;
 
         priv = this->private;
-        fsscan = &child->fsscan;
         fsscrub = &priv->fsscrub;
+        scrub_monitor = &priv->scrub_monitor;
 
         if (!fsscrub->frequency_reconf)
                 return 0;
@@ -947,17 +989,21 @@ br_fsscan_reschedule (xlator_t *this, br_child_t *child)
         gf_time_fmt (timestr, sizeof (timestr),
                      (now.tv_sec + timo), gf_timefmt_FT);
 
-        fsscan->over = _gf_false;
-        ret = gf_tw_mod_timer_pending (priv->timer_wheel, fsscan->timer, timo);
+        pthread_mutex_lock (&scrub_monitor->donelock);
+        {
+                scrub_monitor->done = _gf_false;
+        }
+        pthread_mutex_unlock (&scrub_monitor->donelock);
+
+        ret = gf_tw_mod_timer_pending (priv->timer_wheel, scrub_monitor->timer, timo);
         if (ret == 0)
                 gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Scrubber for %s is currently running and would be "
-                        "rescheduled after completion", child->brick_path);
+                        "Scrubber is currently running and would be "
+                        "rescheduled after completion");
         else {
-                _br_child_set_scrub_state (child, BR_SCRUB_STATE_PENDING);
+                _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PENDING);
                 gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Scrubbing for %s rescheduled to run at %s",
-                        child->brick_path, timestr);
+                        "Scrubbing rescheduled to run at %s", timestr);
         }
 
         return 0;
@@ -1723,15 +1769,174 @@ out:
         return ret;
 }
 
+static int
+wait_for_scrub_to_finish (xlator_t *this)
+{
+        int                  ret               = -1;
+        br_private_t         *priv             = NULL;
+        struct br_monitor    *scrub_monitor    = NULL;
+
+        priv = this->private;
+        scrub_monitor = &priv->scrub_monitor;
+
+        GF_VALIDATE_OR_GOTO ("bit-rot", scrub_monitor, out);
+        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
+
+        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+                "Waiting for all children to start and finish scrub");
+
+        pthread_mutex_lock (&scrub_monitor->donelock);
+        {
+                while (!scrub_monitor->done)
+                        pthread_cond_wait (&scrub_monitor->donecond,
+                                           &scrub_monitor->donelock);
+        }
+        pthread_mutex_unlock (&scrub_monitor->donelock);
+        ret = 0;
+out:
+        return ret;
+}
+
+/**
+ * This function is executed in a separate thread. This is scrubber monitor
+ * thread that takes care of state machine.
+ */
+void *
+br_monitor_thread (void *arg)
+{
+        int32_t              ret               = 0;
+        xlator_t            *this              = NULL;
+        br_private_t        *priv              = NULL;
+        struct br_monitor   *scrub_monitor     = NULL;
+
+        this = arg;
+        priv = this->private;
+
+        /*
+         * Since, this is the topmost xlator, THIS has to be set by bit-rot
+         * xlator itself (STACK_WIND wont help in this case). Also it has
+         * to be done for each thread that gets spawned. Otherwise, a new
+         * thread will get global_xlator's pointer when it does "THIS".
+         */
+        THIS = this;
+
+        scrub_monitor = &priv->scrub_monitor;
+
+        pthread_mutex_lock (&scrub_monitor->mutex);
+        {
+                while (!scrub_monitor->inited)
+                        pthread_cond_wait (&scrub_monitor->cond,
+                                           &scrub_monitor->mutex);
+        }
+        pthread_mutex_unlock (&scrub_monitor->mutex);
+
+        /* this needs to be serialized with reconfigure() */
+        pthread_mutex_lock (&priv->lock);
+        {
+                ret = br_scrub_state_machine (this);
+        }
+        pthread_mutex_unlock (&priv->lock);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, -ret,
+                        BRB_MSG_SSM_FAILED,
+                        "Scrub state machine failed");
+                goto out;
+        }
+
+        while (1) {
+                /* Wait for all children to finish scrubbing */
+                ret = wait_for_scrub_to_finish (this);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, -ret,
+                                BRB_MSG_SCRUB_WAIT_FAILED,
+                                "Scrub wait failed");
+                        goto out;
+                }
+
+                /* scrub exit criteria: Move the state to PENDING */
+                br_scrubber_exit_control (this);
+        }
+
+out:
+        return NULL;
+}
+
+static void
+br_set_scrub_state (struct br_monitor *scrub_monitor, br_scrub_state_t state)
+{
+        LOCK (&scrub_monitor->lock);
+        {
+                _br_monitor_set_scrub_state (scrub_monitor, state);
+        }
+        UNLOCK (&scrub_monitor->lock);
+}
+
+int32_t
+br_scrubber_monitor_init (xlator_t *this, br_private_t *priv)
+{
+        struct br_monitor *scrub_monitor = NULL;
+        int                ret           = 0;
+
+        scrub_monitor = &priv->scrub_monitor;
+
+        LOCK_INIT (&scrub_monitor->lock);
+        scrub_monitor->this = this;
+
+        scrub_monitor->inited = _gf_false;
+        pthread_mutex_init (&scrub_monitor->mutex, NULL);
+        pthread_cond_init (&scrub_monitor->cond, NULL);
+
+        scrub_monitor->kick = _gf_false;
+        scrub_monitor->active_child_count = 0;
+        pthread_mutex_init (&scrub_monitor->wakelock, NULL);
+        pthread_cond_init (&scrub_monitor->wakecond, NULL);
+
+        scrub_monitor->done = _gf_false;
+        pthread_mutex_init (&scrub_monitor->donelock, NULL);
+        pthread_cond_init (&scrub_monitor->donecond, NULL);
+
+        /* Set the state to INACTIVE */
+        br_set_scrub_state (&priv->scrub_monitor, BR_SCRUB_STATE_INACTIVE);
+
+        /* Start the monitor thread */
+        ret = gf_thread_create (&scrub_monitor->thread, NULL, br_monitor_thread, this);
+        if (ret != 0) {
+                gf_msg (this->name, GF_LOG_ERROR, -ret,
+                        BRB_MSG_SPAWN_FAILED, "monitor thread creation failed");
+                ret = -1;
+                goto err;
+        }
+
+        return 0;
+err:
+        pthread_mutex_destroy (&scrub_monitor->mutex);
+        pthread_cond_destroy (&scrub_monitor->cond);
+
+        pthread_mutex_destroy (&scrub_monitor->wakelock);
+        pthread_cond_destroy (&scrub_monitor->wakecond);
+
+        pthread_mutex_destroy (&scrub_monitor->donelock);
+        pthread_cond_destroy (&scrub_monitor->donecond);
+
+        LOCK_DESTROY (&scrub_monitor->lock);
+
+        return ret;
+}
+
 int32_t
 br_scrubber_init (xlator_t *this, br_private_t *priv)
 {
         struct br_scrubber *fsscrub = NULL;
+        int                 ret     = 0;
 
         priv->tbf = br_tbf_init (NULL, 0);
         if (!priv->tbf)
                 return -1;
 
+        ret = br_scrubber_monitor_init (this, priv);
+        if (ret)
+                return -1;
+
         fsscrub = &priv->fsscrub;
 
         fsscrub->this = this;
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
index b27decd41c1..63169068ed4 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
@@ -16,15 +16,21 @@
 
 void *br_fsscanner (void *);
 
-int32_t br_fsscan_schedule (xlator_t *, br_child_t *);
-int32_t br_fsscan_reschedule (xlator_t *, br_child_t *);
-int32_t br_fsscan_activate (xlator_t *, br_child_t *);
-int32_t br_fsscan_deactivate (xlator_t *, br_child_t *);
+int32_t br_fsscan_schedule (xlator_t *);
+int32_t br_fsscan_reschedule (xlator_t *);
+int32_t br_fsscan_activate (xlator_t *);
+int32_t br_fsscan_deactivate (xlator_t *);
 
 int32_t br_scrubber_handle_options (xlator_t *, br_private_t *, dict_t *);
 
+int32_t
+br_scrubber_monitor_init (xlator_t *, br_private_t *);
+
 int32_t br_scrubber_init (xlator_t *, br_private_t *);
 
 int32_t br_collect_bad_objects_from_children (xlator_t *this, dict_t *dict);
 
+void
+br_child_set_scrub_state (br_child_t *, gf_boolean_t);
+
 #endif /* __BIT_ROT_SCRUB_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
index fcffc04feda..d304fc804ee 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
@@ -12,52 +12,73 @@
 #include "bit-rot-scrub.h"
 #include "bit-rot-bitd-messages.h"
 
-int br_scrub_ssm_noop (xlator_t *this, br_child_t *child)
+int br_scrub_ssm_noop (xlator_t *this)
 {
         return 0;
 }
 
 int
-br_scrub_ssm_state_pause (xlator_t *this, br_child_t *child)
+br_scrub_ssm_state_pause (xlator_t *this)
 {
+        br_private_t        *priv               = NULL;
+        struct br_monitor   *scrub_monitor      = NULL;
+
+        priv = this->private;
+        scrub_monitor = &priv->scrub_monitor;
+
         gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
-                "Scrubber paused [Brick: %s]", child->brick_path);
-        _br_child_set_scrub_state (child, BR_SCRUB_STATE_PAUSED);
+                "Scrubber paused");
+        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PAUSED);
         return 0;
 }
 
 int
-br_scrub_ssm_state_ipause (xlator_t *this, br_child_t *child)
+br_scrub_ssm_state_ipause (xlator_t *this)
 {
+        br_private_t        *priv               = NULL;
+        struct br_monitor   *scrub_monitor      = NULL;
+
+        priv = this->private;
+        scrub_monitor = &priv->scrub_monitor;
+
         gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
-                "Scrubber paused [Brick: %s]", child->brick_path);
-        _br_child_set_scrub_state (child, BR_SCRUB_STATE_IPAUSED);
+                "Scrubber paused");
+        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_IPAUSED);
         return 0;
 }
 
 int
-br_scrub_ssm_state_active (xlator_t *this, br_child_t *child)
+br_scrub_ssm_state_active (xlator_t *this)
 {
-        struct br_scanfs *fsscan = &child->fsscan;
+        br_private_t        *priv               = NULL;
+        struct br_monitor   *scrub_monitor      = NULL;
 
-        if (fsscan->over) {
-                (void) br_fsscan_activate (this, child);
+        priv = this->private;
+        scrub_monitor = &priv->scrub_monitor;
+
+        if (scrub_monitor->done) {
+                (void) br_fsscan_activate (this);
         } else {
                 gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
-                        "Scrubbing resumed [Brick %s]", child->brick_path);
-                _br_child_set_scrub_state (child, BR_SCRUB_STATE_ACTIVE);
+                        "Scrubbing resumed");
+                _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_ACTIVE);
         }
 
         return 0;
 }
 
 int
-br_scrub_ssm_state_stall (xlator_t *this, br_child_t *child)
+br_scrub_ssm_state_stall (xlator_t *this)
 {
+        br_private_t        *priv               = NULL;
+        struct br_monitor   *scrub_monitor      = NULL;
+
+        priv = this->private;
+        scrub_monitor = &priv->scrub_monitor;
+
         gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
-                "Brick [%s] is under active scrubbing. Pausing scrub..",
-                child->brick_path);
-        _br_child_set_scrub_state (child, BR_SCRUB_STATE_STALLED);
+                "Volume is under active scrubbing. Pausing scrub..");
+        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_STALLED);
         return 0;
 }
 
@@ -72,22 +93,22 @@ br_scrub_ssm[BR_SCRUB_MAXSTATES][BR_SCRUB_MAXEVENTS] = {
 };
 
 int32_t
-br_scrub_state_machine (xlator_t *this, br_child_t *child)
+br_scrub_state_machine (xlator_t *this)
 {
         br_private_t       *priv      = NULL;
         br_scrub_ssm_call  *call      = NULL;
-        struct br_scanfs   *fsscan    = NULL;
         struct br_scrubber *fsscrub   = NULL;
         br_scrub_state_t    currstate = 0;
         br_scrub_event_t    event     = 0;
+        struct br_monitor  *scrub_monitor = NULL;
 
         priv = this->private;
-        fsscan = &child->fsscan;
         fsscrub = &priv->fsscrub;
+        scrub_monitor = &priv->scrub_monitor;
 
-        currstate = fsscan->state;
+        currstate = scrub_monitor->state;
         event = _br_child_get_scrub_event (fsscrub);
 
         call = br_scrub_ssm[currstate][event];
-        return call (this, child);
+        return call (this);
 }
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
index 72fd62b3630..936ee4d837c 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
@@ -29,8 +29,8 @@ typedef enum br_scrub_event {
         BR_SCRUB_MAXEVENTS,
 } br_scrub_event_t;
 
-struct br_child;
+struct br_monitor;
 
-int32_t br_scrub_state_machine (xlator_t *, struct br_child *);
+int32_t br_scrub_state_machine (xlator_t *);
 
 #endif /* __BIT_ROT_SSM_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c
index e4d9113bd25..7d337bf724f 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.c
@@ -1099,16 +1099,6 @@ br_set_child_state (br_child_t *child, br_child_state_t state)
         UNLOCK (&child->lock);
 }
 
-static void
-br_set_scrub_state (br_child_t *child, br_scrub_state_t state)
-{
-        LOCK (&child->lock);
-        {
-                _br_child_set_scrub_state (child, state);
-        }
-        UNLOCK (&child->lock);
-}
-
 /**
  * At this point a thread is spawned to crawl the filesystem (in
  * tortoise pace) to sign objects that were not signed in previous run(s).
@@ -1168,11 +1158,11 @@ br_launch_scrubber (xlator_t *this, br_child_t *child,
 {
         int32_t ret = -1;
         br_private_t *priv = NULL;
+        struct br_monitor *scrub_monitor = NULL;
 
         priv = this->private;
 
-        fsscan->kick = _gf_false;
-        fsscan->over = _gf_false;
+        scrub_monitor = &priv->scrub_monitor;
         ret = gf_thread_create (&child->thread, NULL, br_fsscanner, child);
         if (ret != 0) {
                 gf_msg (this->name, GF_LOG_ALERT, 0, BRB_MSG_SPAWN_FAILED,
@@ -1181,14 +1171,14 @@ br_launch_scrubber (xlator_t *this, br_child_t *child,
                 goto error_return;
         }
 
-        /* this needs to be serialized with reconfigure() */
-        pthread_mutex_lock (&priv->lock);
+        /* Signal monitor to kick off state machine*/
+        pthread_mutex_lock (&scrub_monitor->mutex);
         {
-                ret = br_scrub_state_machine (this, child);
+                if (!scrub_monitor->inited)
+                        pthread_cond_signal (&scrub_monitor->cond);
+                scrub_monitor->inited = _gf_true;
         }
-        pthread_mutex_unlock (&priv->lock);
-        if (ret)
-                goto cleanup_thread;
+        pthread_mutex_unlock (&scrub_monitor->mutex);
 
         /**
          * Everything has been setup.. add this subvolume to scrubbers
@@ -1203,8 +1193,6 @@ br_launch_scrubber (xlator_t *this, br_child_t *child,
 
         return 0;
 
- cleanup_thread:
-        (void) gf_thread_cleanup_xint (child->thread);
  error_return:
         return -1;
 }
@@ -1237,10 +1225,6 @@ br_enact_scrubber (xlator_t *this, br_child_t *child)
         INIT_LIST_HEAD (&fsscan->queued);
         INIT_LIST_HEAD (&fsscan->ready);
 
-        /* init scheduler related variables */
-        pthread_mutex_init (&fsscan->wakelock, NULL);
-        pthread_cond_init (&fsscan->wakecond, NULL);
-
         ret = br_launch_scrubber (this, child, fsscan, fsscrub);
         if (ret)
                 goto error_return;
@@ -1303,6 +1287,7 @@ br_brick_connect (xlator_t *this, br_child_t *child)
         GF_VALIDATE_OR_GOTO (this->name, child, out);
         GF_VALIDATE_OR_GOTO (this->name, this->private, out);
 
+        br_child_set_scrub_state (child, _gf_false);
         br_set_child_state (child, BR_CHILD_STATE_INITIALIZING);
 
         loc.inode = inode_ref (child->table->root);
@@ -1364,12 +1349,17 @@ br_cleanup_scrubber (xlator_t *this, br_child_t *child)
 {
         int32_t ret = 0;
         br_private_t *priv = NULL;
-        struct br_scanfs *fsscan = NULL;
         struct br_scrubber *fsscrub = NULL;
+        struct br_monitor *scrub_monitor = NULL;
 
         priv    = this->private;
-        fsscan  = &child->fsscan;
         fsscrub = &priv->fsscrub;
+        scrub_monitor = &priv->scrub_monitor;
+
+        if (_br_is_child_scrub_active (child)) {
+                scrub_monitor->active_child_count--;
+                br_child_set_scrub_state (child, _gf_false);
+        }
 
         /**
          * 0x0: child (brick) goes out of rotation
@@ -1401,21 +1391,6 @@ br_cleanup_scrubber (xlator_t *this, br_child_t *child)
                         0, BRB_MSG_SCRUB_THREAD_CLEANUP,
                         "Error cleaning up scanner thread");
 
-        /**
-         * 0x2: free()up resources
-         */
-        if (fsscan->timer) {
-                (void) gf_tw_del_timer (priv->timer_wheel, fsscan->timer);
-
-                GF_FREE (fsscan->timer);
-                fsscan->timer = NULL;
-        }
-
-        /**
-         * 0x3: reset scrubber state
-         */
-        _br_child_set_scrub_state (child, BR_SCRUB_STATE_INACTIVE);
-
         gf_msg (this->name, GF_LOG_INFO,
                 0, BRB_MSG_SCRUBBER_CLEANED,
                 "Cleaned up scrubber for brick [%s]", child->brick_path);
@@ -1432,23 +1407,33 @@ int32_t
 br_brick_disconnect (xlator_t *this, br_child_t *child)
 {
         int32_t ret = 0;
+        struct br_monitor *scrub_monitor = NULL;
         br_private_t *priv = this->private;
 
-        LOCK (&child->lock);
+        scrub_monitor = &priv->scrub_monitor;
+
+        /* Lock order should be wakelock and then child lock to
+         * dead locks.
+         */
+        pthread_mutex_lock (&scrub_monitor->wakelock);
         {
-                if (!_br_is_child_connected (child))
-                        goto unblock;
+                LOCK (&child->lock);
+                {
+                        if (!_br_is_child_connected (child))
+                                goto unblock;
 
-                /* child is on death row.. */
-                _br_set_child_state (child, BR_CHILD_STATE_DISCONNECTED);
+                        /* child is on death row.. */
+                        _br_set_child_state (child, BR_CHILD_STATE_DISCONNECTED);
 
-                if (priv->iamscrubber)
-                        ret = br_cleanup_scrubber (this, child);
-                else
-                        ret = br_cleanup_signer (this, child);
-        }
+                        if (priv->iamscrubber)
+                                ret = br_cleanup_scrubber (this, child);
+                        else
+                                ret = br_cleanup_signer (this, child);
+                }
  unblock:
-        UNLOCK (&child->lock);
+                UNLOCK (&child->lock);
+        }
+        pthread_mutex_unlock (&scrub_monitor->wakelock);
 
          return ret;
 }
@@ -1569,7 +1554,7 @@ br_scrubber_status_get (xlator_t *this, dict_t **dict)
 
         memset (key, 0, 256);
         snprintf (key, 256, "scrubbed-files");
-        ret = dict_set_uint32 (*dict, key, scrub_stats->scrubbed_files);
+        ret = dict_set_uint64 (*dict, key, scrub_stats->scrubbed_files);
         if (ret) {
                 gf_msg_debug (this->name, 0, "Failed to setting scrubbed file "
                               "entry to the dictionary");
@@ -1577,7 +1562,7 @@ br_scrubber_status_get (xlator_t *this, dict_t **dict)
 
         memset (key, 0, 256);
         snprintf (key, 256, "unsigned-files");
-        ret = dict_set_uint32 (*dict, key, scrub_stats->unsigned_files);
+        ret = dict_set_uint64 (*dict, key, scrub_stats->unsigned_files);
         if (ret) {
                 gf_msg_debug (this->name, 0, "Failed to set unsigned file count"
                               " entry to the dictionary");
@@ -1585,7 +1570,7 @@ br_scrubber_status_get (xlator_t *this, dict_t **dict)
 
         memset (key, 0, 256);
         snprintf (key, 256, "scrub-duration");
-        ret = dict_set_uint32 (*dict, key, scrub_stats->scrub_duration);
+        ret = dict_set_uint64 (*dict, key, scrub_stats->scrub_duration);
         if (ret) {
                 gf_msg_debug (this->name, 0, "Failed to set scrub duration"
                               " entry to the dictionary");
@@ -1844,6 +1829,33 @@ br_signer_init (xlator_t *this, br_private_t *priv)
 }
 
 static void
+br_free_scrubber_monitor (xlator_t *this, br_private_t *priv)
+{
+        struct br_monitor *scrub_monitor = &priv->scrub_monitor;
+
+        if (scrub_monitor->timer) {
+                (void) gf_tw_del_timer (priv->timer_wheel, scrub_monitor->timer);
+
+                GF_FREE (scrub_monitor->timer);
+                scrub_monitor->timer = NULL;
+        }
+
+        (void) gf_thread_cleanup_xint (scrub_monitor->thread);
+
+        /* Clean up cond and mutex variables */
+        pthread_mutex_destroy (&scrub_monitor->mutex);
+        pthread_cond_destroy (&scrub_monitor->cond);
+
+        pthread_mutex_destroy (&scrub_monitor->wakelock);
+        pthread_cond_destroy (&scrub_monitor->wakecond);
+
+        pthread_mutex_destroy (&scrub_monitor->donelock);
+        pthread_cond_destroy (&scrub_monitor->donecond);
+
+        LOCK_DESTROY (&scrub_monitor->lock);
+}
+
+static void
 br_free_children (xlator_t *this, br_private_t *priv, int count)
 {
         br_child_t *child = NULL;
@@ -1878,7 +1890,6 @@ br_init_children (xlator_t *this, br_private_t *priv)
                 LOCK_INIT (&child->lock);
                 child->witnessed = 0;
 
-                br_set_scrub_state (child, BR_SCRUB_STATE_INACTIVE);
                 br_set_child_state (child, BR_CHILD_STATE_DISCONNECTED);
 
                 child->this = this;
@@ -1999,6 +2010,9 @@ fini (xlator_t *this)
 
         if (!priv->iamscrubber)
                 br_fini_signer (this, priv);
+        else
+                (void) br_free_scrubber_monitor (this, priv);
+
         br_free_children (this, priv, priv->child_count);
 
         this->private = NULL;
@@ -2008,26 +2022,23 @@ fini (xlator_t *this)
 }
 
 static void
-br_reconfigure_child (xlator_t *this, br_child_t *child)
+br_reconfigure_monitor (xlator_t *this)
 {
         int32_t ret = 0;
 
-        ret = br_scrub_state_machine (this, child);
+        ret = br_scrub_state_machine (this);
         if (ret) {
                 gf_msg (this->name, GF_LOG_ERROR, 0,
                         BRB_MSG_RESCHEDULE_SCRUBBER_FAILED,
-                        "Could not reschedule scrubber for brick: %s. Scubbing "
-                        "will continue according to old frequency.",
-                        child->brick_path);
+                        "Could not reschedule scrubber for the volume. Scrubbing "
+                        "will continue according to old frequency.");
         }
 }
 
 static int
 br_reconfigure_scrubber (xlator_t *this, dict_t *options)
 {
-        int           i     = 0;
         int32_t       ret   = -1;
-        br_child_t   *child = NULL;
         br_private_t *priv  = NULL;
 
         priv = this->private;
@@ -2042,32 +2053,11 @@ br_reconfigure_scrubber (xlator_t *this, dict_t *options)
                 goto err;
 
         /* change state for all _up_ subvolume(s) */
-        for (; i < priv->child_count; i++) {
-                child = &priv->children[i];
-
-                LOCK (&child->lock);
-                {
-                        if (_br_child_failed_conn (child)) {
-                                gf_msg (this->name, GF_LOG_INFO,
-                                        0, BRB_MSG_BRICK_INFO,
-                                        "Scrubber for brick [%s] failed "
-                                        "initialization, rescheduling is "
-                                        "skipped", child->brick_path);
-                                goto unblock;
-                        }
-
-                        if (_br_is_child_connected (child))
-                                br_reconfigure_child (this, child);
-
-                        /**
-                         * for the rest.. either the child is in initialization
-                         * phase or is disconnected. either way, updated values
-                         * would be reflected on successful connection.
-                         */
-                }
-        unblock:
-                UNLOCK (&child->lock);
+        pthread_mutex_lock (&priv->lock);
+        {
+                br_reconfigure_monitor (this);
         }
+        pthread_mutex_unlock (&priv->lock);
 
  err:
         return ret;
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.h b/xlators/features/bit-rot/src/bitd/bit-rot.h
index 04336e641d0..b3d1569ba54 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.h
@@ -26,6 +26,7 @@
 
 #include "bit-rot-common.h"
 #include "bit-rot-stub-mem-types.h"
+#include "bit-rot-scrub-status.h"
 
 #include <openssl/sha.h>
 
@@ -63,18 +64,6 @@ struct br_scanfs {
         unsigned int     entries;
         struct list_head queued;
         struct list_head ready;
-
-        /* scheduler */
-        uint32_t boot;
-        gf_boolean_t kick;
-        gf_boolean_t over;
-
-        br_scrub_state_t state;   /* current scrub state */
-
-        pthread_mutex_t wakelock;
-        pthread_cond_t  wakecond;
-
-        struct gf_tw_timer_list *timer;
 };
 
 /* just need three states to track child status */
@@ -111,6 +100,8 @@ struct br_child {
         struct timeval tv;
 
         struct br_scanfs fsscan;      /* per subvolume FS scanner */
+
+        gf_boolean_t active_scrubbing; /* Actively scrubbing or not */
 };
 
 typedef struct br_child br_child_t;
@@ -152,27 +143,42 @@ struct br_scrubber {
         struct list_head scrublist;
 };
 
-typedef struct br_obj_n_workers br_obj_n_workers_t;
+struct br_monitor {
+        gf_lock_t lock;
+        pthread_t thread;         /* Monitor thread */
 
-typedef struct br_private br_private_t;
+        gf_boolean_t  inited;
+        pthread_mutex_t mutex;
+        pthread_cond_t cond;      /* Thread starts and will be waiting on cond.
+                                     First child which is up wakes this up */
 
-typedef void (*br_scrubbed_file_update) (br_private_t *priv);
+        xlator_t *this;
+        /* scheduler */
+        uint32_t boot;
 
-struct br_scrub_stats {
-        uint32_t       scrubbed_files;       /* Total number of scrubbed file */
+        int32_t active_child_count; /* Number of children currently scrubbing */
+        gf_boolean_t kick;          /* This variable tracks the scrubber is
+                                     * kicked or not. Both 'kick' and
+                                     * 'active_child_count' uses the same pair
+                                     * of mutex-cond variable, i.e, wakelock and
+                                     * wakecond. */
 
-        uint32_t       unsigned_files;       /* Total number of unsigned file */
+        pthread_mutex_t wakelock;
+        pthread_cond_t  wakecond;
 
-        uint32_t       scrub_duration;            /* Duration of last scrub */
+        gf_boolean_t done;
+        pthread_mutex_t donelock;
+        pthread_cond_t  donecond;
 
-        char           last_scrub_time[1024];    /*last scrub completion time */
+        struct gf_tw_timer_list *timer;
+        br_scrub_state_t state;   /* current scrub state */
+};
 
-        struct         timeval scrub_start_tv;   /* Scrubbing starting time*/
+typedef struct br_obj_n_workers br_obj_n_workers_t;
 
-        struct         timeval scrub_end_tv;     /* Scrubbing finishing time */
+typedef struct br_private br_private_t;
 
-        pthread_mutex_t  lock;
-};
+typedef void (*br_scrubbed_file_update) (br_private_t *priv);
 
 struct br_private {
         pthread_mutex_t lock;
@@ -209,6 +215,8 @@ struct br_private {
         struct br_scrub_stats scrub_stat; /* statistics of scrub*/
 
         struct br_scrubber fsscrub;       /* scrubbers for this subvolume */
+
+        struct br_monitor scrub_monitor;  /* scrubber monitor */
 };
 
 struct br_object {
@@ -228,7 +236,7 @@ struct br_object {
 };
 
 typedef struct br_object br_object_t;
-typedef int32_t (br_scrub_ssm_call) (xlator_t *, br_child_t *);
+typedef int32_t (br_scrub_ssm_call) (xlator_t *);
 
 void
 br_log_object (xlator_t *, char *, uuid_t, int32_t);
@@ -259,6 +267,12 @@ _br_is_child_connected (br_child_t *child)
 }
 
 static inline int
+_br_is_child_scrub_active (br_child_t *child)
+{
+        return child->active_scrubbing;
+}
+
+static inline int
 _br_child_failed_conn (br_child_t *child)
 {
         return (child->c_state == BR_CHILD_STATE_CONNFAILED);
@@ -272,10 +286,10 @@ _br_child_witnessed_connection (br_child_t *child)
 
 /* scrub state */
 static inline void
-_br_child_set_scrub_state (br_child_t *child, br_scrub_state_t state)
+_br_monitor_set_scrub_state (struct br_monitor *scrub_monitor,
+                           br_scrub_state_t state)
 {
-        struct br_scanfs *fsscan = &child->fsscan;
-        fsscan->state = state;
+        scrub_monitor->state = state;
 }
 
 static inline br_scrub_event_t