feature/bitrot: Ondemand scrub option for bitrot

The bitrot scrubber takes 'hourly/daily/biweekly/monthly'
as the values for 'scrub-frequency'. There is no way
to schedule the scrubbing when the admin wants it.

Ondemand scrubbing brings in the new option 'ondemand'
with which the admin can start scrubbing ondemand.
It starts the scrubbing immediately.

Ondemand scrubbing is successful only if the scrubber
is in 'Active (Idle)' (waiting for it's next frequency
cycle to start scrubbing). It is not entertained when
the scrubber is in 'Paused' or already running.

Here is the command line syntax.

gluster volume bitrot <vol name> scrub ondemand

Change-Id: I84c28904367eed827a7dae8d6a535c14b28e9f4d
BUG: 1366195
Signed-off-by: Kotresh HR <khiremat@redhat.com>
Reviewed-on: http://review.gluster.org/15111
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Venky Shankar <vshankar@redhat.com>

6 files changed, 97 insertions, 13 deletions

diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
index 1db38e43ee8..72bdb843488 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
@@ -863,6 +863,7 @@ br_fsscan_calculate_delta (uint32_t times)
         return times;
 }
 
+#define BR_SCRUB_ONDEMAND   (1)
 #define BR_SCRUB_MINUTE     (60)
 #define BR_SCRUB_HOURLY     (60 * 60)
 #define BR_SCRUB_DAILY      (1 * 24 * 60 * 60)
@@ -1040,6 +1041,53 @@ br_fsscan_reschedule (xlator_t *this)
         return 0;
 }
 
+int32_t
+br_fsscan_ondemand (xlator_t *this)
+{
+        int32_t             ret     = 0;
+        uint32_t            timo    = 0;
+        char timestr[1024]          = {0,};
+        struct timeval      now     = {0,};
+        br_private_t       *priv    = NULL;
+        struct br_scrubber *fsscrub = NULL;
+        struct br_monitor  *scrub_monitor = NULL;
+
+        priv = this->private;
+        fsscrub = &priv->fsscrub;
+        scrub_monitor = &priv->scrub_monitor;
+
+        if (!fsscrub->frequency_reconf)
+                return 0;
+
+        (void) gettimeofday (&now, NULL);
+
+        timo = BR_SCRUB_ONDEMAND;
+
+        gf_time_fmt (timestr, sizeof (timestr),
+                     (now.tv_sec + timo), gf_timefmt_FT);
+
+        pthread_mutex_lock (&scrub_monitor->donelock);
+        {
+                scrub_monitor->done = _gf_false;
+        }
+        pthread_mutex_unlock (&scrub_monitor->donelock);
+
+        ret = gf_tw_mod_timer_pending (priv->timer_wheel, scrub_monitor->timer,
+                                       timo);
+        if (ret == 0)
+                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+                        "Scrubber is currently running and would be "
+                        "rescheduled after completion");
+        else {
+                _br_monitor_set_scrub_state (scrub_monitor,
+                                             BR_SCRUB_STATE_PENDING);
+                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+                        "Ondemand Scrubbing scheduled to run at %s", timestr);
+        }
+
+        return 0;
+}
+
 #define BR_SCRUB_THREAD_SCALE_LAZY       0
 #define BR_SCRUB_THREAD_SCALE_NORMAL     0.4
 #define BR_SCRUB_THREAD_SCALE_AGGRESSIVE 1.0
@@ -1867,7 +1915,7 @@ br_monitor_thread (void *arg)
         /* this needs to be serialized with reconfigure() */
         pthread_mutex_lock (&priv->lock);
         {
-                ret = br_scrub_state_machine (this);
+                ret = br_scrub_state_machine (this, _gf_false);
         }
         pthread_mutex_unlock (&priv->lock);
         if (ret) {
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
index 63169068ed4..8cc88ec153e 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
@@ -20,6 +20,7 @@ int32_t br_fsscan_schedule (xlator_t *);
 int32_t br_fsscan_reschedule (xlator_t *);
 int32_t br_fsscan_activate (xlator_t *);
 int32_t br_fsscan_deactivate (xlator_t *);
+int32_t br_fsscan_ondemand (xlator_t *);
 
 int32_t br_scrubber_handle_options (xlator_t *, br_private_t *, dict_t *);
 
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
index d304fc804ee..af887a1ff8b 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
@@ -84,16 +84,22 @@ br_scrub_ssm_state_stall (xlator_t *this)
 
 static br_scrub_ssm_call *
 br_scrub_ssm[BR_SCRUB_MAXSTATES][BR_SCRUB_MAXEVENTS] = {
-        {br_fsscan_schedule, br_scrub_ssm_state_ipause},    /* INACTIVE */
-        {br_fsscan_reschedule, br_fsscan_deactivate},       /* PENDING  */
-        {br_scrub_ssm_noop, br_scrub_ssm_state_stall},      /* ACTIVE   */
-        {br_fsscan_activate, br_scrub_ssm_noop},            /* PAUSED   */
-        {br_fsscan_schedule, br_scrub_ssm_noop},            /* IPAUSED  */
-        {br_scrub_ssm_state_active, br_scrub_ssm_noop},     /* STALLED  */
+        /* INACTIVE */
+        {br_fsscan_schedule, br_scrub_ssm_state_ipause, br_scrub_ssm_noop},
+        /* PENDING  */
+        {br_fsscan_reschedule, br_fsscan_deactivate, br_fsscan_ondemand},
+        /* ACTIVE   */
+        {br_scrub_ssm_noop, br_scrub_ssm_state_stall, br_scrub_ssm_noop},
+        /* PAUSED   */
+        {br_fsscan_activate, br_scrub_ssm_noop, br_scrub_ssm_noop},
+        /* IPAUSED  */
+        {br_fsscan_schedule, br_scrub_ssm_noop, br_scrub_ssm_noop},
+        /* STALLED  */
+        {br_scrub_ssm_state_active, br_scrub_ssm_noop, br_scrub_ssm_noop},
 };
 
 int32_t
-br_scrub_state_machine (xlator_t *this)
+br_scrub_state_machine (xlator_t *this, gf_boolean_t scrub_ondemand)
 {
         br_private_t       *priv      = NULL;
         br_scrub_ssm_call  *call      = NULL;
@@ -107,7 +113,10 @@ br_scrub_state_machine (xlator_t *this)
         scrub_monitor = &priv->scrub_monitor;
 
         currstate = scrub_monitor->state;
-        event = _br_child_get_scrub_event (fsscrub);
+        if (scrub_ondemand)
+                event = BR_SCRUB_EVENT_ONDEMAND;
+        else
+                event = _br_child_get_scrub_event (fsscrub);
 
         call = br_scrub_ssm[currstate][event];
         return call (this);
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
index 936ee4d837c..8609477180b 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
@@ -26,11 +26,12 @@ typedef enum br_scrub_state {
 typedef enum br_scrub_event {
         BR_SCRUB_EVENT_SCHEDULE = 0,
         BR_SCRUB_EVENT_PAUSE,
+        BR_SCRUB_EVENT_ONDEMAND,
         BR_SCRUB_MAXEVENTS,
 } br_scrub_event_t;
 
 struct br_monitor;
 
-int32_t br_scrub_state_machine (xlator_t *);
+int32_t br_scrub_state_machine (xlator_t *, gf_boolean_t);
 
 #endif /* __BIT_ROT_SSM_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c
index ca3fc273e9f..882ac1da6b4 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.c
@@ -1534,7 +1534,6 @@ _br_qchild_event (xlator_t *this, br_child_t *child, br_child_handler *call)
 int
 br_scrubber_status_get (xlator_t *this, dict_t **dict)
 {
-
         int                    ret          = -1;
         br_private_t          *priv         = NULL;
         struct br_scrub_stats *scrub_stats  = NULL;
@@ -1600,9 +1599,11 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         br_private_t *priv   = NULL;
         dict_t       *output = NULL;
         va_list       ap;
+        struct br_monitor  *scrub_monitor = NULL;
 
         subvol = (xlator_t *)data;
         priv = this->private;
+        scrub_monitor = &priv->scrub_monitor;
 
         gf_msg_trace (this->name, 0, "Notification received: %d", event);
 
@@ -1676,6 +1677,30 @@ notify (xlator_t *this, int32_t event, void *data, ...)
                 ret = br_scrubber_status_get (this, &output);
                 gf_msg_debug (this->name, 0, "returning %d", ret);
                 break;
+
+        case GF_EVENT_SCRUB_ONDEMAND:
+                gf_log (this->name, GF_LOG_INFO, "BitRot scrub ondemand "
+                              "called");
+
+                if (scrub_monitor->state != BR_SCRUB_STATE_PENDING)
+                        return -2;
+
+                /* Needs synchronization with reconfigure thread */
+                pthread_mutex_lock (&priv->lock);
+                {
+                        ret = br_scrub_state_machine (this, _gf_true);
+                }
+                pthread_mutex_unlock (&priv->lock);
+
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                BRB_MSG_RESCHEDULE_SCRUBBER_FAILED,
+                                "Could not schedule ondemand scrubbing. "
+                                "Scrubbing will continue according to "
+                                "old frequency.");
+                }
+                gf_msg_debug (this->name, 0, "returning %d", ret);
+                break;
         default:
                 default_notify (this, event, data);
         }
@@ -2045,7 +2070,7 @@ br_reconfigure_monitor (xlator_t *this)
 {
         int32_t ret = 0;
 
-        ret = br_scrub_state_machine (this);
+        ret = br_scrub_state_machine (this, _gf_false);
         if (ret) {
                 gf_msg (this->name, GF_LOG_ERROR, 0,
                         BRB_MSG_RESCHEDULE_SCRUBBER_FAILED,
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.h b/xlators/features/bit-rot/src/bitd/bit-rot.h
index b5448f76d52..c2c0c225792 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.h
@@ -297,7 +297,7 @@ static inline br_scrub_event_t
 _br_child_get_scrub_event (struct br_scrubber *fsscrub)
 {
         return (fsscrub->frequency == BR_FSSCRUB_FREQ_STALLED)
-                ? BR_SCRUB_EVENT_PAUSE : BR_SCRUB_EVENT_SCHEDULE;
+             ? BR_SCRUB_EVENT_PAUSE : BR_SCRUB_EVENT_SCHEDULE;
 }
 
 int32_t