features/bitrot: Throttle filesystem scrubber

This patch introduces multithreaded filesystem scrubber based
on throttling option configured for a particular volume. The
implementation "logically" breaks scanning and scrubbing with
the number of scrubber threads auto-configured depending upon
the throttle configuration. Scanning (crawling) is left single
threaded (per brick) with entries scrubbed in bulk. On reaching
this "bulk" watermark, scanner waits until entries are scrubbed.
Bricks for a particular volume have a set of thread(s) assigned
for scrubbing, with entries for each brick scrubbed in a round
robin fashion to avoid scrub "stalls" when a brick (out of N
bricks) is under active scrubbing.

This mechanism helps us implement "pause/resume" with ease: all
one need to do is to cleanup scrubber threads and let the main
scanner thread "wait" untill scrubbing is resumed (where the
scrubber thread(s) are spawned again), therefore continuing
where we left off (unless we restart the deamons, where crawl
initiates from root directory again, but I guess that's OK).

[
    NOTE:

    Throttling is optional for the signer daemon, without which
    it runs full throttle. However, passing "-DBR_RATE_LIMIT_SIGNER"
    predefined in CFLAGS enables CPU throttling (during checksum
    calculation) thereby avoiding high CPU usage.
]

Subsequent patches would introduce CPU throttling during hash
calculation for scrubber.

Change-Id: I5701dd6cd4dff27ca3144ac5e3798a2216b39d4f
BUG: 1207020
Signed-off-by: Venky Shankar <vshankar@redhat.com>
Reviewed-on: http://review.gluster.org/10511
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 39 insertions, 1 deletions

diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.h b/xlators/features/bit-rot/src/bitd/bit-rot.h
index 5b641801916..6f21a6985ba 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.h
@@ -38,8 +38,26 @@
  */
 #define BR_WORKERS 4
 
+typedef enum scrub_throttle {
+        BR_SCRUB_THROTTLE_VOID       = -1,
+        BR_SCRUB_THROTTLE_LAZY       = 0,
+        BR_SCRUB_THROTTLE_NORMAL     = 1,
+        BR_SCRUB_THROTTLE_AGGRESSIVE = 2,
+} scrub_throttle_t;
+
 #define signature_size(hl) (sizeof (br_isignature_t) + hl + 1)
 
+struct br_scanfs {
+        gf_lock_t entrylock;
+
+        pthread_mutex_t waitlock;
+        pthread_cond_t  waitcond;
+
+        unsigned int     entries;
+        struct list_head queued;
+        struct list_head ready;
+};
+
 struct br_child {
         char child_up;                /* Indicates whether this child is
                                          up or not */
@@ -53,12 +71,14 @@ struct br_child {
         xlator_t *this;               /* Bit rot xlator */
 
         pthread_t thread;             /* initial crawler for unsigned
-                                         object(s) */
+                                         object(s) or scrub crawler */
         int threadrunning;            /* active thread */
 
         struct mem_pool *timer_pool;  /* timer-wheel's timer mem-pool */
 
         struct timeval tv;
+
+        struct br_scanfs fsscan;      /* per subvolume FS scanner */
 };
 
 typedef struct br_child br_child_t;
@@ -72,6 +92,23 @@ struct br_obj_n_workers {
                                              signing each object */
 };
 
+struct br_scrubber {
+        xlator_t *this;
+
+        scrub_throttle_t throttle;
+
+        pthread_mutex_t mutex;
+        pthread_cond_t  cond;
+
+        unsigned int nr_scrubbers;
+        struct list_head scrubbers;
+
+        /*
+         * list of "rotatable" subvolume(s) undergoing scrubbing
+         */
+        struct list_head scrublist;
+};
+
 typedef struct br_obj_n_workers br_obj_n_workers_t;
 
 struct br_private {
@@ -100,6 +137,7 @@ struct br_private {
         br_tbf_t *tbf;                    /* token bucket filter */
 
         gf_boolean_t iamscrubber;         /* function as a fs scrubber */
+        struct br_scrubber fsscrub;       /* scrubbers for this subvolume */
 };
 
 typedef struct br_private br_private_t;