features/bitrot: Throttle filesystem scrubber

This patch introduces multithreaded filesystem scrubber based
on throttling option configured for a particular volume. The
implementation "logically" breaks scanning and scrubbing with
the number of scrubber threads auto-configured depending upon
the throttle configuration. Scanning (crawling) is left single
threaded (per brick) with entries scrubbed in bulk. On reaching
this "bulk" watermark, scanner waits until entries are scrubbed.
Bricks for a particular volume have a set of thread(s) assigned
for scrubbing, with entries for each brick scrubbed in a round
robin fashion to avoid scrub "stalls" when a brick (out of N
bricks) is under active scrubbing.

This mechanism helps us implement "pause/resume" with ease: all
one need to do is to cleanup scrubber threads and let the main
scanner thread "wait" untill scrubbing is resumed (where the
scrubber thread(s) are spawned again), therefore continuing
where we left off (unless we restart the deamons, where crawl
initiates from root directory again, but I guess that's OK).

[
    NOTE:

    Throttling is optional for the signer daemon, without which
    it runs full throttle. However, passing "-DBR_RATE_LIMIT_SIGNER"
    predefined in CFLAGS enables CPU throttling (during checksum
    calculation) thereby avoiding high CPU usage.
]

Subsequent patches would introduce CPU throttling during hash
calculation for scrubber.

Change-Id: I5701dd6cd4dff27ca3144ac5e3798a2216b39d4f
BUG: 1207020
Signed-off-by: Venky Shankar <vshankar@redhat.com>
Reviewed-on: http://review.gluster.org/10511
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

4 files changed, 48 insertions, 24 deletions

diff --git a/libglusterfs/src/gf-dirent.c b/libglusterfs/src/gf-dirent.c
index b5f395afc36..99c0eb6441d 100644
--- a/libglusterfs/src/gf-dirent.c
+++ b/libglusterfs/src/gf-dirent.c
@@ -171,6 +171,20 @@ gf_dirent_for_name (const char *name)
         return gf_dirent;
 }
 
+void
+gf_dirent_entry_free (gf_dirent_t *entry)
+{
+        if (!entry)
+                return;
+
+        if (entry->dict)
+                dict_unref (entry->dict);
+        if (entry->inode)
+                inode_unref (entry->inode);
+
+        list_del (&entry->list);
+        GF_FREE (entry);
+}
 
 void
 gf_dirent_free (gf_dirent_t *entries)
@@ -185,16 +199,27 @@ gf_dirent_free (gf_dirent_t *entries)
                 return;
 
         list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                if (entry->dict)
-                        dict_unref (entry->dict);
-                if (entry->inode)
-                        inode_unref (entry->inode);
-
-                list_del (&entry->list);
-                GF_FREE (entry);
+                gf_dirent_entry_free (entry);
         }
 }
 
+gf_dirent_t *
+entry_copy (gf_dirent_t *source)
+{
+        gf_dirent_t *sink = NULL;
+
+        sink = gf_dirent_for_name (source->d_name);
+
+        sink->d_off = source->d_off;
+        sink->d_ino = source->d_ino;
+        sink->d_type = source->d_type;
+        sink->d_stat = source->d_stat;
+
+	if (source->inode)
+		sink->inode = inode_ref (source->inode);
+        return sink;
+}
+
 void
 gf_link_inode_from_dirent (xlator_t *this, inode_t *parent, gf_dirent_t *entry)
 {
diff --git a/libglusterfs/src/gf-dirent.h b/libglusterfs/src/gf-dirent.h
index 07c605f82b0..faeaf411941 100644
--- a/libglusterfs/src/gf-dirent.h
+++ b/libglusterfs/src/gf-dirent.h
@@ -61,6 +61,8 @@ struct _gf_dirent_t {
 #define DT_ISDIR(mode) (mode == DT_DIR)
 
 gf_dirent_t *gf_dirent_for_name (const char *name);
+gf_dirent_t *entry_copy (gf_dirent_t *source);
+void gf_dirent_entry_free (gf_dirent_t *entry);
 void gf_dirent_free (gf_dirent_t *entries);
 int gf_link_inodes_from_dirent (xlator_t *this, inode_t *parent,
                                 gf_dirent_t *entries);
diff --git a/libglusterfs/src/list.h b/libglusterfs/src/list.h
index 875594136a2..b8f9a6eebd8 100644
--- a/libglusterfs/src/list.h
+++ b/libglusterfs/src/list.h
@@ -214,6 +214,20 @@ static inline void list_replace_init(struct list_head *old,
 	INIT_LIST_HEAD(old);
 }
 
+/**
+ * list_rotate_left - rotate the list to the left
+ * @head: the head of the list
+ */
+static inline void list_rotate_left (struct list_head *head)
+{
+	struct list_head *first;
+
+	if (!list_empty (head)) {
+		first = head->next;
+		list_move_tail (first, head);
+	}
+}
+
 #define list_entry(ptr, type, member)					\
 	((type *)((char *)(ptr)-(unsigned long)(&((type *)0)->member)))
 
diff --git a/libglusterfs/src/syncop.c b/libglusterfs/src/syncop.c
index cb08b03d44b..81eae5a9105 100644
--- a/libglusterfs/src/syncop.c
+++ b/libglusterfs/src/syncop.c
@@ -1217,23 +1217,6 @@ syncop_lookup (xlator_t *subvol, loc_t *loc, struct iatt *iatt,
         return args.op_ret;
 }
 
-static gf_dirent_t *
-entry_copy (gf_dirent_t *source)
-{
-        gf_dirent_t *sink = NULL;
-
-        sink = gf_dirent_for_name (source->d_name);
-
-        sink->d_off = source->d_off;
-        sink->d_ino = source->d_ino;
-        sink->d_type = source->d_type;
-        sink->d_stat = source->d_stat;
-
-	if (source->inode)
-		sink->inode = inode_ref (source->inode);
-        return sink;
-}
-
 int32_t
 syncop_readdirp_cbk (call_frame_t *frame,
                      void *cookie,