features/bitrot: Move throttling code to libglusterfs

Backport of  http://review.gluster.org/14846

Since throttling is a separate feature by itself,
move throttling code to libglusterfs.

Change-Id: If9b99885ceb46e5b1865a4af18b2a2caecf59972
BUG: 1357514
Signed-off-by: Kotresh HR <khiremat@redhat.com>
Reviewed-on: http://review.gluster.org/14944
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>

9 files changed, 112 insertions, 105 deletions

diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am
index 85886c1f188..e56d92d6e17 100644
--- a/libglusterfs/src/Makefile.am
+++ b/libglusterfs/src/Makefile.am
@@ -29,7 +29,7 @@ libglusterfs_la_SOURCES = dict.c xlator.c logging.c \
 	strfd.c parse-utils.c $(CONTRIBDIR)/mount/mntent.c \
 	$(CONTRIBDIR)/libexecinfo/execinfo.c quota-common-utils.c rot-buffs.c \
 	$(CONTRIBDIR)/timer-wheel/timer-wheel.c \
-	$(CONTRIBDIR)/timer-wheel/find_last_bit.c tw.c
+	$(CONTRIBDIR)/timer-wheel/find_last_bit.c tw.c throttle-tbf.c
 
 nodist_libglusterfs_la_SOURCES = y.tab.c graph.lex.c
 
@@ -50,7 +50,7 @@ noinst_HEADERS = common-utils.h defaults.h dict.h glusterfs.h hashfn.h timespec.
 	$(CONTRIBDIR)/libexecinfo/execinfo_compat.h \
 	unittest/unittest.h quota-common-utils.h rot-buffs.h \
 	$(CONTRIBDIR)/timer-wheel/timer-wheel.h compat-uuid.h \
-	upcall-utils.h tw.h
+	upcall-utils.h tw.h throttle-tbf.h
 
 if !HAVE_LIBUUID
 # FIXME: unbundle libuuid, see compat-uuid.h.
diff --git a/libglusterfs/src/mem-types.h b/libglusterfs/src/mem-types.h
index 124e5692f9e..455ee641dae 100644
--- a/libglusterfs/src/mem-types.h
+++ b/libglusterfs/src/mem-types.h
@@ -157,6 +157,10 @@ enum gf_common_mem_types_ {
         gf_common_mt_syncenv,
         gf_common_mt_scan_data,
         gf_common_list_node,
+        /* throttle */
+        gf_common_mt_tbf_t,
+        gf_common_mt_tbf_bucket_t,
+        gf_common_mt_tbf_throttle_t,
         gf_common_mt_end
 };
 #endif
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-tbf.c b/libglusterfs/src/throttle-tbf.c
index f8b9b75d575..16630a243c2 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-tbf.c
+++ b/libglusterfs/src/throttle-tbf.c
@@ -24,10 +24,9 @@
  */
 
 #include "mem-pool.h"
-#include "bit-rot-tbf.h"
-#include "bit-rot-stub-mem-types.h"
+#include "throttle-tbf.h"
 
-typedef struct br_tbf_throttle {
+typedef struct tbf_throttle {
         char done;
 
         pthread_mutex_t mutex;
@@ -36,32 +35,15 @@ typedef struct br_tbf_throttle {
         unsigned long tokens;
 
         struct list_head list;
-} br_tbf_throttle_t;
+} tbf_throttle_t;
 
-/**
- * OK. Most implementations of TBF I've come across generate tokens
- * every second (UML, etc..) and some chose sub-second granularity
- * (blk-iothrottle cgroups). TBF algorithm itself does not enforce
- * any logic for choosing generation interval and it seems pretty
- * logical as one could jack up token count per interval w.r.t.
- * generation rate.
- *
- * Value used here is chosen based on a series of test(s) performed
- * to balance object signing time and not maxing out on all available
- * CPU cores. It's obvious to have seconds granularity and jack up
- * token count per interval, thereby achieving close to similar
- * results. Let's stick to this as it seems to be working fine for
- * the set of ops that are throttled.
- */
-#define BR_TBF_TOKENGEN_INTERVAL_USEC  600000
-
-static br_tbf_throttle_t *
-br_tbf_init_throttle (unsigned long tokens_required)
+static tbf_throttle_t *
+tbf_init_throttle (unsigned long tokens_required)
 {
-        br_tbf_throttle_t *throttle = NULL;
+        tbf_throttle_t *throttle = NULL;
 
         throttle = GF_CALLOC (1, sizeof (*throttle),
-                              gf_br_mt_br_tbf_throttle_t);
+                              gf_common_mt_tbf_throttle_t);
         if (!throttle)
                 return NULL;
 
@@ -76,11 +58,11 @@ br_tbf_init_throttle (unsigned long tokens_required)
 }
 
 void
-_br_tbf_dispatch_queued (br_tbf_bucket_t *bucket)
+_tbf_dispatch_queued (tbf_bucket_t *bucket)
 {
         gf_boolean_t xcont = _gf_false;
-        br_tbf_throttle_t *tmp = NULL;
-        br_tbf_throttle_t *throttle = NULL;
+        tbf_throttle_t *tmp = NULL;
+        tbf_throttle_t *throttle = NULL;
 
         list_for_each_entry_safe (throttle, tmp, &bucket->queued, list) {
 
@@ -105,17 +87,19 @@ _br_tbf_dispatch_queued (br_tbf_bucket_t *bucket)
         }
 }
 
-void *br_tbf_tokengenerator (void *arg)
+void *tbf_tokengenerator (void *arg)
 {
         unsigned long tokenrate = 0;
         unsigned long maxtokens = 0;
-        br_tbf_bucket_t *bucket = arg;
+        unsigned long token_gen_interval = 0;
+        tbf_bucket_t *bucket = arg;
 
         tokenrate = bucket->tokenrate;
         maxtokens = bucket->maxtokens;
+        token_gen_interval = bucket->token_gen_interval;
 
         while (1) {
-                usleep (BR_TBF_TOKENGEN_INTERVAL_USEC);
+                usleep (token_gen_interval);
 
                 LOCK (&bucket->lock);
                 {
@@ -124,7 +108,7 @@ void *br_tbf_tokengenerator (void *arg)
                                 bucket->tokens = maxtokens;
 
                         if (!list_empty (&bucket->queued))
-                                _br_tbf_dispatch_queued (bucket);
+                                _tbf_dispatch_queued (bucket);
                 }
                 UNLOCK (&bucket->lock);
         }
@@ -134,18 +118,18 @@ void *br_tbf_tokengenerator (void *arg)
 
 /**
  * There is lazy synchronization between this routine (when invoked
- * under br_tbf_mod() context) and br_tbf_throttle(). *bucket is
+ * under tbf_mod() context) and tbf_throttle(). *bucket is
  * updated _after_ all the required variables are initialized.
  */
 static int32_t
-br_tbf_init_bucket (br_tbf_t *tbf, br_tbf_opspec_t *spec)
+tbf_init_bucket (tbf_t *tbf, tbf_opspec_t *spec)
 {
         int ret = 0;
-        br_tbf_bucket_t *curr = NULL;
-        br_tbf_bucket_t **bucket = NULL;
+        tbf_bucket_t *curr = NULL;
+        tbf_bucket_t **bucket = NULL;
 
-        GF_ASSERT (spec->op >= BR_TBF_OP_MIN);
-        GF_ASSERT (spec->op <= BR_TBF_OP_MAX);
+        GF_ASSERT (spec->op >= TBF_OP_MIN);
+        GF_ASSERT (spec->op <= TBF_OP_MAX);
 
         /* no rate? no throttling. */
         if (!spec->rate)
@@ -153,7 +137,7 @@ br_tbf_init_bucket (br_tbf_t *tbf, br_tbf_opspec_t *spec)
 
         bucket = tbf->bucket + spec->op;
 
-        curr = GF_CALLOC (1, sizeof (*curr), gf_br_mt_br_tbf_bucket_t);
+        curr = GF_CALLOC (1, sizeof (*curr), gf_common_mt_tbf_bucket_t);
         if (!curr)
                 goto error_return;
 
@@ -163,9 +147,10 @@ br_tbf_init_bucket (br_tbf_t *tbf, br_tbf_opspec_t *spec)
         curr->tokens = 0;
         curr->tokenrate = spec->rate;
         curr->maxtokens = spec->maxlimit;
+        curr->token_gen_interval = spec->token_gen_interval;
 
         ret = gf_thread_create (&curr->tokener,
-                                NULL, br_tbf_tokengenerator, curr);
+                                NULL, tbf_tokengenerator, curr);
         if (ret != 0)
                 goto freemem;
 
@@ -179,30 +164,30 @@ br_tbf_init_bucket (br_tbf_t *tbf, br_tbf_opspec_t *spec)
         return -1;
 }
 
-#define BR_TBF_ALLOC_SIZE                                               \
-        (sizeof (br_tbf_t) + (BR_TBF_OP_MAX * sizeof (br_tbf_bucket_t)))
+#define TBF_ALLOC_SIZE                                               \
+        (sizeof (tbf_t) + (TBF_OP_MAX * sizeof (tbf_bucket_t)))
 
-br_tbf_t *
-br_tbf_init (br_tbf_opspec_t *tbfspec, unsigned int count)
+tbf_t *
+tbf_init (tbf_opspec_t *tbfspec, unsigned int count)
 {
         int32_t i = 0;
         int32_t ret = 0;
-        br_tbf_t *tbf = NULL;
-        br_tbf_opspec_t *opspec = NULL;
+        tbf_t *tbf = NULL;
+        tbf_opspec_t *opspec = NULL;
 
-        tbf = GF_CALLOC (1, BR_TBF_ALLOC_SIZE, gf_br_mt_br_tbf_t);
+        tbf = GF_CALLOC (1, TBF_ALLOC_SIZE, gf_common_mt_tbf_t);
         if (!tbf)
                 goto error_return;
 
-        tbf->bucket = (br_tbf_bucket_t **) ((char *)tbf + sizeof (*tbf));
-        for (i = 0; i < BR_TBF_OP_MAX; i++) {
+        tbf->bucket = (tbf_bucket_t **) ((char *)tbf + sizeof (*tbf));
+        for (i = 0; i < TBF_OP_MAX; i++) {
                 *(tbf->bucket + i) = NULL;
         }
 
         for (i = 0; i < count; i++) {
                 opspec = tbfspec + i;
 
-                ret = br_tbf_init_bucket (tbf, opspec);
+                ret = tbf_init_bucket (tbf, opspec);
                 if (ret)
                         break;
         }
@@ -217,7 +202,7 @@ br_tbf_init (br_tbf_opspec_t *tbfspec, unsigned int count)
 }
 
 static void
-br_tbf_mod_bucket (br_tbf_bucket_t *bucket, br_tbf_opspec_t *spec)
+tbf_mod_bucket (tbf_bucket_t *bucket, tbf_opspec_t *spec)
 {
         LOCK (&bucket->lock);
         {
@@ -231,39 +216,39 @@ br_tbf_mod_bucket (br_tbf_bucket_t *bucket, br_tbf_opspec_t *spec)
 }
 
 int
-br_tbf_mod (br_tbf_t *tbf, br_tbf_opspec_t *tbfspec)
+tbf_mod (tbf_t *tbf, tbf_opspec_t *tbfspec)
 {
         int              ret    = 0;
-        br_tbf_bucket_t *bucket = NULL;
-        br_tbf_ops_t     op     = BR_TBF_OP_MIN;
+        tbf_bucket_t *bucket = NULL;
+        tbf_ops_t     op     = TBF_OP_MIN;
 
         if (!tbf || !tbfspec)
                 return -1;
 
         op = tbfspec->op;
 
-        GF_ASSERT (op >= BR_TBF_OP_MIN);
-        GF_ASSERT (op <= BR_TBF_OP_MAX);
+        GF_ASSERT (op >= TBF_OP_MIN);
+        GF_ASSERT (op <= TBF_OP_MAX);
 
         bucket = *(tbf->bucket + op);
         if (bucket) {
-                br_tbf_mod_bucket (bucket, tbfspec);
+                tbf_mod_bucket (bucket, tbfspec);
         } else {
-                ret = br_tbf_init_bucket (tbf, tbfspec);
+                ret = tbf_init_bucket (tbf, tbfspec);
         }
 
         return ret;
 }
 
 void
-br_tbf_throttle (br_tbf_t *tbf, br_tbf_ops_t op, unsigned long tokens_requested)
+tbf_throttle (tbf_t *tbf, tbf_ops_t op, unsigned long tokens_requested)
 {
         char waitq = 0;
-        br_tbf_bucket_t *bucket = NULL;
-        br_tbf_throttle_t *throttle = NULL;
+        tbf_bucket_t *bucket = NULL;
+        tbf_throttle_t *throttle = NULL;
 
-        GF_ASSERT (op >= BR_TBF_OP_MIN);
-        GF_ASSERT (op <= BR_TBF_OP_MAX);
+        GF_ASSERT (op >= TBF_OP_MIN);
+        GF_ASSERT (op <= TBF_OP_MAX);
 
         bucket = *(tbf->bucket + op);
         if (!bucket)
@@ -279,7 +264,7 @@ br_tbf_throttle (br_tbf_t *tbf, br_tbf_ops_t op, unsigned long tokens_requested)
                 if (tokens_requested <= bucket->tokens) {
                         bucket->tokens -= tokens_requested;
                 } else {
-                        throttle = br_tbf_init_throttle (tokens_requested);
+                        throttle = tbf_init_throttle (tokens_requested);
                         if (!throttle) /* let it slip through for now.. */
                                 goto unblock;
 
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-tbf.h b/libglusterfs/src/throttle-tbf.h
index 5a41be4fd95..b6e04962ca4 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-tbf.h
+++ b/libglusterfs/src/throttle-tbf.h
@@ -12,32 +12,34 @@
 #include "xlator.h"
 #include "locking.h"
 
-#ifndef __BIT_ROT_TBF_H__
-#define __BIT_ROT_TBF_H__
+#ifndef THROTTLE_TBF_H__
+#define THROTTLE_TBF_H__
 
-typedef enum br_tbf_ops {
-        BR_TBF_OP_MIN     = -1,
-        BR_TBF_OP_HASH    = 0,    /* checksum calculation  */
-        BR_TBF_OP_READ    = 1,    /* inode read(s)         */
-        BR_TBF_OP_READDIR = 2,    /* dentry read(s)        */
-        BR_TBF_OP_MAX     = 3,
-} br_tbf_ops_t;
+typedef enum tbf_ops {
+        TBF_OP_MIN     = -1,
+        TBF_OP_HASH    = 0,    /* checksum calculation  */
+        TBF_OP_READ    = 1,    /* inode read(s)         */
+        TBF_OP_READDIR = 2,    /* dentry read(s)        */
+        TBF_OP_MAX     = 3,
+} tbf_ops_t;
 
 /**
  * Operation rate specification
  */
-typedef struct br_tbf_opspec {
-        br_tbf_ops_t op;
+typedef struct tbf_opspec {
+        tbf_ops_t op;
 
         unsigned long rate;
 
         unsigned long maxlimit;
-} br_tbf_opspec_t;
+
+        unsigned long token_gen_interval;/* Token generation interval in usec */
+} tbf_opspec_t;
 
 /**
  * Token bucket for each operation type
  */
-typedef struct br_tbf_bucket {
+typedef struct tbf_bucket {
         gf_lock_t lock;
 
         pthread_t tokener;         /* token generator thread          */
@@ -49,22 +51,24 @@ typedef struct br_tbf_bucket {
         unsigned long maxtokens;   /* maximum token in the bucket     */
 
         struct list_head queued;   /* list of non-conformant requests */
-} br_tbf_bucket_t;
 
-typedef struct br_tbf {
-        br_tbf_bucket_t **bucket;
-} br_tbf_t;
+        unsigned long token_gen_interval;/* Token generation interval in usec */
+} tbf_bucket_t;
+
+typedef struct tbf {
+        tbf_bucket_t **bucket;
+} tbf_t;
 
-br_tbf_t *
-br_tbf_init (br_tbf_opspec_t *, unsigned int);
+tbf_t *
+tbf_init (tbf_opspec_t *, unsigned int);
 
 int
-br_tbf_mod (br_tbf_t *, br_tbf_opspec_t *);
+tbf_mod (tbf_t *, tbf_opspec_t *);
 
 void
-br_tbf_throttle (br_tbf_t *, br_tbf_ops_t, unsigned long);
+tbf_throttle (tbf_t *, tbf_ops_t, unsigned long);
 
-#define TBF_THROTTLE_BEGIN(tbf, op, tokens) (br_tbf_throttle (tbf, op, tokens))
-#define TBF_THROTTLE_END(tbf, op, tokens) (void)
+#define TBF_THROTTLE_BEGIN(tbf, op, tokens) (tbf_throttle (tbf, op, tokens))
+#define TBF_THROTTLE_END(tbf, op, tokens)
 
-#endif /** __BIT_ROT_TBF_H__ */
+#endif /** THROTTLE_TBF_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/Makefile.am b/xlators/features/bit-rot/src/bitd/Makefile.am
index cabdf3cd224..3b2b53ec9c9 100644
--- a/xlators/features/bit-rot/src/bitd/Makefile.am
+++ b/xlators/features/bit-rot/src/bitd/Makefile.am
@@ -9,12 +9,12 @@ AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
                  -I$(CONTRIBDIR)/timer-wheel \
                  -I$(top_srcdir)/xlators/features/bit-rot/src/stub
 
-bit_rot_la_SOURCES = bit-rot.c bit-rot-scrub.c bit-rot-tbf.c bit-rot-ssm.c \
+bit_rot_la_SOURCES = bit-rot.c bit-rot-scrub.c bit-rot-ssm.c \
 		     bit-rot-scrub-status.c
 bit_rot_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
                     $(top_builddir)/xlators/features/changelog/lib/src/libgfchangelog.la
 
-noinst_HEADERS = bit-rot.h bit-rot-scrub.h bit-rot-tbf.h bit-rot-bitd-messages.h bit-rot-ssm.h \
+noinst_HEADERS = bit-rot.h bit-rot-scrub.h bit-rot-bitd-messages.h bit-rot-ssm.h \
 		 bit-rot-scrub-status.h
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
index e76bfcb6f90..c09213880ab 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
@@ -1940,7 +1940,7 @@ br_scrubber_init (xlator_t *this, br_private_t *priv)
         struct br_scrubber *fsscrub = NULL;
         int                 ret     = 0;
 
-        priv->tbf = br_tbf_init (NULL, 0);
+        priv->tbf = tbf_init (NULL, 0);
         if (!priv->tbf)
                 return -1;
 
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c
index 46bef085404..99a28672501 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.c
@@ -283,7 +283,7 @@ br_object_read_block_and_sign (xlator_t *this, fd_t *fd, br_child_t *child,
                                off_t offset, size_t size, SHA256_CTX *sha256)
 {
         int32_t        ret    = -1;
-        br_tbf_t      *tbf    = NULL;
+        tbf_t      *tbf    = NULL;
         struct iovec  *iovec  = NULL;
         struct iobref *iobref = NULL;
         br_private_t  *priv   = NULL;
@@ -316,12 +316,12 @@ br_object_read_block_and_sign (xlator_t *this, fd_t *fd, br_child_t *child,
                 goto out;
 
         for (i = 0; i < count; i++) {
-                TBF_THROTTLE_BEGIN (tbf, BR_TBF_OP_HASH, iovec[i].iov_len);
+                TBF_THROTTLE_BEGIN (tbf, TBF_OP_HASH, iovec[i].iov_len);
                 {
                         SHA256_Update (sha256, (const unsigned char *)
                                        (iovec[i].iov_base), iovec[i].iov_len);
                 }
-                TBF_THROTTLE_BEGIN (tbf, BR_TBF_OP_HASH, iovec[i].iov_len);
+                TBF_THROTTLE_BEGIN (tbf, TBF_OP_HASH, iovec[i].iov_len);
         }
 
  out:
@@ -1760,14 +1760,32 @@ static int32_t
 br_rate_limit_signer (xlator_t *this, int child_count, int numbricks)
 {
         br_private_t *priv = NULL;
-        br_tbf_opspec_t spec = {0,};
+        tbf_opspec_t spec = {0,};
 
         priv = this->private;
 
-        spec.op       = BR_TBF_OP_HASH;
+        spec.op       = TBF_OP_HASH;
         spec.rate     = 0;
         spec.maxlimit = 0;
 
+/**
+ * OK. Most implementations of TBF I've come across generate tokens
+ * every second (UML, etc..) and some chose sub-second granularity
+ * (blk-iothrottle cgroups). TBF algorithm itself does not enforce
+ * any logic for choosing generation interval and it seems pretty
+ * logical as one could jack up token count per interval w.r.t.
+ * generation rate.
+ *
+ * Value used here is chosen based on a series of test(s) performed
+ * to balance object signing time and not maxing out on all available
+ * CPU cores. It's obvious to have seconds granularity and jack up
+ * token count per interval, thereby achieving close to similar
+ * results. Let's stick to this as it seems to be working fine for
+ * the set of ops that are throttled.
+ **/
+        spec.token_gen_interval = 600000; /* In usec */
+
+
 #ifdef BR_RATE_LIMIT_SIGNER
 
         double contribution = 0;
@@ -1787,7 +1805,7 @@ br_rate_limit_signer (xlator_t *this, int child_count, int numbricks)
                         "[Rate Limit Info] \"tokens/sec (rate): %lu, "
                         "maxlimit: %lu\"", spec.rate, spec.maxlimit);
 
-        priv->tbf = br_tbf_init (&spec, 1);
+        priv->tbf = tbf_init (&spec, 1);
         return priv->tbf ? 0 : -1;
 }
 
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.h b/xlators/features/bit-rot/src/bitd/bit-rot.h
index 656936956a2..c6ad494524c 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.h
@@ -26,7 +26,7 @@
 #include "changelog.h"
 #include "timer-wheel.h"
 
-#include "bit-rot-tbf.h"
+#include "throttle-tbf.h"
 #include "bit-rot-ssm.h"
 
 #include "bit-rot-common.h"
@@ -214,7 +214,7 @@ struct br_private {
 
         uint32_t expiry_time;              /* objects "wait" time */
 
-        br_tbf_t *tbf;                    /* token bucket filter */
+        tbf_t *tbf;                    /* token bucket filter */
 
         gf_boolean_t iamscrubber;         /* function as a fs scrubber */
 
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
index f70fafbca49..a33577cf598 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
@@ -22,10 +22,6 @@ enum br_mem_types {
         gf_br_mt_br_child_t,
         gf_br_mt_br_object_t,
         gf_br_mt_br_ob_n_wk_t,
-        gf_br_mt_br_tbf_t,
-        gf_br_mt_br_tbf_bucket_t,
-        gf_br_mt_br_tbf_throttle_t,
-        gf_br_mt_br_tbf_opspec_t,
         gf_br_mt_br_scrubber_t,
         gf_br_mt_br_fsscan_entry_t,
         gf_br_stub_mt_br_stub_fd_t,