performance/io-threads: expose io-thread queue depths

The following release-3.8-fb branch patch is upstreamed:

> io-stats: Expose io-thread queue depths
> Commit ID: 69509ee7d2
> https://review.gluster.org/#/c/18143/
> By Shreyas Siravara <sshreyas@fb.com>

Changes in this patch:
- Replace iot_pri_t with gf_fop_pri_t
- Replace IOT_PRI_{HI, LO, NORMAL, MAX, LEAST} with
  GF_FOP_PRI_{HI, LO, NORMAL, MAX, LEAST}
- Use dict_unref() instead of dict_destroy()

This patch is required to forward port io-threads namespace patch.
Updates: #401
Change-Id: I1b47a63185a441a30fbc423ca1015df7b36c2518
Signed-off-by: Varsha Rao <varao@redhat.com>

6 files changed, 111 insertions, 47 deletions

diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index 75b2bf8e22c..cdd22475cbe 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -1648,6 +1648,15 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
                 return 0;
 
         /*
+         * Heal daemons don't have IO threads ... and as a result they
+         * send this getxattr down and eventually crash :(
+         */
+        if (strcmp (name, IO_THREADS_QUEUE_SIZE_KEY) == 0) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        /*
          * Special xattrs which need responses from all subvols
          */
         if (afr_is_special_xattr (name, &cbk, 0)) {
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 5717650dd12..445bd590c4d 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -4562,7 +4562,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
 
         xlator_t     *subvol        = NULL;
         xlator_t     *hashed_subvol = NULL;
-        xlator_t     *mds_subvol = NULL;
+        xlator_t     *mds_subvol    = NULL;
         xlator_t     *cached_subvol = NULL;
         dht_conf_t   *conf          = NULL;
         dht_local_t  *local         = NULL;
@@ -4572,6 +4572,9 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
         int           cnt           = 0;
         char         *node_uuid_key = NULL;
         int           ret           = -1;
+
+        GF_CHECK_XATTR_KEY_AND_GOTO (key, IO_THREADS_QUEUE_SIZE_KEY,
+                                     op_errno, err);
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index 4ea45b058ae..985c5fbc389 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -36,6 +36,7 @@
 #include "logging.h"
 #include "cli1-xdr.h"
 #include "statedump.h"
+#include "syncop.h"
 #include <pwd.h>
 #include <grp.h>
 #include "upcall-utils.h"
@@ -798,6 +799,8 @@ io_stats_dump_global_to_json_logfp (xlator_t *this,
         double                weighted_fop_ave_usec = 0.0;
         double                weighted_fop_ave_usec_sum = 0.0;
         long                  total_fop_hits = 0;
+        loc_t                 unused_loc = {0, };
+        dict_t                *xattr = NULL;
 
         interval_sec = ((now->tv_sec * 1000000.0 + now->tv_usec) -
                 (stats->started_at.tv_sec * 1000000.0 +
@@ -950,6 +953,30 @@ io_stats_dump_global_to_json_logfp (xlator_t *this,
                 }
         }
 
+        ret = syncop_getxattr (this, &unused_loc, &xattr,
+                               IO_THREADS_QUEUE_SIZE_KEY, NULL, NULL);
+        if (xattr) {
+                /*
+                 * Iterate over the dictionary returned to us by io-threads and
+                 * dump the results to the stats file.
+                 */
+                data_pair_t *curr = NULL;
+
+                dict_for_each (xattr, curr) {
+                        ios_log (this, logfp,
+                                 "\"%s.%s.%s.queue_size\": \"%d\",",
+                                 key_prefix, str_prefix, curr->key,
+                                 data_to_int32 (curr->value));
+                }
+
+                /* Free the dictionary */
+                dict_unref (xattr);
+        } else {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unable to get queue size counts from "
+                        "the io-threads translator!");
+        }
+
         if (interval == -1) {
                 ios_log (this, logfp, "\"%s.%s.uptime\": \"%"PRId64"\",",
                          key_prefix, str_prefix,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 31564751c9a..0c984c97192 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -4983,7 +4983,7 @@ static gf_boolean_t
 volgen_is_shd_compatible_xl (char *xl_type)
 {
         char            *shd_xls[] = {"cluster/replicate", "cluster/disperse",
-                                      NULL};
+                                      "debug/io-stats", NULL};
         if (gf_get_index_by_elem (shd_xls, xl_type) != -1)
                 return _gf_true;
 
diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c
index 80d0168bf4c..a5a9543ae39 100644
--- a/xlators/performance/io-threads/src/io-threads.c
+++ b/xlators/performance/io-threads/src/io-threads.c
@@ -56,10 +56,10 @@ iot_get_ctx (xlator_t *this, client_t *client)
         int                      i;
 
         if (client_ctx_get (client, this, (void **)&ctx) != 0) {
-                ctx = GF_CALLOC (IOT_PRI_MAX, sizeof(*ctx),
+                ctx = GF_CALLOC (GF_FOP_PRI_MAX, sizeof(*ctx),
                                  gf_iot_mt_client_ctx_t);
                 if (ctx) {
-                        for (i = 0; i < IOT_PRI_MAX; ++i) {
+                        for (i = 0; i < GF_FOP_PRI_MAX; ++i) {
                                 INIT_LIST_HEAD (&ctx[i].clients);
                                 INIT_LIST_HEAD (&ctx[i].reqs);
                         }
@@ -82,7 +82,7 @@ __iot_dequeue (iot_conf_t *conf, int *pri)
         iot_client_ctx_t        *ctx;
 
         *pri = -1;
-        for (i = 0; i < IOT_PRI_MAX; i++) {
+        for (i = 0; i < GF_FOP_PRI_MAX; i++) {
 
                 if (conf->ac_iot_count[i] >= conf->ac_iot_limit[i]) {
                         continue;
@@ -133,8 +133,8 @@ __iot_enqueue (iot_conf_t *conf, call_stub_t *stub, int pri)
         client_t                *client = stub->frame->root->client;
         iot_client_ctx_t        *ctx;
 
-        if (pri < 0 || pri >= IOT_PRI_MAX)
-                pri = IOT_PRI_MAX-1;
+        if (pri < 0 || pri >= GF_FOP_PRI_MAX)
+                pri = GF_FOP_PRI_MAX-1;
 
         if (client) {
                 ctx = iot_get_ctx (THIS, client);
@@ -252,25 +252,28 @@ do_iot_schedule (iot_conf_t *conf, call_stub_t *stub, int pri)
 }
 
 char*
-iot_get_pri_meaning (iot_pri_t pri)
+iot_get_pri_meaning (gf_fop_pri_t pri)
 {
         char    *name = NULL;
         switch (pri) {
-        case IOT_PRI_HI:
+        case GF_FOP_PRI_HI:
                 name = "fast";
                 break;
-        case IOT_PRI_NORMAL:
+        case GF_FOP_PRI_NORMAL:
                 name = "normal";
                 break;
-        case IOT_PRI_LO:
+        case GF_FOP_PRI_LO:
                 name = "slow";
                 break;
-        case IOT_PRI_LEAST:
+        case GF_FOP_PRI_LEAST:
                 name = "least priority";
                 break;
-        case IOT_PRI_MAX:
+        case GF_FOP_PRI_MAX:
                 name = "invalid";
                 break;
+        case GF_FOP_PRI_UNSPEC:
+                name = "unspecified";
+                break;
         }
         return name;
 }
@@ -279,11 +282,11 @@ int
 iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
 {
         int             ret = -1;
-        iot_pri_t       pri = IOT_PRI_MAX - 1;
+        gf_fop_pri_t       pri = GF_FOP_PRI_MAX - 1;
         iot_conf_t      *conf = this->private;
 
         if ((frame->root->pid < GF_CLIENT_PID_MAX) && conf->least_priority) {
-                pri = IOT_PRI_LEAST;
+                pri = GF_FOP_PRI_LEAST;
                 goto out;
         }
 
@@ -302,7 +305,7 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
         case GF_FOP_SETACTIVELK:
         case GF_FOP_ICREATE:
         case GF_FOP_NAMELINK:
-                pri = IOT_PRI_HI;
+                pri = GF_FOP_PRI_HI;
                 break;
 
         case GF_FOP_CREATE:
@@ -328,7 +331,7 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
         case GF_FOP_FSETXATTR:
         case GF_FOP_REMOVEXATTR:
         case GF_FOP_FREMOVEXATTR:
-                pri = IOT_PRI_NORMAL;
+                pri = GF_FOP_PRI_NORMAL;
                 break;
 
         case GF_FOP_READ:
@@ -344,7 +347,7 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
         case GF_FOP_DISCARD:
         case GF_FOP_ZEROFILL:
         case GF_FOP_SEEK:
-                pri = IOT_PRI_LO;
+                pri = GF_FOP_PRI_LO;
                 break;
 
         case GF_FOP_FORGET:
@@ -606,6 +609,36 @@ int
 iot_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
               const char *name, dict_t *xdata)
 {
+        iot_conf_t *conf = NULL;
+        dict_t     *depths = NULL;
+        int i = 0;
+
+        conf = this->private;
+
+        if (conf && name && strcmp (name, IO_THREADS_QUEUE_SIZE_KEY) == 0) {
+                /*
+                 * We explicitly do not want a reference count
+                 * for this dict in this translator
+                 */
+                depths = get_new_dict ();
+                if (!depths)
+                        goto unwind_special_getxattr;
+
+                for (i = 0; i < GF_FOP_PRI_MAX; i++) {
+                        if (dict_set_int32 (depths,
+                                            (char *)fop_pri_to_string (i),
+                                            conf->queue_sizes[i]) != 0) {
+                                dict_unref (depths);
+                                depths = NULL;
+                                goto unwind_special_getxattr;
+                        }
+                }
+
+unwind_special_getxattr:
+                STACK_UNWIND_STRICT (getxattr, frame, 0, 0, depths, xdata);
+                return 0;
+        }
+
         IOT_FOP (getxattr, frame, this, loc, name, xdata);
         return 0;
 }
@@ -793,7 +826,7 @@ __iot_workers_scale (iot_conf_t *conf)
         int       i = 0;
         char      thread_name[GF_THREAD_NAMEMAX] = {0,};
 
-        for (i = 0; i < IOT_PRI_MAX; i++)
+        for (i = 0; i < GF_FOP_PRI_MAX; i++)
                 scale += min (conf->queue_sizes[i], conf->ac_iot_limit[i]);
 
         if (scale < IOT_MIN_THREADS)
@@ -931,13 +964,13 @@ iot_priv_dump (xlator_t *this)
         gf_proc_dump_write("idle_time", "%d", conf->idle_time);
         gf_proc_dump_write("stack_size", "%zd", conf->stack_size);
         gf_proc_dump_write("high_priority_threads", "%d",
-                           conf->ac_iot_limit[IOT_PRI_HI]);
+                           conf->ac_iot_limit[GF_FOP_PRI_HI]);
         gf_proc_dump_write("normal_priority_threads", "%d",
-                           conf->ac_iot_limit[IOT_PRI_NORMAL]);
+                           conf->ac_iot_limit[GF_FOP_PRI_NORMAL]);
         gf_proc_dump_write("low_priority_threads", "%d",
-                           conf->ac_iot_limit[IOT_PRI_LO]);
+                           conf->ac_iot_limit[GF_FOP_PRI_LO]);
         gf_proc_dump_write("least_priority_threads", "%d",
-                           conf->ac_iot_limit[IOT_PRI_LEAST]);
+                           conf->ac_iot_limit[GF_FOP_PRI_LEAST]);
 
         return 0;
 }
@@ -955,17 +988,19 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("thread-count", conf->max_count, options, int32, out);
 
         GF_OPTION_RECONF ("high-prio-threads",
-                          conf->ac_iot_limit[IOT_PRI_HI], options, int32, out);
+                          conf->ac_iot_limit[GF_FOP_PRI_HI], options, int32,
+                          out);
 
         GF_OPTION_RECONF ("normal-prio-threads",
-                          conf->ac_iot_limit[IOT_PRI_NORMAL], options, int32,
+                          conf->ac_iot_limit[GF_FOP_PRI_NORMAL], options, int32,
                           out);
 
         GF_OPTION_RECONF ("low-prio-threads",
-                          conf->ac_iot_limit[IOT_PRI_LO], options, int32, out);
+                          conf->ac_iot_limit[GF_FOP_PRI_LO], options, int32,
+                          out);
 
         GF_OPTION_RECONF ("least-prio-threads",
-                          conf->ac_iot_limit[IOT_PRI_LEAST], options, int32,
+                          conf->ac_iot_limit[GF_FOP_PRI_LEAST], options, int32,
                           out);
         GF_OPTION_RECONF ("enable-least-priority", conf->least_priority,
                           options, bool, out);
@@ -1029,16 +1064,16 @@ init (xlator_t *this)
         GF_OPTION_INIT ("thread-count", conf->max_count, int32, out);
 
         GF_OPTION_INIT ("high-prio-threads",
-                        conf->ac_iot_limit[IOT_PRI_HI], int32, out);
+                        conf->ac_iot_limit[GF_FOP_PRI_HI], int32, out);
 
         GF_OPTION_INIT ("normal-prio-threads",
-                        conf->ac_iot_limit[IOT_PRI_NORMAL], int32, out);
+                        conf->ac_iot_limit[GF_FOP_PRI_NORMAL], int32, out);
 
         GF_OPTION_INIT ("low-prio-threads",
-                        conf->ac_iot_limit[IOT_PRI_LO], int32, out);
+                        conf->ac_iot_limit[GF_FOP_PRI_LO], int32, out);
 
         GF_OPTION_INIT ("least-prio-threads",
-                        conf->ac_iot_limit[IOT_PRI_LEAST], int32, out);
+                        conf->ac_iot_limit[GF_FOP_PRI_LEAST], int32, out);
 
         GF_OPTION_INIT ("idle-time", conf->idle_time, int32, out);
         GF_OPTION_INIT ("enable-least-priority", conf->least_priority,
@@ -1046,7 +1081,7 @@ init (xlator_t *this)
 
         conf->this = this;
 
-        for (i = 0; i < IOT_PRI_MAX; i++) {
+        for (i = 0; i < GF_FOP_PRI_MAX; i++) {
                 INIT_LIST_HEAD (&conf->clients[i]);
                 INIT_LIST_HEAD (&conf->no_client[i].clients);
                 INIT_LIST_HEAD (&conf->no_client[i].reqs);
diff --git a/xlators/performance/io-threads/src/io-threads.h b/xlators/performance/io-threads/src/io-threads.h
index 9648f74f39b..bd1c3f523c4 100644
--- a/xlators/performance/io-threads/src/io-threads.h
+++ b/xlators/performance/io-threads/src/io-threads.h
@@ -36,18 +36,8 @@ struct iot_conf;
 #define IOT_DEFAULT_THREADS     16
 #define IOT_MAX_THREADS         64
 
-
 #define IOT_THREAD_STACK_SIZE   ((size_t)(256*1024))
 
-
-typedef enum {
-        IOT_PRI_HI = 0, /* low latency */
-        IOT_PRI_NORMAL, /* normal */
-        IOT_PRI_LO,     /* bulk */
-        IOT_PRI_LEAST,  /* least */
-        IOT_PRI_MAX,
-} iot_pri_t;
-
 typedef struct {
         struct list_head        clients;
         struct list_head        reqs;
@@ -63,18 +53,18 @@ struct iot_conf {
 
         int32_t              idle_time;   /* in seconds */
 
-        struct list_head     clients[IOT_PRI_MAX];
+        struct list_head     clients[GF_FOP_PRI_MAX];
         /*
          * It turns out that there are several ways a frame can get to us
          * without having an associated client (server_first_lookup was the
          * first one I hit).  Instead of trying to update all such callers,
          * we use this to queue them.
          */
-        iot_client_ctx_t     no_client[IOT_PRI_MAX];
+        iot_client_ctx_t     no_client[GF_FOP_PRI_MAX];
 
-        int32_t              ac_iot_limit[IOT_PRI_MAX];
-        int32_t              ac_iot_count[IOT_PRI_MAX];
-        int                  queue_sizes[IOT_PRI_MAX];
+        int32_t              ac_iot_limit[GF_FOP_PRI_MAX];
+        int32_t              ac_iot_count[GF_FOP_PRI_MAX];
+        int                  queue_sizes[GF_FOP_PRI_MAX];
         int                  queue_size;
         pthread_attr_t       w_attr;
         gf_boolean_t         least_priority; /*Enable/Disable least-priority */