performance/io-threads: Provide option to turn off least-priority

RCA:
In cases when self-heal is in progress, self-heal fops are starved
because of least-priority. This affects other fops with conflicting
inode, entry locks with self-heal.

Fix:
This patch provides configuring enable/disable of least-priority.

Additional changes:
Moved RCHECKSUM fop to low instead of least because it will still
affect the performance of other fops if RCHECKSUM is in LEAST
priority.

Tests:
Tested that the enabling/disabling of fops is working fine.
Tested that RCHECKSUM fop priority is assigned LOW when
least-priority is disabled.

BUG: 843704
Signed-off-by: Pranith Kumar K <pranithk@gluster.com>
Change-Id: I892f99d6d0a3e0ae6c0a280f82e2203af0c346f6
Reviewed-on: http://review.gluster.com/3751
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

3 files changed, 19 insertions, 10 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 9bcf2cc78..7cd97ed5d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -154,11 +154,12 @@ static struct volopt_map_entry glusterd_volopt_map[] = {
         {"performance.cache-size",               "performance/quick-read",    NULL, NULL, NO_DOC, 0 },
         {"performance.flush-behind",             "performance/write-behind",  "flush-behind", NULL, DOC, 0},
 
-        {"performance.io-thread-count",          "performance/io-threads",    "thread-count", DOC, 0},
-        {"performance.high-prio-threads",        "performance/io-threads",    NULL, DOC, 0},
-        {"performance.normal-prio-threads",      "performance/io-threads",    NULL, DOC, 0},
-        {"performance.low-prio-threads",         "performance/io-threads",    NULL, DOC, 0},
-        {"performance.least-prio-threads",       "performance/io-threads",    NULL, DOC, 0},
+        {"performance.io-thread-count",          "performance/io-threads",    "thread-count", NULL, DOC, 0},
+        {"performance.high-prio-threads",        "performance/io-threads",    NULL, NULL, DOC, 0},
+        {"performance.normal-prio-threads",      "performance/io-threads",    NULL, NULL, DOC, 0},
+        {"performance.low-prio-threads",         "performance/io-threads",    NULL, NULL, DOC, 0},
+        {"performance.least-prio-threads",       "performance/io-threads",    NULL, NULL, DOC, 0},
+        {"performance.enable-least-priority",    "performance/io-threads",    NULL, NULL, DOC, 0},
         {"performance.disk-usage-limit",         "performance/quota",         NULL, NULL, NO_DOC, 0},
         {"performance.min-free-disk-limit",      "performance/quota",         NULL, NULL, NO_DOC, 0},
         {"performance.write-behind-window-size", "performance/write-behind",  "cache-size", NULL, DOC},
diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c
index 97109328f..7230c2eda 100644
--- a/xlators/performance/io-threads/src/io-threads.c
+++ b/xlators/performance/io-threads/src/io-threads.c
@@ -192,8 +192,9 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
 {
         int             ret = -1;
         iot_pri_t       pri = IOT_PRI_MAX - 1;
+        iot_conf_t      *conf = this->private;
 
-        if (frame->root->pid < GF_CLIENT_PID_MAX) {
+        if ((frame->root->pid < GF_CLIENT_PID_MAX) && conf->least_priority) {
                 pri = IOT_PRI_LEAST;
                 goto out;
         }
@@ -245,11 +246,8 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
         case GF_FOP_FSYNCDIR:
         case GF_FOP_XATTROP:
         case GF_FOP_FXATTROP:
-                pri = IOT_PRI_LO;
-                break;
-
         case GF_FOP_RCHECKSUM:
-                pri = IOT_PRI_LEAST;
+                pri = IOT_PRI_LO;
                 break;
 
         case GF_FOP_NULL:
@@ -2470,6 +2468,8 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("least-prio-threads",
                           conf->ac_iot_limit[IOT_PRI_LEAST], options, int32,
                           out);
+        GF_OPTION_RECONF ("enable-least-priority", conf->least_priority,
+                          options, bool, out);
 
 	ret = 0;
 out:
@@ -2532,6 +2532,8 @@ init (xlator_t *this)
                         conf->ac_iot_limit[IOT_PRI_LEAST], int32, out);
 
         GF_OPTION_INIT ("idle-time", conf->idle_time, int32, out);
+        GF_OPTION_INIT ("enable-least-priority", conf->least_priority,
+                        bool, out);
 
         conf->this = this;
 
@@ -2661,6 +2663,11 @@ struct volume_options options[] = {
           .description = "Max number of threads in IO threads translator which "
                          "perform least priority IO operations at a given time"
 	},
+        { .key  = {"enable-least-priority"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Enable/Disable least priority"
+        },
         {.key   = {"idle-time"},
          .type  = GF_OPTION_TYPE_INT,
          .min   = 1,
diff --git a/xlators/performance/io-threads/src/io-threads.h b/xlators/performance/io-threads/src/io-threads.h
index 7fd6cfea4..feac5ae73 100644
--- a/xlators/performance/io-threads/src/io-threads.h
+++ b/xlators/performance/io-threads/src/io-threads.h
@@ -70,6 +70,7 @@ struct iot_conf {
         int                  queue_sizes[IOT_PRI_MAX];
         int                  queue_size;
         pthread_attr_t       w_attr;
+        gf_boolean_t         least_priority; /*Enable/Disable least-priority */
 
         xlator_t            *this;
 };