io-threads: Allow scaling to be switched off

The default is also to provide no scaling. For both, ordered and
unordered request pools, when scaling is off, we maintain atleast the
minimum number of threads specified in the volfile.

Signed-off-by: Anand V. Avati <avati@amp.gluster.com>

2 files changed, 73 insertions, 11 deletions

diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c
index 1d255d97267..fbd002267f3 100644
--- a/xlators/performance/io-threads/src/io-threads.c
+++ b/xlators/performance/io-threads/src/io-threads.c
@@ -41,6 +41,30 @@ void iot_startup_workers (iot_worker_t **workers, int start_idx, int count,
 void * iot_worker_unordered (void *arg);
 void * iot_worker_ordered (void *arg);
 
+int
+iot_unordered_request_balancer (iot_conf_t *conf)
+{
+        long int        rand = 0;
+        int             idx = 0;
+
+        /* Decide which thread will service the request.
+         * FIXME: This should change into some form of load-balancing.
+         * */
+        rand = random ();
+
+        /* If scaling is on, we can choose from any thread
+        * that has been allocated upto, max_o_threads, but
+        * with scaling off, we'll never have threads more
+        * than min_o_threads.
+        */
+        if (iot_unordered_scaling_on (conf))
+                idx = (rand % conf->max_u_threads);
+        else
+                idx = (rand % conf->min_u_threads);
+
+        return idx;
+}
+
 void
 iot_schedule_unordered (iot_conf_t *conf,
                 inode_t *inode,
@@ -50,10 +74,7 @@ iot_schedule_unordered (iot_conf_t *conf,
         iot_worker_t    *selected_worker = NULL;
         iot_request_t   *req = NULL;
 
-        /* First decide which thread will service the request.
-         * FIXME: This should change into some form of load-balancing.
-         * */
-        idx = (random() % conf->max_u_threads);
+        idx = iot_unordered_request_balancer (conf);
         selected_worker = conf->uworkers[idx];
 
         req = iot_init_request (stub);
@@ -73,6 +94,33 @@ iot_schedule_unordered (iot_conf_t *conf,
         pthread_mutex_unlock (&selected_worker->qlock);
 }
 
+/* Assumes inode lock is held. */
+int
+iot_ordered_request_balancer (iot_conf_t *conf,
+                inode_t *inode)
+{
+        int             ctxret = 0;
+        long int        rand = 0;
+        uint64_t        idx = 0;
+
+        ctxret = __inode_ctx_get (inode, conf->this, &idx);
+        if (ctxret < 0) {
+                rand = random ();
+                /* If scaling is on, we can choose from any thread
+                 * that has been allocated upto, max_o_threads, but
+                 * with scaling off, we'll never have threads more
+                 * than min_o_threads.
+                 */
+                if (iot_ordered_scaling_on (conf))
+                        idx = (rand % conf->max_o_threads);
+                else
+                        idx = (rand % conf->min_o_threads);
+                __inode_ctx_put (inode, conf->this, idx);
+        }
+
+        return idx;
+}
+
 void
 iot_schedule_ordered (iot_conf_t *conf,
                 inode_t *inode,
@@ -81,7 +129,6 @@ iot_schedule_ordered (iot_conf_t *conf,
         uint64_t        idx = 0;
         iot_worker_t    *selected_worker = NULL;
         iot_request_t   * req = NULL;
-        int             ctxret = 0;
 
         if (inode == NULL) {
                 gf_log (conf->this->name, GF_LOG_ERROR,
@@ -91,11 +138,7 @@ iot_schedule_ordered (iot_conf_t *conf,
         req = iot_init_request (stub);
         LOCK (&inode->lock);
         {
-                ctxret = __inode_ctx_get (inode, conf->this, &idx);
-                if (ctxret < 0) {
-                        idx = (random () % conf->max_o_threads);
-                        __inode_ctx_put (inode, conf->this, idx);
-                }
+                idx = iot_ordered_request_balancer (conf, inode);
                 /* inode lock once acquired, cannot be left here
                  * because other gluster main threads might be
                  * contending on it to append a request for this file.
@@ -1762,12 +1805,14 @@ init (xlator_t *this)
         conf->max_u_threads = IOT_MAX_THREADS;
         conf->min_u_threads = IOT_MIN_THREADS;
         conf->u_idle_time = IOT_DEFAULT_IDLE;
+        conf->u_scaling = IOT_SCALING_OFF;
 
         /* Init params for ordered workers. */
         pthread_mutex_init (&conf->otlock, NULL);
         conf->max_o_threads = IOT_MAX_THREADS;
         conf->min_o_threads = IOT_MIN_THREADS;
         conf->o_idle_time = IOT_DEFAULT_IDLE;
+        conf->o_scaling = IOT_SCALING_OFF;
 
         conf->this = this;
 	workers_init (conf);
diff --git a/xlators/performance/io-threads/src/io-threads.h b/xlators/performance/io-threads/src/io-threads.h
index 412b3cc7638..a613884da45 100644
--- a/xlators/performance/io-threads/src/io-threads.h
+++ b/xlators/performance/io-threads/src/io-threads.h
@@ -60,6 +60,11 @@ struct iot_request {
 #define IOT_MIN_THREADS         32
 #define IOT_MAX_THREADS         512
 
+#define IOT_SCALING_OFF                 1
+#define IOT_SCALING_ON                  2
+#define iot_ordered_scaling_on(conf)    ((conf)->o_scaling == IOT_SCALING_ON)
+#define iot_unordered_scaling_on(conf)  ((conf)->u_scaling == IOT_SCALING_ON)
+
 struct iot_worker {
   struct list_head rqlist;      /* List of requests assigned to me. */
   struct iot_conf *conf;
@@ -98,7 +103,12 @@ struct iot_conf {
   int o_idle_time;              /* in Secs. The idle time after which an
                                    ordered thread exits.
                                    */
-
+  int o_scaling;                /* Set to IOT_SCALING_OFF if user does not want
+                                   thread scaling on ordered threads.
+                                   If scaling is off, io-threads maintains
+                                   at least min_o_threads number of threads
+                                   and never lets any thread exit.
+                                   */
   struct iot_worker **oworkers; /* Ordered thread pool. */
 
 
@@ -114,6 +124,13 @@ struct iot_conf {
                                    request for this amount of secs, it should
                                    try to die.
                                    */
+  int u_scaling;                /* Set to IOT_SCALING_OFF if user does not want
+                                   thread scaling on unordered threads.
+                                   If scaling is off, io-threads maintains
+                                   at least min_u_threads number of threads
+                                   and never lets any thread exit.
+                                   */
+
 };
 
 typedef struct iot_conf iot_conf_t;