epoll: Adding the ability to configure epoll threads

Add the ability to configure the number of event threads
for various gluster services.

Currently with the multi thread epoll patch, it is possible
to have more than one thread waiting on socket activity and
processing the same. This thread count is currently static,
which this commit makes dynamic.

The current services which use IO path, i.e brick processes,
any client process (nfs, FUSE, gfapi, heal,
rebalance, etc.a), gain 2 set parameters to control the number
of threads that are processing events. These settings are,
  - client.event-threads <n>
  - server.event-threads <n>

The client setting affects the client graph consumers, and the
server setting affects the brick processes. These are processed
and inited/reconfigured using the client/server protocol xlators.

Other services (say glusterd) would need to extend similar
configuration settings to take advantage of multi threaded event
processing.

At present glusterd is not enabled with this commit, as it does not
stand to gain from this multi-threading (as I understand it).

Change-Id: Id8422fc57a9f95a135158eb6477ccf9d3c9ea4d9
BUG: 1104462
Signed-off-by: Shyam <srangana@redhat.com>
Reviewed-on: http://review.gluster.org/9488
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

5 files changed, 95 insertions, 0 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 1d015a94698..e9473658176 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -836,6 +836,10 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .type        = NO_DOC,
           .op_version  = GD_OP_VERSION_3_7_0,
         },
+        { .key         = "client.event-threads",
+          .voltype     = "protocol/client",
+          .op_version  = GD_OP_VERSION_3_7_0,
+        },
 
         /* Server xlator options */
         { .key         = "network.tcp-window-size",
@@ -939,6 +943,10 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .type        = NO_DOC,
           .op_version  = GD_OP_VERSION_3_7_0,
         },
+        { .key         = "server.event-threads",
+          .voltype     = "protocol/server",
+          .op_version  = GD_OP_VERSION_3_7_0,
+        },
 
         /* Generic transport options */
         { .key         = SSL_CERT_DEPTH_OPT,
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index fbd0ff22737..999a4a5c836 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -20,6 +20,7 @@
 #include "glusterfs.h"
 #include "statedump.h"
 #include "compat-errno.h"
+#include "event.h"
 
 #include "xdr-rpc.h"
 #include "glusterfs3.h"
@@ -2513,6 +2514,23 @@ out:
 }
 
 int
+client_check_event_threads (xlator_t *this, dict_t *options, clnt_conf_t *conf)
+{
+        int          ret = -1;
+        int          eventthreads = 0;
+
+        /* Read event-threads from the new configuration */
+        ret = dict_get_int32 (options, "event-threads", &eventthreads);
+        if (!ret) {
+                conf->event_threads = eventthreads;
+        }
+        ret = event_reconfigure_threads (this->ctx->event_pool,
+                                         conf->event_threads);
+
+        return ret;
+}
+
+int
 reconfigure (xlator_t *this, dict_t *options)
 {
 	clnt_conf_t *conf              = NULL;
@@ -2531,6 +2549,10 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("ping-timeout", conf->opt.ping_timeout,
                           options, int32, out);
 
+        ret = client_check_event_threads (this, options, conf);
+        if (ret)
+                goto out;
+
         ret = client_check_remote_host (this, options);
         if (ret)
                 goto out;
@@ -2609,6 +2631,13 @@ init (xlator_t *this)
         conf->grace_timer        = NULL;
         conf->grace_timer_needed = _gf_true;
 
+        /* Set event threads to a default */
+        conf->event_threads = STARTING_EVENT_THREADS;
+
+        ret = client_check_event_threads (this, this->options, conf);
+        if (ret)
+                goto out;
+
         ret = client_init_grace_timer (this, this->options, conf);
         if (ret)
                 goto out;
@@ -2936,5 +2965,15 @@ struct volume_options options[] = {
           .type  = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
         },
+        { .key   = {"event-threads"},
+          .type  = GF_OPTION_TYPE_INT,
+          .min   = 1,
+          .max   = 32,
+          .default_value = "2",
+          .description = "Specifies the number of event threads to execute in"
+                         "in parallel. Larger values would help process"
+                         " responses faster, depending on available processing"
+                         " power. Range 1-32 threads."
+        },
         { .key   = {NULL} },
 };
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index b4809310939..af70926b178 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -125,6 +125,9 @@ typedef struct clnt_conf {
         uint64_t               setvol_count;
 
         gf_boolean_t           send_gids; /* let the server resolve gids */
+
+        int                     event_threads; /* # of event threads
+                                                * configured */
 } clnt_conf_t;
 
 typedef struct _client_fd_ctx {
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index 7a2b7fa3297..92113c7c28b 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -25,6 +25,7 @@
 #include "statedump.h"
 #include "defaults.h"
 #include "authenticate.h"
+#include "event.h"
 
 void
 grace_time_handler (void *data)
@@ -674,6 +675,24 @@ out:
 }
 
 int
+server_check_event_threads (xlator_t *this, dict_t *options,
+                            server_conf_t *conf)
+{
+        int              ret = -1;
+        int              eventthreads = 0;
+
+        /* Read event-threads from the new configuration */
+        ret = dict_get_int32 (options, "event-threads", &eventthreads);
+        if (!ret) {
+                conf->event_threads = eventthreads;
+        }
+        ret = event_reconfigure_threads (this->ctx->event_pool,
+                                         conf->event_threads);
+
+        return ret;
+}
+
+int
 reconfigure (xlator_t *this, dict_t *options)
 {
 
@@ -693,6 +712,7 @@ reconfigure (xlator_t *this, dict_t *options)
                 gf_log_callingfn (this->name, GF_LOG_DEBUG, "conf == null!!!");
                 goto out;
         }
+
         if (dict_get_int32 ( options, "inode-lru-limit", &inode_lru_limit) == 0){
                 conf->inode_lru_limit = inode_lru_limit;
                 gf_log (this->name, GF_LOG_TRACE, "Reconfigured inode-lru-limit"
@@ -790,6 +810,11 @@ reconfigure (xlator_t *this, dict_t *options)
                                         "Reconfigure not found for transport" );
                 }
         }
+
+        ret = server_check_event_threads (this, options, conf);
+        if (ret)
+                goto out;
+
         ret = server_init_grace_timer (this, options, conf);
 
 out:
@@ -846,6 +871,13 @@ init (xlator_t *this)
         INIT_LIST_HEAD (&conf->xprt_list);
         pthread_mutex_init (&conf->mutex, NULL);
 
+         /* Set event threads to a default */
+        conf->event_threads = STARTING_EVENT_THREADS;
+
+        ret = server_check_event_threads (this, this->options, conf);
+        if (ret)
+                goto out;
+
         ret = server_init_grace_timer (this, this->options, conf);
         if (ret)
                 goto out;
@@ -1199,6 +1231,16 @@ struct volume_options options[] = {
           .default_value = "2",
           .description = "Timeout in seconds for the cached groups to expire."
         },
+        { .key   = {"event-threads"},
+          .type  = GF_OPTION_TYPE_INT,
+          .min   = 1,
+          .max   = 32,
+          .default_value = "2",
+          .description = "Specifies the number of event threads to execute in"
+                         "in parallel. Larger values would help process"
+                         " responses faster, depending on available processing"
+                         " power. Range 1-32 threads."
+        },
 
         { .key   = {NULL} },
 };
diff --git a/xlators/protocol/server/src/server.h b/xlators/protocol/server/src/server.h
index 3e1feacb94b..dc64edd0ab2 100644
--- a/xlators/protocol/server/src/server.h
+++ b/xlators/protocol/server/src/server.h
@@ -63,6 +63,9 @@ struct server_conf {
         gf_boolean_t            server_manage_gids; /* resolve gids on brick */
         gid_cache_t             gid_cache;
         int32_t                 gid_cache_timeout;
+
+        int                     event_threads; /* # of event threads
+                                                * configured */
 };
 typedef struct server_conf server_conf_t;