core: run many bricks within one glusterfsd process

This patch adds support for multiple brick translator stacks running
in a single brick server process.  This reduces our per-brick memory usage by
approximately 3x, and our appetite for TCP ports even more.  It also creates
potential to avoid process/thread thrashing, and to improve QoS by scheduling
more carefully across the bricks, but realizing that potential will require
further work.

Multiplexing is controlled by the "cluster.brick-multiplex" global option.  By
default it's off, and bricks are started in separate processes as before.  If
multiplexing is enabled, then *compatible* bricks (mostly those with the same
transport options) will be started in the same process.

Change-Id: I45059454e51d6f4cbb29a4953359c09a408695cb
BUG: 1385758
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-on: https://review.gluster.org/14763
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 69 insertions, 83 deletions

diff --git a/xlators/protocol/server/src/server-handshake.c b/xlators/protocol/server/src/server-handshake.c
index a33efb8c33a..249dde7de76 100644
--- a/xlators/protocol/server/src/server-handshake.c
+++ b/xlators/protocol/server/src/server-handshake.c
@@ -36,27 +36,6 @@ gf_compare_client_version (rpcsvc_request_t *req, int fop_prognum,
         return ret;
 }
 
-void __check_and_set (xlator_t *each, void *data)
-{
-        if (!strcmp (each->name,
-                     ((struct __get_xl_struct *) data)->name))
-                ((struct __get_xl_struct *) data)->reply = each;
-}
-
-static xlator_t *
-get_xlator_by_name (xlator_t *some_xl, const char *name)
-{
-        struct __get_xl_struct get = {
-                .name = name,
-                .reply = NULL
-        };
-
-        xlator_foreach (some_xl, __check_and_set, &get);
-
-        return get.reply;
-}
-
-
 int
 _volfile_update_checksum (xlator_t *this, char *key, uint32_t checksum)
 {
@@ -426,13 +405,14 @@ server_setvolume (rpcsvc_request_t *req)
         int32_t              ret           = -1;
         int32_t              op_ret        = -1;
         int32_t              op_errno      = EINVAL;
-        int32_t              fop_version   = 0;
-        int32_t              mgmt_version  = 0;
         uint32_t             lk_version    = 0;
         char                *buf           = NULL;
         gf_boolean_t        cancelled      = _gf_false;
         uint32_t            opversion      = 0;
         rpc_transport_t     *xprt          = NULL;
+        int32_t              fop_version   = 0;
+        int32_t              mgmt_version  = 0;
+
 
         params = dict_new ();
         reply  = dict_new ();
@@ -446,32 +426,6 @@ server_setvolume (rpcsvc_request_t *req)
 
         this = req->svc->xl;
 
-        config_params = dict_copy_with_ref (this->options, NULL);
-        conf          = this->private;
-
-        if (conf->parent_up == _gf_false) {
-                /* PARENT_UP indicates that all xlators in graph are inited
-                 * successfully
-                 */
-                op_ret = -1;
-                op_errno = EAGAIN;
-
-                ret = dict_set_str (reply, "ERROR",
-                                    "xlator graph in server is not initialised "
-                                    "yet. Try again later");
-                if (ret < 0)
-                        gf_msg_debug (this->name, 0, "failed to set error: "
-                                      "xlator graph in server is not "
-                                      "initialised yet. Try again later");
-                goto fail;
-        }
-
-        ret = dict_set_int32 (reply, "child_up", conf->child_up);
-        if (ret < 0)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        PS_MSG_DICT_GET_FAILED, "Failed to set 'child_up' "
-                        "in the reply dict");
-
         buf = memdup (args.dict.dict_val, args.dict.dict_len);
         if (buf == NULL) {
                 op_ret = -1;
@@ -497,6 +451,65 @@ server_setvolume (rpcsvc_request_t *req)
         params->extra_free = buf;
         buf = NULL;
 
+        ret = dict_get_str (params, "remote-subvolume", &name);
+        if (ret < 0) {
+                ret = dict_set_str (reply, "ERROR",
+                                    "No remote-subvolume option specified");
+                if (ret < 0)
+                        gf_msg_debug (this->name, 0, "failed to set error "
+                                      "msg");
+
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto fail;
+        }
+
+        xl = get_xlator_by_name (this, name);
+        if (xl == NULL) {
+                ret = gf_asprintf (&msg, "remote-subvolume \"%s\" is not found",
+                                   name);
+                if (-1 == ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                PS_MSG_ASPRINTF_FAILED,
+                                "asprintf failed while setting error msg");
+                        goto fail;
+                }
+                ret = dict_set_dynstr (reply, "ERROR", msg);
+                if (ret < 0)
+                        gf_msg_debug (this->name, 0, "failed to set error "
+                                      "msg");
+
+                op_ret = -1;
+                op_errno = ENOENT;
+                goto fail;
+        }
+
+        config_params = dict_copy_with_ref (xl->options, NULL);
+        conf          = this->private;
+
+        if (conf->parent_up == _gf_false) {
+                /* PARENT_UP indicates that all xlators in graph are inited
+                 * successfully
+                 */
+                op_ret = -1;
+                op_errno = EAGAIN;
+
+                ret = dict_set_str (reply, "ERROR",
+                                    "xlator graph in server is not initialised "
+                                    "yet. Try again later");
+                if (ret < 0)
+                        gf_msg_debug (this->name, 0, "failed to set error: "
+                                      "xlator graph in server is not "
+                                      "initialised yet. Try again later");
+                goto fail;
+        }
+
+        ret = dict_set_int32 (reply, "child_up", conf->child_up);
+        if (ret < 0)
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        PS_MSG_DICT_GET_FAILED, "Failed to set 'child_up' "
+                        "in the reply dict");
+
         ret = dict_get_str (params, "process-uuid", &client_uid);
         if (ret < 0) {
                 ret = dict_set_str (reply, "ERROR",
@@ -603,39 +616,6 @@ server_setvolume (rpcsvc_request_t *req)
                 goto fail;
         }
 
-        ret = dict_get_str (params, "remote-subvolume", &name);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "No remote-subvolume option specified");
-                if (ret < 0)
-                        gf_msg_debug (this->name, 0, "failed to set error "
-                                      "msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        xl = get_xlator_by_name (this, name);
-        if (xl == NULL) {
-                ret = gf_asprintf (&msg, "remote-subvolume \"%s\" is not found",
-                                   name);
-                if (-1 == ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                PS_MSG_ASPRINTF_FAILED,
-                                "asprintf failed while setting error msg");
-                        goto fail;
-                }
-                ret = dict_set_dynstr (reply, "ERROR", msg);
-                if (ret < 0)
-                        gf_msg_debug (this->name, 0, "failed to set error "
-                                      "msg");
-
-                op_ret = -1;
-                op_errno = ENOENT;
-                goto fail;
-        }
-
         if (conf->verify_volfile) {
                 ret = dict_get_uint32 (params, "volfile-checksum", &checksum);
                 if (ret == 0) {
@@ -850,7 +830,13 @@ fail:
 
         dict_unref (params);
         dict_unref (reply);
-        dict_unref (config_params);
+        if (config_params) {
+                /*
+                 * This might be null if we couldn't even find the translator
+                 * (brick) to copy it from.
+                 */
+                dict_unref (config_params);
+        }
 
         GF_FREE (buf);