From a6fd403a15ffead93daba8a737cd7c6d6a943744 Mon Sep 17 00:00:00 2001 From: Amar Tumballi Date: Fri, 17 Jun 2011 06:10:53 +0000 Subject: fix multiple transport related portmap issues in client handshake Signed-off-by: Amar Tumballi Signed-off-by: Anand Avati BUG: 2258 (enhance gluster volume rebalance) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=2258 --- glusterfsd/src/glusterfsd-mgmt.c | 60 ++++++++++++++++++++++++++ glusterfsd/src/glusterfsd.c | 14 +++++- libglusterfs/src/glusterfs.h | 1 + xlators/mgmt/glusterd/src/glusterd-store.c | 16 ++++++- xlators/mgmt/glusterd/src/glusterd-store.h | 1 + xlators/mgmt/glusterd/src/glusterd-utils.c | 28 +++++++++--- xlators/mgmt/glusterd/src/glusterd.h | 1 + xlators/protocol/client/src/client-handshake.c | 21 +++++++++ xlators/protocol/client/src/client.h | 3 ++ 9 files changed, 137 insertions(+), 8 deletions(-) diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c index a6d8a4800..02b622f98 100644 --- a/glusterfsd/src/glusterfsd-mgmt.c +++ b/glusterfsd/src/glusterfsd-mgmt.c @@ -545,6 +545,41 @@ out: } +static int +mgmt_pmap_signin2_cbk (struct rpc_req *req, struct iovec *iov, int count, + void *myframe) +{ + pmap_signin_rsp rsp = {0,}; + call_frame_t *frame = NULL; + int ret = 0; + + frame = myframe; + + if (-1 == req->rpc_status) { + rsp.op_ret = -1; + rsp.op_errno = EINVAL; + goto out; + } + + ret = xdr_to_pmap_signin_rsp (*iov, &rsp); + if (ret < 0) { + gf_log (frame->this->name, GF_LOG_ERROR, "XDR decode error"); + rsp.op_ret = -1; + rsp.op_errno = EINVAL; + goto out; + } + + if (-1 == rsp.op_ret) { + gf_log (frame->this->name, GF_LOG_ERROR, + "failed to register the port with glusterd"); + goto out; + } +out: + STACK_DESTROY (frame->root); + return 0; + +} + static int mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count, void *myframe) @@ -552,6 +587,10 @@ mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count, pmap_signin_rsp rsp = {0,}; call_frame_t *frame = NULL; int ret = 0; + pmap_signin_req pmap_req = {0, }; + cmd_args_t *cmd_args = NULL; + glusterfs_ctx_t *ctx = NULL; + char brick_name[PATH_MAX] = {0,}; frame = myframe; @@ -574,6 +613,27 @@ mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count, "failed to register the port with glusterd"); goto out; } + + ctx = glusterfs_ctx_get (); + cmd_args = &ctx->cmd_args; + + if (!cmd_args->brick_port2) { + /* We are done with signin process */ + goto out; + } + + snprintf (brick_name, PATH_MAX, "%s.rdma", cmd_args->brick_name); + pmap_req.port = cmd_args->brick_port2; + pmap_req.brick = brick_name; + + ret = mgmt_submit_request (&pmap_req, frame, ctx, &clnt_pmap_prog, + GF_PMAP_SIGNIN, xdr_from_pmap_signin_req, + mgmt_pmap_signin2_cbk); + if (ret) + goto out; + + return 0; + out: STACK_DESTROY (frame->root); diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c index 87d16c7dc..f17ac814e 100644 --- a/glusterfsd/src/glusterfsd.c +++ b/glusterfsd/src/glusterfsd.c @@ -426,6 +426,8 @@ parse_opts (int key, char *arg, struct argp_state *state) gf_boolean_t b = _gf_false; char *pwd = NULL; char tmp_buf[2048] = {0,}; + char *tmp_str = NULL; + char *port_str = NULL; cmd_args = state->input; @@ -657,8 +659,18 @@ parse_opts (int key, char *arg, struct argp_state *state) case ARGP_BRICK_PORT_KEY: n = 0; - if (gf_string2uint_base10 (arg, &n) == 0) { + port_str = strtok_r (arg, ",", &tmp_str); + if (gf_string2uint_base10 (port_str, &n) == 0) { cmd_args->brick_port = n; + port_str = strtok_r (NULL, ",", &tmp_str); + if (port_str) { + if (gf_string2uint_base10 (port_str, &n) == 0) + cmd_args->brick_port2 = n; + break; + + argp_failure (state, -1, 0, + "wrong brick (listen) port %s", arg); + } break; } diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h index ea0c6d50b..0ad17ad63 100644 --- a/libglusterfs/src/glusterfs.h +++ b/libglusterfs/src/glusterfs.h @@ -286,6 +286,7 @@ struct _cmd_args { /* required for portmap */ int brick_port; char *brick_name; + int brick_port2; }; typedef struct _cmd_args cmd_args_t; diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c index b55204f31..153329f9e 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.c +++ b/xlators/mgmt/glusterd/src/glusterd-store.c @@ -157,6 +157,12 @@ glusterd_store_create_brick (glusterd_volinfo_t *volinfo, if (ret) gf_log ("", GF_LOG_TRACE, "failed to write brick->port"); + snprintf (buf, sizeof(buf), "%s=%d", GLUSTERD_STORE_KEY_BRICK_RDMA_PORT, + brickinfo->rdma_port); + ret = write (shandle->fd, buf, strlen (buf)); + if (ret) + gf_log ("", GF_LOG_TRACE, "failed to write brick->rdma_port"); + ret = 0; snprintf (buf, sizeof (buf), "%s-%d",GLUSTERD_STORE_KEY_VOL_BRICK, @@ -269,7 +275,6 @@ glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo) closedir (dir); ret = rmdir (brickdir); - out: gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret); return ret; @@ -364,6 +369,7 @@ glusterd_store_create_volume (glusterd_volinfo_t *volinfo) if (ret) goto out; */ + snprintf (buf, sizeof (buf), "%d", volinfo->sub_count); ret = glusterd_store_save_value (volinfo->shandle, GLUSTERD_STORE_KEY_VOL_SUB_COUNT, buf); @@ -1081,6 +1087,14 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo) pmap = pmap_registry_get (THIS); if (pmap->last_alloc <= brickinfo->port) pmap->last_alloc = brickinfo->port + 1; + } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT, + strlen (GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) { + gf_string2int (value, &brickinfo->rdma_port); + /* This is required to have proper ports + assigned to bricks after restart */ + pmap = pmap_registry_get (THIS); + if (pmap->last_alloc <= brickinfo->rdma_port) + pmap->last_alloc = brickinfo->rdma_port + 1; } else { gf_log ("", GF_LOG_ERROR, "Unknown key: %s", key); diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h index 472a6ef4f..b2e7bcc5e 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.h +++ b/xlators/mgmt/glusterd/src/glusterd-store.h @@ -53,6 +53,7 @@ #define GLUSTERD_STORE_KEY_BRICK_HOSTNAME "hostname" #define GLUSTERD_STORE_KEY_BRICK_PATH "path" #define GLUSTERD_STORE_KEY_BRICK_PORT "listen-port" +#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT "rdma.listen-port" #define GLUSTERD_STORE_KEY_PEER_UUID "uuid" #define GLUSTERD_STORE_KEY_PEER_HOSTNAME "hostname" diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index a2da435c7..e83b01e3a 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -916,6 +916,7 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo, char exp_path[PATH_MAX] = {0,}; char logfile[PATH_MAX] = {0,}; int port = 0; + int rdma_port = 0; FILE *file = NULL; gf_boolean_t is_locked = _gf_false; @@ -996,12 +997,26 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo, if (!port) port = pmap_registry_alloc (THIS); - snprintf (cmd_str, 8192, - "%s/sbin/glusterfsd --xlator-option %s-server.listen-port=%d " - "-s localhost --volfile-id %s -p %s --brick-name %s " - "--brick-port %d -l %s", GFS_PREFIX, volinfo->volname, - port, volfile, pidfile, brickinfo->path, port, - brickinfo->logfile); + if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) { + snprintf (cmd_str, 8192, + "%s/sbin/glusterfsd --xlator-option %s-server.listen-port=%d " + "-s localhost --volfile-id %s -p %s --brick-name %s " + "--brick-port %d -l %s", GFS_PREFIX, volinfo->volname, + port, volfile, pidfile, brickinfo->path, port, + brickinfo->logfile); + } else { + rdma_port = brickinfo->rdma_port; + if (!rdma_port) + rdma_port = pmap_registry_alloc (THIS); + + snprintf (cmd_str, 8192, + "%s/sbin/glusterfsd --xlator-option %s-server.listen-port=%d " + "--xlator-option %s-server.transport.rdma.listen-port=%d -s localhost " + "--volfile-id %s -p %s --brick-name %s " + "--brick-port %d,%d -l %s", GFS_PREFIX, volinfo->volname, + port, volinfo->volname, rdma_port, volfile, pidfile, + brickinfo->path, port, rdma_port, brickinfo->logfile); + } gf_log ("",GF_LOG_DEBUG,"Starting GlusterFS Command Executed: \n %s \n", cmd_str); ret = gf_system (cmd_str); @@ -1009,6 +1024,7 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo, if (ret == 0) { //pmap_registry_bind (THIS, port, brickinfo->path); brickinfo->port = port; + brickinfo->rdma_port = rdma_port; } out: if (is_locked && file) diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index 9e139fd95..408138a32 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -118,6 +118,7 @@ struct glusterd_brickinfo { struct list_head brick_list; uuid_t uuid; int port; + int rdma_port; char *logfile; gf_boolean_t signed_in; glusterd_store_handle_t *shandle; diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c index 265c053c0..71c5883ad 100644 --- a/xlators/protocol/client/src/client-handshake.c +++ b/xlators/protocol/client/src/client-handshake.c @@ -857,6 +857,8 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m conf->connecting = 0; conf->connected = 1; + conf->need_different_port = 0; + /* TODO: more to test */ client_post_handshake (frame, frame->this); @@ -1117,6 +1119,8 @@ client_query_portmap (xlator_t *this, struct rpc_clnt *rpc) clnt_conf_t *conf = NULL; dict_t *options = NULL; char *remote_subvol = NULL; + char *xprt = NULL; + char brick_name[PATH_MAX] = {0,}; options = this->options; conf = this->private; @@ -1130,6 +1134,23 @@ client_query_portmap (xlator_t *this, struct rpc_clnt *rpc) req.brick = remote_subvol; + /* FIXME: Dirty work around */ + if (!dict_get_str (options, "transport-type", &xprt)) { + /* This logic is required only in case of 'rdma' client + transport-type and the volume is of 'tcp,rdma' + transport type. */ + if (!strcmp (xprt, "rdma")) { + if (!conf->need_different_port) { + snprintf (brick_name, PATH_MAX, "%s.rdma", + remote_subvol); + req.brick = brick_name; + conf->need_different_port = 1; + } else { + conf->need_different_port = 0; + } + } + } + fr = create_frame (this, this->ctx->pool); if (!fr) { ret = -1; diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h index dfae8fb2c..f4fd3945f 100644 --- a/xlators/protocol/client/src/client.h +++ b/xlators/protocol/client/src/client.h @@ -63,6 +63,9 @@ typedef struct clnt_conf { int last_sent_event; /* Flag used to make sure we are not repeating the same event which was sent earlier */ + char need_different_port; /* flag used to change the + portmap path in case of + 'tcp,rdma' on server */ } clnt_conf_t; typedef struct _client_fd_ctx { -- cgit