summaryrefslogtreecommitdiffstats
path: root/xlators
diff options
context:
space:
mode:
authorAmar Tumballi <amar@gluster.com>2011-06-17 06:10:53 +0000
committerAnand Avati <avati@gluster.com>2011-06-19 02:16:34 -0700
commita6fd403a15ffead93daba8a737cd7c6d6a943744 (patch)
tree618fd1c03ead241bb4605a774e55c14425c2c026 /xlators
parent3d6efb17f12fa23482936ec7a5a666bbf9c0f78b (diff)
fix multiple transport related portmap issues in client handshake
Signed-off-by: Amar Tumballi <amar@gluster.com> Signed-off-by: Anand Avati <avati@gluster.com> BUG: 2258 (enhance gluster volume rebalance) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=2258
Diffstat (limited to 'xlators')
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-store.c16
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-store.h1
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-utils.c28
-rw-r--r--xlators/mgmt/glusterd/src/glusterd.h1
-rw-r--r--xlators/protocol/client/src/client-handshake.c21
-rw-r--r--xlators/protocol/client/src/client.h3
6 files changed, 63 insertions, 7 deletions
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index b55204f31..153329f9e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -157,6 +157,12 @@ glusterd_store_create_brick (glusterd_volinfo_t *volinfo,
if (ret)
gf_log ("", GF_LOG_TRACE, "failed to write brick->port");
+ snprintf (buf, sizeof(buf), "%s=%d", GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+ brickinfo->rdma_port);
+ ret = write (shandle->fd, buf, strlen (buf));
+ if (ret)
+ gf_log ("", GF_LOG_TRACE, "failed to write brick->rdma_port");
+
ret = 0;
snprintf (buf, sizeof (buf), "%s-%d",GLUSTERD_STORE_KEY_VOL_BRICK,
@@ -269,7 +275,6 @@ glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo)
closedir (dir);
ret = rmdir (brickdir);
-
out:
gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
return ret;
@@ -364,6 +369,7 @@ glusterd_store_create_volume (glusterd_volinfo_t *volinfo)
if (ret)
goto out;
*/
+
snprintf (buf, sizeof (buf), "%d", volinfo->sub_count);
ret = glusterd_store_save_value (volinfo->shandle,
GLUSTERD_STORE_KEY_VOL_SUB_COUNT, buf);
@@ -1081,6 +1087,14 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
pmap = pmap_registry_get (THIS);
if (pmap->last_alloc <= brickinfo->port)
pmap->last_alloc = brickinfo->port + 1;
+ } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+ strlen (GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
+ gf_string2int (value, &brickinfo->rdma_port);
+ /* This is required to have proper ports
+ assigned to bricks after restart */
+ pmap = pmap_registry_get (THIS);
+ if (pmap->last_alloc <= brickinfo->rdma_port)
+ pmap->last_alloc = brickinfo->rdma_port + 1;
} else {
gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
key);
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index 472a6ef4f..b2e7bcc5e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -53,6 +53,7 @@
#define GLUSTERD_STORE_KEY_BRICK_HOSTNAME "hostname"
#define GLUSTERD_STORE_KEY_BRICK_PATH "path"
#define GLUSTERD_STORE_KEY_BRICK_PORT "listen-port"
+#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT "rdma.listen-port"
#define GLUSTERD_STORE_KEY_PEER_UUID "uuid"
#define GLUSTERD_STORE_KEY_PEER_HOSTNAME "hostname"
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index a2da435c7..e83b01e3a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -916,6 +916,7 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo,
char exp_path[PATH_MAX] = {0,};
char logfile[PATH_MAX] = {0,};
int port = 0;
+ int rdma_port = 0;
FILE *file = NULL;
gf_boolean_t is_locked = _gf_false;
@@ -996,12 +997,26 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo,
if (!port)
port = pmap_registry_alloc (THIS);
- snprintf (cmd_str, 8192,
- "%s/sbin/glusterfsd --xlator-option %s-server.listen-port=%d "
- "-s localhost --volfile-id %s -p %s --brick-name %s "
- "--brick-port %d -l %s", GFS_PREFIX, volinfo->volname,
- port, volfile, pidfile, brickinfo->path, port,
- brickinfo->logfile);
+ if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) {
+ snprintf (cmd_str, 8192,
+ "%s/sbin/glusterfsd --xlator-option %s-server.listen-port=%d "
+ "-s localhost --volfile-id %s -p %s --brick-name %s "
+ "--brick-port %d -l %s", GFS_PREFIX, volinfo->volname,
+ port, volfile, pidfile, brickinfo->path, port,
+ brickinfo->logfile);
+ } else {
+ rdma_port = brickinfo->rdma_port;
+ if (!rdma_port)
+ rdma_port = pmap_registry_alloc (THIS);
+
+ snprintf (cmd_str, 8192,
+ "%s/sbin/glusterfsd --xlator-option %s-server.listen-port=%d "
+ "--xlator-option %s-server.transport.rdma.listen-port=%d -s localhost "
+ "--volfile-id %s -p %s --brick-name %s "
+ "--brick-port %d,%d -l %s", GFS_PREFIX, volinfo->volname,
+ port, volinfo->volname, rdma_port, volfile, pidfile,
+ brickinfo->path, port, rdma_port, brickinfo->logfile);
+ }
gf_log ("",GF_LOG_DEBUG,"Starting GlusterFS Command Executed: \n %s \n", cmd_str);
ret = gf_system (cmd_str);
@@ -1009,6 +1024,7 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo,
if (ret == 0) {
//pmap_registry_bind (THIS, port, brickinfo->path);
brickinfo->port = port;
+ brickinfo->rdma_port = rdma_port;
}
out:
if (is_locked && file)
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 9e139fd95..408138a32 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -118,6 +118,7 @@ struct glusterd_brickinfo {
struct list_head brick_list;
uuid_t uuid;
int port;
+ int rdma_port;
char *logfile;
gf_boolean_t signed_in;
glusterd_store_handle_t *shandle;
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 265c053c0..71c5883ad 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -857,6 +857,8 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
conf->connecting = 0;
conf->connected = 1;
+ conf->need_different_port = 0;
+
/* TODO: more to test */
client_post_handshake (frame, frame->this);
@@ -1117,6 +1119,8 @@ client_query_portmap (xlator_t *this, struct rpc_clnt *rpc)
clnt_conf_t *conf = NULL;
dict_t *options = NULL;
char *remote_subvol = NULL;
+ char *xprt = NULL;
+ char brick_name[PATH_MAX] = {0,};
options = this->options;
conf = this->private;
@@ -1130,6 +1134,23 @@ client_query_portmap (xlator_t *this, struct rpc_clnt *rpc)
req.brick = remote_subvol;
+ /* FIXME: Dirty work around */
+ if (!dict_get_str (options, "transport-type", &xprt)) {
+ /* This logic is required only in case of 'rdma' client
+ transport-type and the volume is of 'tcp,rdma'
+ transport type. */
+ if (!strcmp (xprt, "rdma")) {
+ if (!conf->need_different_port) {
+ snprintf (brick_name, PATH_MAX, "%s.rdma",
+ remote_subvol);
+ req.brick = brick_name;
+ conf->need_different_port = 1;
+ } else {
+ conf->need_different_port = 0;
+ }
+ }
+ }
+
fr = create_frame (this, this->ctx->pool);
if (!fr) {
ret = -1;
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index dfae8fb2c..f4fd3945f 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -63,6 +63,9 @@ typedef struct clnt_conf {
int last_sent_event; /* Flag used to make sure we are
not repeating the same event
which was sent earlier */
+ char need_different_port; /* flag used to change the
+ portmap path in case of
+ 'tcp,rdma' on server */
} clnt_conf_t;
typedef struct _client_fd_ctx {