fix multiple transport portmap issues in client handshake

Signed-off-by: Amar Tumballi <amar@gluster.com>
Signed-off-by: Anand Avati <avati@gluster.com>

BUG: 2294 (Currently there is no way through cli to make a volume listen on both the transports (socket/rdma))
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=2294

6 files changed, 53 insertions, 1 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index ab1dad827..f589a8dfc 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -319,6 +319,10 @@ glusterd_store_brickinfo_write (int fd, glusterd_brickinfo_t *brickinfo)
         ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PORT,
                                          value);
 
+        snprintf (value, sizeof(value), "%d", brickinfo->rdma_port);
+        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+                                         value);
+
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -1448,6 +1452,14 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                                 pmap = pmap_registry_get (THIS);
                                 if (pmap->last_alloc <= brickinfo->port)
                                         pmap->last_alloc = brickinfo->port + 1;
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+                                    strlen (GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
+                                gf_string2int (value, &brickinfo->rdma_port);
+                                /* This is required to have proper ports
+                                   assigned to bricks after restart */
+                                pmap = pmap_registry_get (THIS);
+                                if (pmap->last_alloc <= brickinfo->rdma_port)
+                                        pmap->last_alloc = brickinfo->rdma_port + 1;
                         } else {
                                 gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
                                         key);
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index 9d6462335..1dbd6dcf1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -58,6 +58,7 @@ typedef enum glusterd_store_ver_ac_{
 #define GLUSTERD_STORE_KEY_BRICK_HOSTNAME "hostname"
 #define GLUSTERD_STORE_KEY_BRICK_PATH     "path"
 #define GLUSTERD_STORE_KEY_BRICK_PORT     "listen-port"
+#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT "rdma.listen-port"
 
 #define GLUSTERD_STORE_KEY_PEER_UUID      "uuid"
 #define GLUSTERD_STORE_KEY_PEER_HOSTNAME  "hostname"
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 8ef38fc8b..8c042317b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1028,6 +1028,7 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         char                    exp_path[PATH_MAX] = {0,};
         char                    logfile[PATH_MAX] = {0,};
         int                     port = 0;
+        int                     rdma_port = 0;
         FILE                    *file = NULL;
         gf_boolean_t            is_locked = _gf_false;
         char                    socketpath[PATH_MAX] = {0};
@@ -1117,7 +1118,19 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
                          "-p", pidfile, "-S", socketpath,
                          "--brick-name", brickinfo->path,
                          "-l", brickinfo->logfile, "--brick-port",  NULL);
-        runner_argprintf (&runner, "%d", port);
+
+        if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) {
+                runner_argprintf (&runner, "%d", port);
+        } else {
+                rdma_port = brickinfo->rdma_port;
+                if (!rdma_port)
+                        rdma_port = pmap_registry_alloc (THIS);
+                runner_argprintf (&runner, "%d,%d", port, rdma_port);
+                runner_add_arg (&runner, "--xlator-option");
+                runner_argprintf (&runner, "%s-server.transport.rdma.listen-port=%d",
+                                  volinfo->volname, rdma_port);
+        }
+
         runner_add_arg (&runner, "--xlator-option");
         runner_argprintf (&runner, "%s-server.listen-port=%d",
                           volinfo->volname, port);
@@ -1128,6 +1141,7 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         if (ret == 0) {
                 //pmap_registry_bind (THIS, port, brickinfo->path);
                 brickinfo->port = port;
+                brickinfo->rdma_port = rdma_port;
         }
 
 connect:
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 9fe4d5d15..3411d3d3e 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -122,6 +122,7 @@ struct glusterd_brickinfo {
         struct list_head  brick_list;
         uuid_t  uuid;
         int     port;
+        int     rdma_port;
         char   *logfile;
         gf_boolean_t signed_in;
         glusterd_store_handle_t *shandle;
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 9489e764f..9c90b6bce 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -918,6 +918,8 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
         conf->connecting = 0;
         conf->connected = 1;
 
+        conf->need_different_port = 0;
+
         /* TODO: more to test */
         client_post_handshake (frame, frame->this);
 
@@ -1184,6 +1186,8 @@ client_query_portmap (xlator_t *this, struct rpc_clnt *rpc)
         clnt_conf_t             *conf            = NULL;
         dict_t                  *options         = NULL;
         char                    *remote_subvol   = NULL;
+        char                    *xprt            = NULL;
+        char                     brick_name[PATH_MAX] = {0,};
 
         options = this->options;
         conf    = this->private;
@@ -1197,6 +1201,23 @@ client_query_portmap (xlator_t *this, struct rpc_clnt *rpc)
 
         req.brick = remote_subvol;
 
+        /* FIXME: Dirty work around */
+        if (!dict_get_str (options, "transport-type", &xprt)) {
+                /* This logic is required only in case of 'rdma' client
+                   transport-type and the volume is of 'tcp,rdma'
+                   transport type. */
+                if (!strcmp (xprt, "rdma")) {
+                        if (!conf->need_different_port) {
+                                snprintf (brick_name, PATH_MAX, "%s.rdma",
+                                          remote_subvol);
+                                req.brick = brick_name;
+                                conf->need_different_port = 1;
+                        } else {
+                                conf->need_different_port = 0;
+                        }
+                }
+        }
+
         fr  = create_frame (this, this->ctx->pool);
         if (!fr) {
                 ret = -1;
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 40a3d5d3d..96080ef61 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -66,6 +66,9 @@ typedef struct clnt_conf {
                                                    which was sent earlier */
         char                   portmap_err_logged; /* flag used to prevent
                                                       excessive logging */
+        char                   need_different_port; /* flag used to change the
+                                                       portmap path in case of
+                                                       'tcp,rdma' on server */
 } clnt_conf_t;
 
 typedef struct _client_fd_ctx {