fix multiple transport related portmap issues in client handshake

Signed-off-by: Amar Tumballi <amar@gluster.com>
Signed-off-by: Anand Avati <avati@gluster.com>

BUG: 2258 (enhance gluster volume rebalance)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=2258

9 files changed, 137 insertions, 8 deletions

diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index a6d8a480076..02b622f9884 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -546,12 +546,51 @@ out:
 
 
 static int
+mgmt_pmap_signin2_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+        pmap_signin_rsp  rsp   = {0,};
+        call_frame_t    *frame = NULL;
+        int              ret   = 0;
+
+        frame = myframe;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_pmap_signin_rsp (*iov, &rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR, "XDR decode error");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        if (-1 == rsp.op_ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "failed to register the port with glusterd");
+                goto out;
+        }
+out:
+        STACK_DESTROY (frame->root);
+        return 0;
+
+}
+
+static int
 mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         pmap_signin_rsp  rsp   = {0,};
         call_frame_t    *frame = NULL;
         int              ret   = 0;
+        pmap_signin_req  pmap_req = {0, };
+        cmd_args_t      *cmd_args = NULL;
+        glusterfs_ctx_t *ctx      = NULL;
+        char             brick_name[PATH_MAX] = {0,};
 
         frame = myframe;
 
@@ -574,6 +613,27 @@ mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         "failed to register the port with glusterd");
                 goto out;
         }
+
+        ctx = glusterfs_ctx_get ();
+        cmd_args = &ctx->cmd_args;
+
+        if (!cmd_args->brick_port2) {
+                /* We are done with signin process */
+                goto out;
+        }
+
+        snprintf (brick_name, PATH_MAX, "%s.rdma", cmd_args->brick_name);
+        pmap_req.port  = cmd_args->brick_port2;
+        pmap_req.brick = brick_name;
+
+        ret = mgmt_submit_request (&pmap_req, frame, ctx, &clnt_pmap_prog,
+                                   GF_PMAP_SIGNIN, xdr_from_pmap_signin_req,
+                                   mgmt_pmap_signin2_cbk);
+        if (ret)
+                goto out;
+
+        return 0;
+
 out:
 
         STACK_DESTROY (frame->root);
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 87d16c7dcf9..f17ac814eb9 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -426,6 +426,8 @@ parse_opts (int key, char *arg, struct argp_state *state)
         gf_boolean_t  b             = _gf_false;
         char         *pwd           = NULL;
         char          tmp_buf[2048] = {0,};
+        char         *tmp_str       = NULL;
+        char         *port_str      = NULL;
 
         cmd_args = state->input;
 
@@ -657,8 +659,18 @@ parse_opts (int key, char *arg, struct argp_state *state)
         case ARGP_BRICK_PORT_KEY:
                 n = 0;
 
-                if (gf_string2uint_base10 (arg, &n) == 0) {
+                port_str = strtok_r (arg, ",", &tmp_str);
+                if (gf_string2uint_base10 (port_str, &n) == 0) {
                         cmd_args->brick_port = n;
+                        port_str = strtok_r (NULL, ",", &tmp_str);
+                        if (port_str) {
+                                if (gf_string2uint_base10 (port_str, &n) == 0)
+                                        cmd_args->brick_port2 = n;
+                                break;
+
+                                argp_failure (state, -1, 0,
+                                              "wrong brick (listen) port %s", arg);
+                        }
                         break;
                 }
 
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index ea0c6d50b8f..0ad17ad63ea 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -286,6 +286,7 @@ struct _cmd_args {
         /* required for portmap */
         int             brick_port;
         char           *brick_name;
+        int             brick_port2;
 };
 typedef struct _cmd_args cmd_args_t;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index b55204f31a9..153329f9e1e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -157,6 +157,12 @@ glusterd_store_create_brick (glusterd_volinfo_t *volinfo,
         if (ret)
                 gf_log ("", GF_LOG_TRACE, "failed to write brick->port");
 
+        snprintf (buf, sizeof(buf), "%s=%d", GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+                  brickinfo->rdma_port);
+        ret = write (shandle->fd, buf, strlen (buf));
+        if (ret)
+                gf_log ("", GF_LOG_TRACE, "failed to write brick->rdma_port");
+
         ret = 0;
 
         snprintf (buf, sizeof (buf), "%s-%d",GLUSTERD_STORE_KEY_VOL_BRICK,
@@ -269,7 +275,6 @@ glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo)
         closedir (dir);
 
         ret = rmdir (brickdir);
-
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
@@ -364,6 +369,7 @@ glusterd_store_create_volume (glusterd_volinfo_t *volinfo)
         if (ret)
                 goto out;
 */
+
         snprintf (buf, sizeof (buf), "%d", volinfo->sub_count);
         ret = glusterd_store_save_value (volinfo->shandle,
                                         GLUSTERD_STORE_KEY_VOL_SUB_COUNT, buf);
@@ -1081,6 +1087,14 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                                 pmap = pmap_registry_get (THIS);
                                 if (pmap->last_alloc <= brickinfo->port)
                                         pmap->last_alloc = brickinfo->port + 1;
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+                                    strlen (GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
+                                gf_string2int (value, &brickinfo->rdma_port);
+                                /* This is required to have proper ports
+                                   assigned to bricks after restart */
+                                pmap = pmap_registry_get (THIS);
+                                if (pmap->last_alloc <= brickinfo->rdma_port)
+                                        pmap->last_alloc = brickinfo->rdma_port + 1;
                         } else {
                                 gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
                                         key);
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index 472a6ef4ffe..b2e7bcc5ea0 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -53,6 +53,7 @@
 #define GLUSTERD_STORE_KEY_BRICK_HOSTNAME "hostname"
 #define GLUSTERD_STORE_KEY_BRICK_PATH     "path"
 #define GLUSTERD_STORE_KEY_BRICK_PORT     "listen-port"
+#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT "rdma.listen-port"
 
 #define GLUSTERD_STORE_KEY_PEER_UUID      "uuid"
 #define GLUSTERD_STORE_KEY_PEER_HOSTNAME  "hostname"
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index a2da435c785..e83b01e3aca 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -916,6 +916,7 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         char                    exp_path[PATH_MAX] = {0,};
         char                    logfile[PATH_MAX] = {0,};
         int                     port = 0;
+        int                     rdma_port = 0;
         FILE                    *file = NULL;
         gf_boolean_t            is_locked = _gf_false;
 
@@ -996,12 +997,26 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         if (!port)
                 port = pmap_registry_alloc (THIS);
 
-        snprintf (cmd_str, 8192,
-                  "%s/sbin/glusterfsd --xlator-option %s-server.listen-port=%d "
-                  "-s localhost --volfile-id %s -p %s --brick-name %s "
-                  "--brick-port %d -l %s", GFS_PREFIX, volinfo->volname,
-                  port, volfile, pidfile, brickinfo->path, port,
-                  brickinfo->logfile);
+        if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) {
+                snprintf (cmd_str, 8192,
+                          "%s/sbin/glusterfsd --xlator-option %s-server.listen-port=%d "
+                          "-s localhost --volfile-id %s -p %s --brick-name %s "
+                          "--brick-port %d -l %s", GFS_PREFIX, volinfo->volname,
+                          port, volfile, pidfile, brickinfo->path, port,
+                          brickinfo->logfile);
+        } else {
+                rdma_port = brickinfo->rdma_port;
+                if (!rdma_port)
+                        rdma_port = pmap_registry_alloc (THIS);
+
+                snprintf (cmd_str, 8192,
+                          "%s/sbin/glusterfsd --xlator-option %s-server.listen-port=%d "
+                          "--xlator-option %s-server.transport.rdma.listen-port=%d -s localhost "
+                          "--volfile-id %s -p %s --brick-name %s "
+                          "--brick-port %d,%d -l %s", GFS_PREFIX, volinfo->volname,
+                          port, volinfo->volname, rdma_port, volfile, pidfile,
+                          brickinfo->path, port, rdma_port, brickinfo->logfile);
+        }
 
 	gf_log ("",GF_LOG_DEBUG,"Starting GlusterFS Command Executed: \n %s \n", cmd_str);
         ret = gf_system (cmd_str);
@@ -1009,6 +1024,7 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         if (ret == 0) {
                 //pmap_registry_bind (THIS, port, brickinfo->path);
                 brickinfo->port = port;
+                brickinfo->rdma_port = rdma_port;
         }
 out:
         if (is_locked && file)
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 9e139fd95ce..408138a3297 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -118,6 +118,7 @@ struct glusterd_brickinfo {
         struct list_head  brick_list;
         uuid_t  uuid;
         int     port;
+        int     rdma_port;
         char   *logfile;
         gf_boolean_t signed_in;
         glusterd_store_handle_t *shandle;
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 265c053c00e..71c5883ad43 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -857,6 +857,8 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
         conf->connecting = 0;
         conf->connected = 1;
 
+        conf->need_different_port = 0;
+
         /* TODO: more to test */
         client_post_handshake (frame, frame->this);
 
@@ -1117,6 +1119,8 @@ client_query_portmap (xlator_t *this, struct rpc_clnt *rpc)
         clnt_conf_t             *conf            = NULL;
         dict_t                  *options         = NULL;
         char                    *remote_subvol   = NULL;
+        char                    *xprt            = NULL;
+        char                     brick_name[PATH_MAX] = {0,};
 
         options = this->options;
         conf    = this->private;
@@ -1130,6 +1134,23 @@ client_query_portmap (xlator_t *this, struct rpc_clnt *rpc)
 
         req.brick = remote_subvol;
 
+        /* FIXME: Dirty work around */
+        if (!dict_get_str (options, "transport-type", &xprt)) {
+                /* This logic is required only in case of 'rdma' client
+                   transport-type and the volume is of 'tcp,rdma'
+                   transport type. */
+                if (!strcmp (xprt, "rdma")) {
+                        if (!conf->need_different_port) {
+                                snprintf (brick_name, PATH_MAX, "%s.rdma",
+                                          remote_subvol);
+                                req.brick = brick_name;
+                                conf->need_different_port = 1;
+                        } else {
+                                conf->need_different_port = 0;
+                        }
+                }
+        }
+
         fr  = create_frame (this, this->ctx->pool);
         if (!fr) {
                 ret = -1;
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index dfae8fb2c22..f4fd3945fe7 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -63,6 +63,9 @@ typedef struct clnt_conf {
         int                    last_sent_event; /* Flag used to make sure we are
                                                    not repeating the same event
                                                    which was sent earlier */
+        char                   need_different_port; /* flag used to change the
+                                                       portmap path in case of
+                                                       'tcp,rdma' on server */
 } clnt_conf_t;
 
 typedef struct _client_fd_ctx {