rdma: client connection establishment takes more time

For rdma type only volume client connection establishment
with server takes more than three seconds. Because for
tcp,rdma type volume, will have 2 ports one for tcp and
one for rdma, tcp port is stored with brickname and rdma
port is stored as "brickname.rdma" during pamap_sighin.
During the handshake when trying to get the brick port
for rdma clients, since we are not aware of server
transport type, we will append '.rdma' with brick name.
So for tcp,rdma volume there will be an entry with
'.rdma', but it will fail for rdma type only volume.
So we will try again, this time without appending '.rdma'
using a flag variable need_different_port, and it will succeed,
but the reconnection happens only after 3 seconds.
In this patch for rdma only type volume 
we will append '.rdma' during the pmap_signin. So during the
handshake we will get the correct port for first try itself.
Since we don't need to retry , we can remove the
need_different_port flag variable.

Change-Id: Ie8e3a7f532d4104829dbe995e99b35e95571466c
BUG: 1153569
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
Reviewed-on: http://review.gluster.org/8934
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Tested-by: Raghavendra G <rgowdapp@redhat.com>

3 files changed, 39 insertions, 16 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 8bfa2d4bdd6..5a98d497137 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -1390,14 +1390,16 @@ umount:
  * after all commit acks are received.
  */
 static int
-rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict,
-                         dict_t *req_dict, int32_t replace_op)
+rb_update_srcbrick_port (glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *src_brickinfo,
+                         dict_t *rsp_dict, dict_t *req_dict, int32_t replace_op)
 {
-        xlator_t *this            = NULL;
-        dict_t   *ctx             = NULL;
-        int       ret             = 0;
-        int       dict_ret        = 0;
-        int       src_port        = 0;
+        xlator_t *this                  = NULL;
+        dict_t   *ctx                   = NULL;
+        int       ret                   = 0;
+        int       dict_ret              = 0;
+        int       src_port              = 0;
+        char      brickname[PATH_MAX]   = {0,};
 
         this = THIS;
 
@@ -1409,8 +1411,15 @@ rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict,
                 gf_log ("", GF_LOG_INFO,
                         "adding src-brick port no");
 
+                if (volinfo->transport_type == GF_TRANSPORT_RDMA) {
+                        snprintf (brickname, sizeof(brickname), "%s.rdma",
+                                  src_brickinfo->path);
+                } else
+                        snprintf (brickname, sizeof(brickname), "%s",
+                                  src_brickinfo->path);
+
                 src_brickinfo->port = pmap_registry_search (this,
-                                      src_brickinfo->path, GF_PMAP_PORT_BRICKSERVER);
+                                      brickname, GF_PMAP_PORT_BRICKSERVER);
                 if (!src_brickinfo->port &&
                     replace_op != GF_REPLACE_OP_COMMIT_FORCE ) {
                         gf_log ("", GF_LOG_ERROR,
@@ -1645,7 +1654,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                 goto out;
         }
 
-        ret = rb_update_srcbrick_port (src_brickinfo, rsp_dict,
+        ret = rb_update_srcbrick_port (volinfo, src_brickinfo, rsp_dict,
                                        dict, replace_op);
         if (ret)
                 goto out;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index bb9fa45d24a..ad824be73f3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1968,6 +1968,12 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
                                   bind_address);
         }
 
+        if (volinfo->transport_type == GF_TRANSPORT_RDMA)
+                runner_argprintf (&runner, "--volfile-server-transport=rdma");
+        else if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA)
+                runner_argprintf (&runner,
+                                  "--volfile-server-transport=socket,rdma");
+
         if (volinfo->memory_accounting)
                 runner_add_arg (&runner, "--mem-accounting");
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 4685fa6a8e4..dd3308c40d2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2583,12 +2583,13 @@ int
 glusterd_clearlocks_get_local_client_ports (glusterd_volinfo_t *volinfo,
                                             char **xl_opts)
 {
-        glusterd_brickinfo_t    *brickinfo      = NULL;
-        glusterd_conf_t         *priv           = NULL;
-        int                     index           = 0;
-        int                     ret             = -1;
-        int                     i               = 0;
-        int                     port            = 0;
+        glusterd_brickinfo_t    *brickinfo          = NULL;
+        glusterd_conf_t         *priv               = NULL;
+        char                    brickname[PATH_MAX] = {0,};
+        int                     index               = 0;
+        int                     ret                 = -1;
+        int                     i                   = 0;
+        int                     port                = 0;
 
         GF_ASSERT (xl_opts);
         if (!xl_opts) {
@@ -2605,7 +2606,14 @@ glusterd_clearlocks_get_local_client_ports (glusterd_volinfo_t *volinfo,
                 if (uuid_compare (brickinfo->uuid, MY_UUID))
                         continue;
 
-                port = pmap_registry_search (THIS, brickinfo->path,
+                if (volinfo->transport_type == GF_TRANSPORT_RDMA) {
+                        snprintf (brickname, sizeof(brickname), "%s.rdma",
+                                  brickinfo->path);
+                } else
+                        snprintf (brickname, sizeof(brickname), "%s",
+                                  brickinfo->path);
+
+                port = pmap_registry_search (THIS, brickname,
                                              GF_PMAP_PORT_BRICKSERVER);
                 if (!port) {
                         ret = -1;