rdma: client connection establishment takes more time

        Backport of http://review.gluster.org/8934

For rdma type only volume client connection establishment
with server takes more than three seconds. Because for
tcp,rdma type volume, will have 2 ports one for tcp and
one for rdma, tcp port is stored with brickname and rdma
port is stored as "brickname.rdma" during pamap_sighin.
During the handshake when trying to get the brick port
for rdma clients, since we are not aware of server
transport type, we will append '.rdma' with brick name.
So for tcp,rdma volume there will be an entry with
'.rdma', but it will fail for rdma type only volume.
So we will try again, this time without appending '.rdma'
using a flag variable need_different_port, and it will succeed,
but the reconnection happens only after 3 seconds.
In this patch for rdma only type volume
we will append '.rdma' during the pmap_signin. So during the
handshake we will get the correct port for first try itself.
Since we don't need to retry , we can remove the
need_different_port flag variable.

Change-Id: I82a8a27f0e65a2e287f321e5e8292d86c6baf5b4
BUG: 1166515
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
Reviewed-on: http://review.gluster.org/8934
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Tested-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-on: http://review.gluster.org/9177
Reviewed-by: Raghavendra Bhat <raghavendra@redhat.com>

7 files changed, 65 insertions, 43 deletions

diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 9addd77cb26..3ff3337c01d 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -2160,10 +2160,11 @@ out:
 int
 glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx)
 {
-        call_frame_t     *frame = NULL;
-        pmap_signin_req   req = {0, };
-        int               ret = -1;
-        cmd_args_t       *cmd_args = NULL;
+        call_frame_t     *frame                 = NULL;
+        pmap_signin_req   req                   = {0, };
+        int               ret                   = -1;
+        cmd_args_t       *cmd_args              = NULL;
+        char              brick_name[PATH_MAX]  = {0,};
 
         frame = create_frame (THIS, ctx->pool);
         cmd_args = &ctx->cmd_args;
@@ -2174,8 +2175,15 @@ glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx)
                 goto out;
         }
 
+        if (cmd_args->volfile_server_transport &&
+                      !strcmp(cmd_args->volfile_server_transport, "rdma")) {
+                snprintf (brick_name, sizeof(brick_name), "%s.rdma",
+                          cmd_args->brick_name);
+                req.brick = brick_name;
+        } else
+                req.brick = cmd_args->brick_name;
+
         req.port  = cmd_args->brick_port;
-        req.brick = cmd_args->brick_name;
 
         ret = mgmt_submit_request (&req, frame, ctx, &clnt_pmap_prog,
                                    GF_PMAP_SIGNIN, mgmt_pmap_signin_cbk,
@@ -2226,6 +2234,7 @@ glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx)
         pmap_signout_req  req = {0, };
         call_frame_t     *frame = NULL;
         cmd_args_t       *cmd_args = NULL;
+        char              brick_name[PATH_MAX]  = {0,};
 
         frame = create_frame (THIS, ctx->pool);
         cmd_args = &ctx->cmd_args;
@@ -2236,8 +2245,15 @@ glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx)
                 goto out;
         }
 
+        if (cmd_args->volfile_server_transport &&
+                      !strcmp(cmd_args->volfile_server_transport, "rdma")) {
+                snprintf (brick_name, sizeof(brick_name), "%s.rdma",
+                          cmd_args->brick_name);
+                req.brick = brick_name;
+        } else
+                req.brick = cmd_args->brick_name;
+
         req.port  = cmd_args->brick_port;
-        req.brick = cmd_args->brick_name;
         req.rdma_port = cmd_args->brick_port2;
         ret = mgmt_submit_request (&req, frame, ctx, &clnt_pmap_prog,
                                    GF_PMAP_SIGNOUT, mgmt_pmap_signout_cbk,
diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c
index 89ed4558d72..1e0cce54b7a 100644
--- a/rpc/rpc-transport/rdma/src/rdma.c
+++ b/rpc/rpc-transport/rdma/src/rdma.c
@@ -4274,7 +4274,7 @@ gf_rdma_disconnect (rpc_transport_t *this)
         int32_t            ret  = 0;
 
         priv = this->private;
-        gf_log_callingfn (this->name, GF_LOG_WARNING,
+        gf_log_callingfn (this->name, GF_LOG_DEBUG,
                           "disconnect called (peer:%s)",
                           this->peerinfo.identifier);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 8bfa2d4bdd6..5a98d497137 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -1390,14 +1390,16 @@ umount:
  * after all commit acks are received.
  */
 static int
-rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict,
-                         dict_t *req_dict, int32_t replace_op)
+rb_update_srcbrick_port (glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *src_brickinfo,
+                         dict_t *rsp_dict, dict_t *req_dict, int32_t replace_op)
 {
-        xlator_t *this            = NULL;
-        dict_t   *ctx             = NULL;
-        int       ret             = 0;
-        int       dict_ret        = 0;
-        int       src_port        = 0;
+        xlator_t *this                  = NULL;
+        dict_t   *ctx                   = NULL;
+        int       ret                   = 0;
+        int       dict_ret              = 0;
+        int       src_port              = 0;
+        char      brickname[PATH_MAX]   = {0,};
 
         this = THIS;
 
@@ -1409,8 +1411,15 @@ rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict,
                 gf_log ("", GF_LOG_INFO,
                         "adding src-brick port no");
 
+                if (volinfo->transport_type == GF_TRANSPORT_RDMA) {
+                        snprintf (brickname, sizeof(brickname), "%s.rdma",
+                                  src_brickinfo->path);
+                } else
+                        snprintf (brickname, sizeof(brickname), "%s",
+                                  src_brickinfo->path);
+
                 src_brickinfo->port = pmap_registry_search (this,
-                                      src_brickinfo->path, GF_PMAP_PORT_BRICKSERVER);
+                                      brickname, GF_PMAP_PORT_BRICKSERVER);
                 if (!src_brickinfo->port &&
                     replace_op != GF_REPLACE_OP_COMMIT_FORCE ) {
                         gf_log ("", GF_LOG_ERROR,
@@ -1645,7 +1654,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                 goto out;
         }
 
-        ret = rb_update_srcbrick_port (src_brickinfo, rsp_dict,
+        ret = rb_update_srcbrick_port (volinfo, src_brickinfo, rsp_dict,
                                        dict, replace_op);
         if (ret)
                 goto out;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index d2eab044dc6..fcaa215b578 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1952,6 +1952,12 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
                                   bind_address);
         }
 
+        if (volinfo->transport_type == GF_TRANSPORT_RDMA)
+                runner_argprintf (&runner, "--volfile-server-transport=rdma");
+        else if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA)
+                runner_argprintf (&runner,
+                                  "--volfile-server-transport=socket,rdma");
+
         if (volinfo->memory_accounting)
                 runner_add_arg (&runner, "--mem-accounting");
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index b55275308e2..cb5d6f832f6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2342,12 +2342,13 @@ int
 glusterd_clearlocks_get_local_client_ports (glusterd_volinfo_t *volinfo,
                                             char **xl_opts)
 {
-        glusterd_brickinfo_t    *brickinfo      = NULL;
-        glusterd_conf_t         *priv           = NULL;
-        int                     index           = 0;
-        int                     ret             = -1;
-        int                     i               = 0;
-        int                     port            = 0;
+        glusterd_brickinfo_t    *brickinfo          = NULL;
+        glusterd_conf_t         *priv               = NULL;
+        char                    brickname[PATH_MAX] = {0,};
+        int                     index               = 0;
+        int                     ret                 = -1;
+        int                     i                   = 0;
+        int                     port                = 0;
 
         GF_ASSERT (xl_opts);
         if (!xl_opts) {
@@ -2364,7 +2365,14 @@ glusterd_clearlocks_get_local_client_ports (glusterd_volinfo_t *volinfo,
                 if (uuid_compare (brickinfo->uuid, MY_UUID))
                         continue;
 
-                port = pmap_registry_search (THIS, brickinfo->path,
+                if (volinfo->transport_type == GF_TRANSPORT_RDMA) {
+                        snprintf (brickname, sizeof(brickname), "%s.rdma",
+                                  brickinfo->path);
+                } else
+                        snprintf (brickname, sizeof(brickname), "%s",
+                                  brickinfo->path);
+
+                port = pmap_registry_search (THIS, brickname,
                                              GF_PMAP_PORT_BRICKSERVER);
                 if (!port) {
                         ret = -1;
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 8bd3d6a3d09..42b7ac0745e 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -1205,8 +1205,6 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
         conf->connecting = 0;
         conf->connected = 1;
 
-        conf->need_different_port = 0;
-
         if (lk_ver != client_get_lk_ver (conf)) {
                 gf_log (this->name, GF_LOG_INFO, "Server and Client "
                         "lk-version numbers are not same, reopening the fds");
@@ -1546,22 +1544,11 @@ client_query_portmap (xlator_t *this, struct rpc_clnt *rpc)
 
         req.brick = remote_subvol;
 
-        /* FIXME: Dirty work around */
         if (!dict_get_str (options, "transport-type", &xprt)) {
-                /* This logic is required only in case of 'rdma' client
-                   transport-type and the volume is of 'tcp,rdma'
-                   transport type. */
                 if (!strcmp (xprt, "rdma")) {
-                        if (!conf->need_different_port) {
-                                snprintf (brick_name, PATH_MAX, "%s.rdma",
-                                          remote_subvol);
-                                req.brick = brick_name;
-                                conf->need_different_port = 1;
-                                conf->skip_notify = 1;
-                        } else {
-                                conf->need_different_port = 0;
-                                conf->skip_notify = 0;
-                        }
+                        snprintf (brick_name, sizeof(brick_name), "%s.rdma",
+                                  remote_subvol);
+                        req.brick = brick_name;
                 }
         }
 
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 7f7d511910e..69f77570cd0 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -96,10 +96,6 @@ typedef struct clnt_conf {
         char                   disconnect_err_logged; /* flag used to prevent
                                                          excessive disconnect
                                                          logging */
-
-        char                   need_different_port; /* flag used to change the
-                                                       portmap path in case of
-                                                       'tcp,rdma' on server */
         gf_boolean_t           lk_heal;
         uint16_t               lk_version; /* this variable is used to distinguish
                                               client-server transaction while