rpc: fix binding brick issue while bind-insecure is enabled

This patch is backport of http://review.gluster.org/#/c/11512/

> problem:

> When bind-insecure is turned on (which is the default now), it may happen
> that brick is not able to bind to port assigned by Glusterd for example
> 49192-49195...
>
> It seems to occur because the rpc_clnt connections are binding to ports in
> the same range. so brick fails to bind to a port which is already used by
> someone else
>
> solution:
>
> fix for now is to  make rpc_clnt to get port numbers from 65535 in a
> descending
> order, as a result port clash is minimized
>
> other fixes:
>
> previously rdma binds to port >= 1024 if it cannot find a free port < 1024,
> even when bind insecure was turned off(ref to commit '0e3fd04e'), this patch
> add's a check for bind-insecure in gf_rdma_client_bind function
>
> This patch also re-enable bind-insecure and allow insecure by default
> which was reverted (ref: commit cef1720) previously

> Change-Id: Ia1cfa93c5454e2ae0ff57813689b75de282ebd07
> BUG: 1238661
> Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>

Change-Id: Iea55f9b2a57b5e24d3df2c5fafae12fe99e9dee0
BUG: 1246481
Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
Reviewed-on: http://review.gluster.org/11758
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>

2 files changed, 73 insertions, 50 deletions

diff --git a/rpc/rpc-transport/rdma/src/name.c b/rpc/rpc-transport/rdma/src/name.c
index 8b403ed4c4c..f723efc2360 100644
--- a/rpc/rpc-transport/rdma/src/name.c
+++ b/rpc/rpc-transport/rdma/src/name.c
@@ -33,36 +33,41 @@ gf_resolve_ip6 (const char *hostname,
                 void **dnscache,
                 struct addrinfo **addr_info);
 
+
+static void
+_assign_port (struct sockaddr *sockaddr, uint16_t port)
+{
+        switch (sockaddr->sa_family) {
+        case AF_INET6:
+                ((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port);
+                break;
+
+        case AF_INET_SDP:
+        case AF_INET:
+                ((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
+                break;
+        }
+}
+
 static int32_t
 af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id,
                                  struct sockaddr *sockaddr,
-                                 socklen_t sockaddr_len, int ceiling)
+                                 socklen_t sockaddr_len, uint32_t ceiling)
 {
         int32_t        ret        = -1;
         uint16_t      port        = ceiling - 1;
         /* by default assume none of the ports are blocked and all are available */
-        gf_boolean_t  ports[1024] = {_gf_false,};
+        gf_boolean_t  ports[GF_PORT_MAX] = {_gf_false,};
         int           i           = 0;
 
-        ret = gf_process_reserved_ports (ports);
+        ret = gf_process_reserved_ports (ports, ceiling);
         if (ret != 0) {
-                for (i = 0; i < 1024; i++)
+                for (i = 0; i < GF_PORT_MAX; i++)
                         ports[i] = _gf_false;
         }
 
         while (port) {
-                switch (sockaddr->sa_family) {
-                case AF_INET6:
-                        ((struct sockaddr_in6 *)sockaddr)->sin6_port
-                                = htons (port);
-                        break;
-
-                case AF_INET_SDP:
-                case AF_INET:
-                        ((struct sockaddr_in *)sockaddr)->sin_port
-                                = htons (port);
-                        break;
-                }
+                _assign_port (sockaddr, port);
                 /* ignore the reserved ports */
                 if (ports[port] == _gf_true) {
                         port--;
@@ -425,22 +430,26 @@ gf_rdma_client_bind (rpc_transport_t *this, struct sockaddr *sockaddr,
                 *sockaddr_len = sizeof (struct sockaddr_in);
 
         case AF_INET6:
-                ret = af_inet_bind_to_port_lt_ceiling (cm_id, sockaddr,
+                if (!this->bind_insecure) {
+                        ret = af_inet_bind_to_port_lt_ceiling (cm_id, sockaddr,
                                                        *sockaddr_len,
                                                        GF_CLIENT_PORT_CEILING);
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                RDMA_MSG_PORT_BIND_FAILED,
-                                "cannot bind rdma_cm_id to port "
-                                "less than %d", GF_CLIENT_PORT_CEILING);
-                        if (sockaddr->sa_family == AF_INET6) {
-                                ((struct sockaddr_in6 *)sockaddr)->sin6_port
-                                        = htons (0);
-                        } else {
-                                ((struct sockaddr_in *)sockaddr)->sin_port
-                                        = htons (0);
+                        if (ret == -1) {
+                                gf_msg (this->name, GF_LOG_WARNING, errno,
+                                        RDMA_MSG_PORT_BIND_FAILED,
+                                        "cannot bind rdma_cm_id to port "
+                                        "less than %d", GF_CLIENT_PORT_CEILING);
+                        }
+                } else {
+                        ret = af_inet_bind_to_port_lt_ceiling (cm_id, sockaddr,
+                                                       *sockaddr_len,
+                                                       GF_PORT_MAX);
+                        if (ret == -1) {
+                                gf_msg (this->name, GF_LOG_WARNING, errno,
+                                        RDMA_MSG_PORT_BIND_FAILED,
+                                        "cannot bind rdma_cm_id to port "
+                                        "less than %d", GF_PORT_MAX);
                         }
-                        ret = rdma_bind_addr (cm_id, sockaddr);
                 }
                 break;
 
diff --git a/rpc/rpc-transport/socket/src/name.c b/rpc/rpc-transport/socket/src/name.c
index f731bab4b0a..650c5a747be 100644
--- a/rpc/rpc-transport/socket/src/name.c
+++ b/rpc/rpc-transport/socket/src/name.c
@@ -23,35 +23,40 @@
 #include "socket.h"
 #include "common-utils.h"
 
+static void
+_assign_port (struct sockaddr *sockaddr, uint16_t port)
+{
+        switch (sockaddr->sa_family) {
+        case AF_INET6:
+                ((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port);
+                break;
+
+        case AF_INET_SDP:
+        case AF_INET:
+                ((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
+                break;
+        }
+}
+
 static int32_t
 af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
-                                 socklen_t sockaddr_len, int ceiling)
+                                 socklen_t sockaddr_len, uint32_t ceiling)
 {
         int32_t        ret        = -1;
         uint16_t      port        = ceiling - 1;
         // by default assume none of the ports are blocked and all are available
-        gf_boolean_t  ports[1024] = {_gf_false,};
+        gf_boolean_t  ports[GF_PORT_MAX] = {_gf_false,};
         int           i           = 0;
 
-        ret = gf_process_reserved_ports (ports);
+        ret = gf_process_reserved_ports (ports, ceiling);
         if (ret != 0) {
-                for (i = 0; i < 1024; i++)
+                for (i = 0; i < GF_PORT_MAX; i++)
                         ports[i] = _gf_false;
         }
 
         while (port)
         {
-                switch (sockaddr->sa_family)
-                {
-                case AF_INET6:
-                        ((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port);
-                        break;
-
-                case AF_INET_SDP:
-                case AF_INET:
-                        ((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
-                        break;
-                }
+                _assign_port (sockaddr, port);
                 // ignore the reserved ports
                 if (ports[port] == _gf_true) {
                         port--;
@@ -440,12 +445,21 @@ client_bind (rpc_transport_t *this,
                 if (!this->bind_insecure) {
                         ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr,
                                                        *sockaddr_len, GF_CLIENT_PORT_CEILING);
-                }
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "cannot bind inet socket (%d) to port less than %d (%s)",
-                                sock, GF_CLIENT_PORT_CEILING, strerror (errno));
-                        ret = 0;
+                        if (ret == -1) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "cannot bind inet socket (%d) to port less than %d (%s)",
+                                        sock, GF_CLIENT_PORT_CEILING, strerror (errno));
+                                ret = 0;
+                        }
+                } else {
+                        ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr,
+                                                       *sockaddr_len, GF_PORT_MAX);
+                        if (ret == -1) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "failed while binding to less than %d (%s)",
+                                        GF_PORT_MAX, strerror (errno));
+                                ret = 0;
+                        }
                 }
                 break;