socket,rdma: before binding to any port check if it is a reserved port

* The list of ports which are reserved is obtained by reading the file
  /proc/sys/net/ipv4/ip_local_reserved_ports.

Change-Id: I3a832f53efbb79dd719b34a30cbaf28269f25816
BUG: 762989
Signed-off-by: Raghavendra Bhat <raghavendra@redhat.com>
Reviewed-on: http://review.gluster.org/4131
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

5 files changed, 214 insertions, 24 deletions

diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 8c18b91a7b4..7a654314ee5 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -2166,3 +2166,143 @@ generate_glusterfs_ctx_id (void)
         return gf_strdup (tmp_str);
 }
 
+char *
+gf_get_reserved_ports ()
+{
+        int     proc_fd      = -1;
+        char    *proc_file   = "/proc/sys/net/ipv4/ip_local_reserved_ports";
+        char    buffer[4096] = {0,};
+        char    *ports_info  = NULL;
+        int32_t ret          = -1;
+
+        proc_fd = open (proc_file, O_RDONLY);
+        if (proc_fd == -1) {
+                /* What should be done in this case? error out from here
+                 * and thus stop the glusterfs process from starting or
+                 * continue with older method of using any of the available
+                 * port? For now 2nd option is considered.
+                 */
+                gf_log ("glusterfs", GF_LOG_WARNING, "could not open "
+                        "the file /proc/sys/net/ipv4/ip_local_reserved_ports "
+                        "for getting reserved ports info (%s)",
+                        strerror (errno));
+                goto out;
+        }
+
+        ret = read (proc_fd, buffer, sizeof (buffer));
+        if (ret < 0) {
+                gf_log ("glusterfs", GF_LOG_WARNING, "could not "
+                        "read the file %s for getting reserved ports "
+                        "info (%s)", proc_file, strerror (errno));
+                goto out;
+        }
+        ports_info = gf_strdup (buffer);
+
+out:
+        if (proc_fd != -1)
+                close (proc_fd);
+        return ports_info;
+}
+
+int
+gf_process_reserved_ports (gf_boolean_t *ports)
+{
+        int      ret         = -1;
+        char    *ports_info  = NULL;
+        char    *tmp         = NULL;
+        char    *blocked_port = NULL;
+        gf_boolean_t  result = _gf_false;
+
+        ports_info = gf_get_reserved_ports ();
+        if (!ports_info) {
+                gf_log ("glusterfs", GF_LOG_WARNING, "Not able to get reserved "
+                        "ports, hence there is a possibility that glusterfs "
+                        "may consume reserved port");
+                goto out;
+        }
+
+        blocked_port = strtok_r (ports_info, ",",&tmp);
+        if (!blocked_port || !strcmp (blocked_port, ports_info)) {
+                if (!blocked_port)
+                        blocked_port = ports_info;
+                result = gf_ports_reserved (blocked_port, ports);
+                blocked_port = strtok_r (NULL, ",", &tmp);
+        }
+
+        while (blocked_port) {
+                result = gf_ports_reserved (blocked_port, ports);
+                blocked_port = strtok_r (NULL, ",", &tmp);
+        }
+
+        ret = 0;
+
+out:
+        GF_FREE (ports_info);
+        return ret;
+}
+
+gf_boolean_t
+gf_ports_reserved (char *blocked_port, gf_boolean_t *ports)
+{
+        gf_boolean_t    result   = _gf_false;
+        char            *range_port = NULL;
+        int16_t         tmp_port1, tmp_port2 = -1;
+
+        if (strstr (blocked_port, "-") == NULL) {
+                /* get rid of the new line character*/
+                if (blocked_port[strlen(blocked_port) -1] == '\n')
+                        blocked_port[strlen(blocked_port) -1] = '\0';
+                if (gf_string2int16 (blocked_port, &tmp_port1) == 0) {
+                        if (tmp_port1 > (GF_CLIENT_PORT_CEILING - 1)
+                            || tmp_port1 < 0) {
+                                gf_log ("glusterfs-socket", GF_LOG_WARNING,
+                                        "invalid port %d", tmp_port1);
+                                result = _gf_true;
+                                goto out;
+                        } else {
+                                gf_log ("glusterfs", GF_LOG_DEBUG,
+                                        "blocking port %d", tmp_port1);
+                                ports[tmp_port1] = _gf_true;
+                        }
+                } else {
+                        gf_log ("glusterfs-socket", GF_LOG_WARNING, "%s is "
+                                "not a valid port identifier", blocked_port);
+                        result = _gf_true;
+                        goto out;
+                }
+        } else {
+                range_port = strtok (blocked_port, "-");
+                if (!range_port){
+                        result = _gf_true;
+                        goto out;
+                }
+                if (gf_string2int16 (range_port, &tmp_port1) == 0) {
+                        if (tmp_port1 > (GF_CLIENT_PORT_CEILING - 1))
+                                tmp_port1 = GF_CLIENT_PORT_CEILING - 1;
+                        if (tmp_port1 < 0)
+                                tmp_port1 = 0;
+                }
+                range_port = strtok (NULL, "-");
+                if (!range_port) {
+                        result = _gf_true;
+                        goto out;
+                }
+                /* get rid of the new line character*/
+                if (range_port[strlen(range_port) -1] == '\n')
+                        range_port[strlen(range_port) - 1] = '\0';
+                if (gf_string2int16 (range_port, &tmp_port2) == 0) {
+                        if (tmp_port2 >
+                            (GF_CLIENT_PORT_CEILING - 1))
+                                tmp_port2 = GF_CLIENT_PORT_CEILING - 1;
+                        if (tmp_port2 < 0)
+                                tmp_port2 = 0;
+                }
+                gf_log ("glusterfs", GF_LOG_DEBUG, "lower: %d, higher: %d",
+                        tmp_port1, tmp_port2);
+                for (; tmp_port1 <= tmp_port2; tmp_port1++)
+                        ports[tmp_port1] = _gf_true;
+        }
+
+out:
+        return result;
+}
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index 183837b9650..6d46a997105 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -76,6 +76,7 @@ void trap (void);
  * nfs port in volume status.
  */
 #define GF_NFS3_PORT    38467
+#define GF_CLIENT_PORT_CEILING 1024
 
 enum _gf_boolean
 {
@@ -553,5 +554,8 @@ uint64_t get_mem_size ();
 int gf_strip_whitespace (char *str, int len);
 int gf_canonicalize_path (char *path);
 char *generate_glusterfs_ctx_id (void);
+char *gf_get_reserved_ports();
+int gf_process_reserved_ports (gf_boolean_t ports[]);
+gf_boolean_t gf_ports_reserved (char *blocked_port, gf_boolean_t *ports);
 
 #endif /* _COMMON_UTILS_H */
diff --git a/rpc/rpc-transport/rdma/src/name.c b/rpc/rpc-transport/rdma/src/name.c
index 3718cd7648e..6b0ee11ff0b 100644
--- a/rpc/rpc-transport/rdma/src/name.c
+++ b/rpc/rpc-transport/rdma/src/name.c
@@ -14,18 +14,14 @@
 #include <netdb.h>
 #include <string.h>
 
-#ifdef CLIENT_PORT_CEILING
-#undef CLIENT_PORT_CEILING
-#endif
-
-#define CLIENT_PORT_CEILING 1024
-
 #ifndef AF_INET_SDP
 #define AF_INET_SDP 27
 #endif
 
 #include "rpc-transport.h"
 #include "rdma.h"
+#include "common-utils.h"
+
 
 int32_t
 gf_resolve_ip6 (const char *hostname,
@@ -38,9 +34,17 @@ static int32_t
 af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
                                  socklen_t sockaddr_len, int ceiling)
 {
-        int32_t ret = -1;
-        /*  struct sockaddr_in sin = {0, }; */
-        uint16_t port = ceiling - 1;
+        int32_t        ret        = -1;
+        uint16_t      port        = ceiling - 1;
+        // by default assume none of the ports are blocked and all are available
+        gf_boolean_t  ports[1024] = {_gf_false,};
+        int           i           = 0;
+
+        ret = gf_process_reserved_ports (ports);
+        if (ret != 0) {
+                for (i = 0; i < 1024; i++)
+                        ports[i] = _gf_false;
+        }
 
         while (port)
         {
@@ -55,7 +59,11 @@ af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
                         ((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
                         break;
                 }
-
+                // ignore the reserved ports
+                if (ports[port] == _gf_true) {
+                        port--;
+                        continue;
+                }
                 ret = bind (fd, sockaddr, sockaddr_len);
 
                 if (ret == 0)
@@ -419,12 +427,12 @@ gf_rdma_client_bind (rpc_transport_t *this,
         case AF_INET6:
                 ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr,
                                                        *sockaddr_len,
-                                                       CLIENT_PORT_CEILING);
+                                                       GF_CLIENT_PORT_CEILING);
                 if (ret == -1) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "cannot bind inet socket (%d) to port "
                                 "less than %d (%s)",
-                                sock, CLIENT_PORT_CEILING, strerror (errno));
+                                sock, GF_CLIENT_PORT_CEILING, strerror (errno));
                         ret = 0;
                 }
                 break;
diff --git a/rpc/rpc-transport/socket/src/name.c b/rpc/rpc-transport/socket/src/name.c
index d37c83e1857..1647d5b6b64 100644
--- a/rpc/rpc-transport/socket/src/name.c
+++ b/rpc/rpc-transport/socket/src/name.c
@@ -15,12 +15,6 @@
 #include <netdb.h>
 #include <string.h>
 
-#ifdef CLIENT_PORT_CEILING
-#undef CLIENT_PORT_CEILING
-#endif
-
-#define CLIENT_PORT_CEILING 1024
-
 #ifndef AF_INET_SDP
 #define AF_INET_SDP 27
 #endif
@@ -40,9 +34,17 @@ static int32_t
 af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
                                  socklen_t sockaddr_len, int ceiling)
 {
-        int32_t ret = -1;
-        /*  struct sockaddr_in sin = {0, }; */
-        uint16_t port = ceiling - 1;
+        int32_t        ret        = -1;
+        uint16_t      port        = ceiling - 1;
+        // by default assume none of the ports are blocked and all are available
+        gf_boolean_t  ports[1024] = {_gf_false,};
+        int           i           = 0;
+
+        ret = gf_process_reserved_ports (ports);
+        if (ret != 0) {
+                for (i = 0; i < 1024; i++)
+                        ports[i] = _gf_false;
+        }
 
         while (port)
         {
@@ -57,7 +59,11 @@ af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
                         ((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
                         break;
                 }
-
+                // ignore the reserved ports
+                if (ports[port] == _gf_true) {
+                        port--;
+                        continue;
+                }
                 ret = bind (fd, sockaddr, sockaddr_len);
 
                 if (ret == 0)
@@ -440,12 +446,12 @@ client_bind (rpc_transport_t *this,
         case AF_INET6:
                 if (!this->bind_insecure) {
                         ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr,
-                                                       *sockaddr_len, CLIENT_PORT_CEILING);
+                                                       *sockaddr_len, GF_CLIENT_PORT_CEILING);
                 }
                 if (ret == -1) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "cannot bind inet socket (%d) to port less than %d (%s)",
-                                sock, CLIENT_PORT_CEILING, strerror (errno));
+                                sock, GF_CLIENT_PORT_CEILING, strerror (errno));
                         ret = 0;
                 }
                 break;
diff --git a/tests/bugs/bug-762989.t b/tests/bugs/bug-762989.t
new file mode 100755
index 00000000000..386aec1cd47
--- /dev/null
+++ b/tests/bugs/bug-762989.t
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+## reserve port 1023
+older_ports=$(cat /proc/sys/net/ipv4/ip_local_reserved_ports);
+echo "1023" > /proc/sys/net/ipv4/ip_local_reserved_ports;
+
+## Start and create a volume
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+TEST $CLI volume start $V0;
+
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 \
+$M0;
+
+## Wait for volume to register with rpc.mountd
+sleep 6;
+## check if port 1023 (which has been reserved) is used by the gluster processes
+op=$(netstat -antlp | grep gluster | grep 1023);
+EXPECT "" echo $op;
+
+#set the reserved ports to the older values
+echo $older_ports > /proc/sys/net/ipv4/ip_local_reserved_ports
+
+cleanup;