From 5ebf298ec03bc929a4142e70ed105130cf9c58df Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Fri, 24 Jul 2015 17:35:16 +0530 Subject: rpc: fix binding brick issue while bind-insecure is enabled This patch is backport of http://review.gluster.org/#/c/11512/ > problem: > When bind-insecure is turned on (which is the default now), it may happen > that brick is not able to bind to port assigned by Glusterd for example > 49192-49195... > > It seems to occur because the rpc_clnt connections are binding to ports in > the same range. so brick fails to bind to a port which is already used by > someone else > > solution: > > fix for now is to make rpc_clnt to get port numbers from 65535 in a > descending > order, as a result port clash is minimized > > other fixes: > > previously rdma binds to port >= 1024 if it cannot find a free port < 1024, > even when bind insecure was turned off(ref to commit '0e3fd04e'), this patch > add's a check for bind-insecure in gf_rdma_client_bind function > > This patch also re-enable bind-insecure and allow insecure by default > which was reverted (ref: commit cef1720) previously > Change-Id: Ia1cfa93c5454e2ae0ff57813689b75de282ebd07 > BUG: 1238661 > Signed-off-by: Prasanna Kumar Kalever Change-Id: Iea55f9b2a57b5e24d3df2c5fafae12fe99e9dee0 BUG: 1246481 Signed-off-by: Prasanna Kumar Kalever Reviewed-on: http://review.gluster.org/11758 Tested-by: NetBSD Build System Tested-by: Gluster Build System Reviewed-by: Raghavendra G --- libglusterfs/src/common-utils.c | 17 +++++----- libglusterfs/src/common-utils.h | 6 ++-- rpc/rpc-lib/src/rpc-transport.c | 3 +- rpc/rpc-lib/src/rpcsvc-auth.c | 13 ++++++- rpc/rpc-lib/src/rpcsvc.c | 6 ++-- rpc/rpc-transport/rdma/src/name.c | 67 +++++++++++++++++++++---------------- rpc/rpc-transport/socket/src/name.c | 56 +++++++++++++++++++------------ 7 files changed, 103 insertions(+), 65 deletions(-) diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c index 284c444ccfd..6a9e1a6ea65 100644 --- a/libglusterfs/src/common-utils.c +++ b/libglusterfs/src/common-utils.c @@ -2806,7 +2806,7 @@ out: } int -gf_process_reserved_ports (gf_boolean_t *ports) +gf_process_reserved_ports (gf_boolean_t *ports, uint32_t ceiling) { int ret = -1; #if defined GF_LINUX_HOST_OS @@ -2826,7 +2826,7 @@ gf_process_reserved_ports (gf_boolean_t *ports) blocked_port = strtok_r (ports_info, ",\n",&tmp); while (blocked_port) { - gf_ports_reserved (blocked_port, ports); + gf_ports_reserved (blocked_port, ports, ceiling); blocked_port = strtok_r (NULL, ",\n", &tmp); } @@ -2839,7 +2839,7 @@ out: } gf_boolean_t -gf_ports_reserved (char *blocked_port, gf_boolean_t *ports) +gf_ports_reserved (char *blocked_port, gf_boolean_t *ports, uint32_t ceiling) { gf_boolean_t result = _gf_false; char *range_port = NULL; @@ -2850,7 +2850,7 @@ gf_ports_reserved (char *blocked_port, gf_boolean_t *ports) if (blocked_port[strlen(blocked_port) -1] == '\n') blocked_port[strlen(blocked_port) -1] = '\0'; if (gf_string2int16 (blocked_port, &tmp_port1) == 0) { - if (tmp_port1 > (GF_CLIENT_PORT_CEILING - 1) + if (tmp_port1 > ceiling || tmp_port1 < 0) { gf_msg ("glusterfs-socket", GF_LOG_WARNING, 0, LG_MSG_INVALID_PORT, "invalid port %d", @@ -2876,8 +2876,8 @@ gf_ports_reserved (char *blocked_port, gf_boolean_t *ports) goto out; } if (gf_string2int16 (range_port, &tmp_port1) == 0) { - if (tmp_port1 > (GF_CLIENT_PORT_CEILING - 1)) - tmp_port1 = GF_CLIENT_PORT_CEILING - 1; + if (tmp_port1 > ceiling) + tmp_port1 = ceiling; if (tmp_port1 < 0) tmp_port1 = 0; } @@ -2890,9 +2890,8 @@ gf_ports_reserved (char *blocked_port, gf_boolean_t *ports) if (range_port[strlen(range_port) -1] == '\n') range_port[strlen(range_port) - 1] = '\0'; if (gf_string2int16 (range_port, &tmp_port2) == 0) { - if (tmp_port2 > - (GF_CLIENT_PORT_CEILING - 1)) - tmp_port2 = GF_CLIENT_PORT_CEILING - 1; + if (tmp_port2 > ceiling) + tmp_port2 = ceiling; if (tmp_port2 < 0) tmp_port2 = 0; } diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h index 5302a47cb1d..67728350508 100644 --- a/libglusterfs/src/common-utils.h +++ b/libglusterfs/src/common-utils.h @@ -88,6 +88,7 @@ void trap (void); */ #define GF_NFS3_PORT 2049 #define GF_CLIENT_PORT_CEILING 1024 +#define GF_PORT_MAX 65535 #define GF_MINUTE_IN_SECONDS 60 #define GF_HOUR_IN_SECONDS (60*60) @@ -703,8 +704,9 @@ int gf_strip_whitespace (char *str, int len); int gf_canonicalize_path (char *path); char *generate_glusterfs_ctx_id (void); char *gf_get_reserved_ports(); -int gf_process_reserved_ports (gf_boolean_t ports[]); -gf_boolean_t gf_ports_reserved (char *blocked_port, gf_boolean_t *ports); +int gf_process_reserved_ports (gf_boolean_t ports[], uint32_t ceiling); +gf_boolean_t +gf_ports_reserved (char *blocked_port, gf_boolean_t *ports, uint32_t ceiling); int gf_get_hostname_from_ip (char *client_ip, char **hostname); gf_boolean_t gf_is_local_addr (char *hostname); gf_boolean_t gf_is_same_address (char *host1, char *host2); diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c index 23fbf37360d..0a791abfddd 100644 --- a/rpc/rpc-lib/src/rpc-transport.c +++ b/rpc/rpc-lib/src/rpc-transport.c @@ -267,7 +267,8 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name) else trans->bind_insecure = 0; } else { - trans->bind_insecure = 0; + /* By default allow bind insecure */ + trans->bind_insecure = 1; } ret = dict_get_str (options, "transport-type", &type); diff --git a/rpc/rpc-lib/src/rpcsvc-auth.c b/rpc/rpc-lib/src/rpcsvc-auth.c index 6b4c7937437..b7d6c2216ef 100644 --- a/rpc/rpc-lib/src/rpcsvc-auth.c +++ b/rpc/rpc-lib/src/rpcsvc-auth.c @@ -221,9 +221,20 @@ rpcsvc_set_allow_insecure (rpcsvc_t *svc, dict_t *options) else svc->allow_insecure = 0; } + } else { + /* By default set allow-insecure to true */ + svc->allow_insecure = 1; + + /* setting in options for the sake of functions that look + * configuration params for allow insecure, eg: gf_auth + */ + ret = dict_set_str (options, "rpc-auth-allow-insecure", "on"); + if (ret < 0) + gf_log ("rpc-auth", GF_LOG_DEBUG, + "dict_set failed for 'allow-insecure'"); } - return 0; + return ret; } int diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c index be95d25b1b1..8eb38ed8eff 100644 --- a/rpc/rpc-lib/src/rpcsvc.c +++ b/rpc/rpc-lib/src/rpcsvc.c @@ -636,8 +636,10 @@ rpcsvc_handle_rpc_call (rpcsvc_t *svc, rpc_transport_t *trans, gf_log (GF_RPCSVC, GF_LOG_ERROR, "Request received from non-" "privileged port. Failing request"); - rpcsvc_request_destroy (req); - return -1; + req->rpc_status = MSG_DENIED; + req->rpc_err = AUTH_ERROR; + req->auth_err = RPCSVC_AUTH_REJECT; + goto err_reply; } /* DRC */ diff --git a/rpc/rpc-transport/rdma/src/name.c b/rpc/rpc-transport/rdma/src/name.c index 8b403ed4c4c..f723efc2360 100644 --- a/rpc/rpc-transport/rdma/src/name.c +++ b/rpc/rpc-transport/rdma/src/name.c @@ -33,36 +33,41 @@ gf_resolve_ip6 (const char *hostname, void **dnscache, struct addrinfo **addr_info); + +static void +_assign_port (struct sockaddr *sockaddr, uint16_t port) +{ + switch (sockaddr->sa_family) { + case AF_INET6: + ((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port); + break; + + case AF_INET_SDP: + case AF_INET: + ((struct sockaddr_in *)sockaddr)->sin_port = htons (port); + break; + } +} + static int32_t af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id, struct sockaddr *sockaddr, - socklen_t sockaddr_len, int ceiling) + socklen_t sockaddr_len, uint32_t ceiling) { int32_t ret = -1; uint16_t port = ceiling - 1; /* by default assume none of the ports are blocked and all are available */ - gf_boolean_t ports[1024] = {_gf_false,}; + gf_boolean_t ports[GF_PORT_MAX] = {_gf_false,}; int i = 0; - ret = gf_process_reserved_ports (ports); + ret = gf_process_reserved_ports (ports, ceiling); if (ret != 0) { - for (i = 0; i < 1024; i++) + for (i = 0; i < GF_PORT_MAX; i++) ports[i] = _gf_false; } while (port) { - switch (sockaddr->sa_family) { - case AF_INET6: - ((struct sockaddr_in6 *)sockaddr)->sin6_port - = htons (port); - break; - - case AF_INET_SDP: - case AF_INET: - ((struct sockaddr_in *)sockaddr)->sin_port - = htons (port); - break; - } + _assign_port (sockaddr, port); /* ignore the reserved ports */ if (ports[port] == _gf_true) { port--; @@ -425,22 +430,26 @@ gf_rdma_client_bind (rpc_transport_t *this, struct sockaddr *sockaddr, *sockaddr_len = sizeof (struct sockaddr_in); case AF_INET6: - ret = af_inet_bind_to_port_lt_ceiling (cm_id, sockaddr, + if (!this->bind_insecure) { + ret = af_inet_bind_to_port_lt_ceiling (cm_id, sockaddr, *sockaddr_len, GF_CLIENT_PORT_CEILING); - if (ret == -1) { - gf_msg (this->name, GF_LOG_WARNING, errno, - RDMA_MSG_PORT_BIND_FAILED, - "cannot bind rdma_cm_id to port " - "less than %d", GF_CLIENT_PORT_CEILING); - if (sockaddr->sa_family == AF_INET6) { - ((struct sockaddr_in6 *)sockaddr)->sin6_port - = htons (0); - } else { - ((struct sockaddr_in *)sockaddr)->sin_port - = htons (0); + if (ret == -1) { + gf_msg (this->name, GF_LOG_WARNING, errno, + RDMA_MSG_PORT_BIND_FAILED, + "cannot bind rdma_cm_id to port " + "less than %d", GF_CLIENT_PORT_CEILING); + } + } else { + ret = af_inet_bind_to_port_lt_ceiling (cm_id, sockaddr, + *sockaddr_len, + GF_PORT_MAX); + if (ret == -1) { + gf_msg (this->name, GF_LOG_WARNING, errno, + RDMA_MSG_PORT_BIND_FAILED, + "cannot bind rdma_cm_id to port " + "less than %d", GF_PORT_MAX); } - ret = rdma_bind_addr (cm_id, sockaddr); } break; diff --git a/rpc/rpc-transport/socket/src/name.c b/rpc/rpc-transport/socket/src/name.c index f731bab4b0a..650c5a747be 100644 --- a/rpc/rpc-transport/socket/src/name.c +++ b/rpc/rpc-transport/socket/src/name.c @@ -23,35 +23,40 @@ #include "socket.h" #include "common-utils.h" +static void +_assign_port (struct sockaddr *sockaddr, uint16_t port) +{ + switch (sockaddr->sa_family) { + case AF_INET6: + ((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port); + break; + + case AF_INET_SDP: + case AF_INET: + ((struct sockaddr_in *)sockaddr)->sin_port = htons (port); + break; + } +} + static int32_t af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr, - socklen_t sockaddr_len, int ceiling) + socklen_t sockaddr_len, uint32_t ceiling) { int32_t ret = -1; uint16_t port = ceiling - 1; // by default assume none of the ports are blocked and all are available - gf_boolean_t ports[1024] = {_gf_false,}; + gf_boolean_t ports[GF_PORT_MAX] = {_gf_false,}; int i = 0; - ret = gf_process_reserved_ports (ports); + ret = gf_process_reserved_ports (ports, ceiling); if (ret != 0) { - for (i = 0; i < 1024; i++) + for (i = 0; i < GF_PORT_MAX; i++) ports[i] = _gf_false; } while (port) { - switch (sockaddr->sa_family) - { - case AF_INET6: - ((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port); - break; - - case AF_INET_SDP: - case AF_INET: - ((struct sockaddr_in *)sockaddr)->sin_port = htons (port); - break; - } + _assign_port (sockaddr, port); // ignore the reserved ports if (ports[port] == _gf_true) { port--; @@ -440,12 +445,21 @@ client_bind (rpc_transport_t *this, if (!this->bind_insecure) { ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr, *sockaddr_len, GF_CLIENT_PORT_CEILING); - } - if (ret == -1) { - gf_log (this->name, GF_LOG_DEBUG, - "cannot bind inet socket (%d) to port less than %d (%s)", - sock, GF_CLIENT_PORT_CEILING, strerror (errno)); - ret = 0; + if (ret == -1) { + gf_log (this->name, GF_LOG_DEBUG, + "cannot bind inet socket (%d) to port less than %d (%s)", + sock, GF_CLIENT_PORT_CEILING, strerror (errno)); + ret = 0; + } + } else { + ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr, + *sockaddr_len, GF_PORT_MAX); + if (ret == -1) { + gf_log (this->name, GF_LOG_DEBUG, + "failed while binding to less than %d (%s)", + GF_PORT_MAX, strerror (errno)); + ret = 0; + } } break; -- cgit