glusterd: introduce max-port range

glusterd.vol file always had an option (commented out) to indicate the
base-port to start the portmapper allocation. This patch brings in the
max-port configuration where one can limit the range of ports which
gluster can be allowed to bind.

Fixes: #305
Change-Id: Id7a864f818227b9530a07e13d605138edacd9aa9
Signed-off-by: Atin Mukherjee <amukherj@redhat.com>
Reviewed-on: https://review.gluster.org/18016
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Prashanth Pai <ppai@redhat.com>
Reviewed-by: Niels de Vos <ndevos@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Gaurav Yadav <gyadav@redhat.com>

7 files changed, 63 insertions, 14 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h
index eebc14901a1..dbc146babc7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-messages.h
+++ b/xlators/mgmt/glusterd/src/glusterd-messages.h
@@ -41,7 +41,7 @@
 
 #define GLUSTERD_COMP_BASE      GLFS_MSGID_GLUSTERD
 
-#define GLFS_NUM_MESSAGES       611
+#define GLFS_NUM_MESSAGES       612
 
 #define GLFS_MSGID_END          (GLUSTERD_COMP_BASE + GLFS_NUM_MESSAGES + 1)
 /* Messaged with message IDs */
@@ -4937,6 +4937,14 @@
  */
 #define GD_MSG_LOCALTIME_LOGGING_DISABLE (GLUSTERD_COMP_BASE + 611)
 
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+#define GD_MSG_PORTS_EXHAUSTED (GLUSTERD_COMP_BASE + 612)
+
 /*------------*/
 
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.c b/xlators/mgmt/glusterd/src/glusterd-pmap.c
index 2a754769c95..6d8cf3f894f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.c
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.c
@@ -61,8 +61,8 @@ pmap_registry_new (xlator_t *this)
 
         pmap->base_port = pmap->last_alloc =
                 ((glusterd_conf_t *)(this->private))->base_port;
-
-        for (i = pmap->base_port; i <= GF_PORT_MAX; i++) {
+        pmap->max_port = ((glusterd_conf_t *)(this->private))->max_port;
+        for (i = pmap->base_port; i <= pmap->max_port; i++) {
                 if (pmap_port_isfree (i))
                         pmap->ports[i].type = GF_PMAP_PORT_FREE;
                 else
@@ -184,10 +184,12 @@ pmap_registry_search_by_xprt (xlator_t *this, void *xprt,
 static char *
 pmap_registry_search_by_port (xlator_t *this, int port)
 {
-        struct pmap_registry *pmap = NULL;
-        char *brickname = NULL;
+        struct  pmap_registry *pmap = NULL;
+        char   *brickname           = NULL;
+        int     max_port            = 0;
 
-        if (port > GF_PORT_MAX)
+        max_port = ((glusterd_conf_t *)(this->private))->max_port;
+        if (port > max_port)
                 goto out;
 
         pmap = pmap_registry_get (this);
@@ -209,7 +211,7 @@ pmap_registry_alloc (xlator_t *this)
 
         pmap = pmap_registry_get (this);
 
-        for (p = pmap->base_port; p <= GF_PORT_MAX; p++) {
+        for (p = pmap->base_port; p <= pmap->max_port; p++) {
                 /* GF_PMAP_PORT_FOREIGN may be freed up ? */
                 if ((pmap->ports[p].type == GF_PMAP_PORT_FREE) ||
                     (pmap->ports[p].type == GF_PMAP_PORT_FOREIGN)) {
@@ -260,7 +262,7 @@ pmap_registry_bind (xlator_t *this, int port, const char *brickname,
 
         pmap = pmap_registry_get (this);
 
-        if (port > GF_PORT_MAX)
+        if (port > pmap->max_port)
                 goto out;
 
         p = port;
@@ -296,7 +298,7 @@ pmap_registry_extend (xlator_t *this, int port, const char *brickname)
 
         pmap = pmap_registry_get (this);
 
-        if (port > GF_PORT_MAX) {
+        if (port > pmap->max_port) {
                 return -1;
         }
 
@@ -355,7 +357,7 @@ pmap_registry_remove (xlator_t *this, int port, const char *brickname,
                 goto out;
 
         if (port) {
-                if (port > GF_PORT_MAX)
+                if (port > pmap->max_port)
                         goto out;
 
                 p = port;
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.h b/xlators/mgmt/glusterd/src/glusterd-pmap.h
index 9965a9577b5..c0475db539e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.h
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.h
@@ -31,8 +31,9 @@ struct pmap_port_status {
 
 struct pmap_registry {
         int     base_port;
+        int     max_port;
         int     last_alloc;
-        struct  pmap_port_status ports[65536];
+        struct  pmap_port_status ports[GF_PORT_MAX + 1];
 };
 
 int pmap_assign_port (xlator_t *this, int port, const char *path);
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c
index ea56d8012aa..b6e6a814ca2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c
@@ -302,6 +302,14 @@ glusterd_snapdsvc_start (glusterd_svc_t *svc, int flags)
                          NULL);
 
         snapd_port = pmap_assign_port (THIS, volinfo->snapd.port, snapd_id);
+        if (!snapd_port) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_PORTS_EXHAUSTED,
+                        "All the ports in the range are exhausted, can't start "
+                        "snapd for volume %s", volinfo->volname);
+                ret = -1;
+                goto out;
+        }
+
         volinfo->snapd.port = snapd_port;
 
         runner_add_arg (&runner, "--brick-port");
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 9e98225e73e..dfdd795f186 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -2002,7 +2002,14 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         }
 
         port = pmap_assign_port (THIS, brickinfo->port, brickinfo->path);
-
+        if (!port) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_PORTS_EXHAUSTED,
+                        "All the ports in the range are exhausted, can't start "
+                        "brick %s for volume %s", brickinfo->path,
+                        volinfo->volname);
+                ret = -1;
+                goto out;
+        }
         /* Build the exp_path, before starting the glusterfsd even in
            valgrind mode. Otherwise all the glusterfsd processes start
            writing the valgrind log to the same file.
@@ -2077,6 +2084,15 @@ retry:
                           brickinfo->path);
                 rdma_port = pmap_assign_port (THIS, brickinfo->rdma_port,
                                               rdma_brick_path);
+                if (!rdma_port) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_PORTS_EXHAUSTED, "All rdma ports in the "
+                                "range are exhausted, can't start brick %s for "
+                                "volume %s", rdma_brick_path,
+                                volinfo->volname);
+                        ret = -1;
+                        goto out;
+                }
                 runner_argprintf (&runner, "%d,%d", port, rdma_port);
                 runner_add_arg (&runner, "--xlator-option");
                 runner_argprintf (&runner, "%s-server.transport.rdma.listen-port=%d",
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 4887ff413bf..556423f2f7f 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1823,12 +1823,20 @@ init (xlator_t *this)
         if (ret)
                 goto out;
 
-         conf->base_port = GF_IANA_PRIV_PORTS_START;
-         if (dict_get_uint32(this->options, "base-port", &conf->base_port) == 0) {
+        conf->base_port = GF_IANA_PRIV_PORTS_START;
+        if (dict_get_uint32 (this->options, "base-port",
+                             &conf->base_port) == 0) {
                 gf_msg (this->name, GF_LOG_INFO, 0,
                         GD_MSG_DICT_SET_FAILED,
                         "base-port override: %d", conf->base_port);
          }
+         conf->max_port = GF_PORT_MAX;
+         if (dict_get_uint32 (this->options, "max-port",
+                              &conf->max_port) == 0) {
+                gf_msg (this->name, GF_LOG_INFO, 0,
+                        GD_MSG_DICT_SET_FAILED,
+                        "max-port override: %d", conf->max_port);
+         }
 
         /* Set option to run bricks on valgrind if enabled in glusterd.vol */
         this->ctx->cmd_args.valgrind = valgrind;
@@ -2134,6 +2142,11 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_INT,
           .description = "Sets the base port for portmap query"
         },
+        { .key = {"max-port"},
+          .type = GF_OPTION_TYPE_INT,
+          .max = GF_PORT_MAX,
+          .description = "Sets the max port for portmap query"
+        },
         { .key = {"snap-brick-path"},
           .type = GF_OPTION_TYPE_STR,
           .description = "directory where the bricks for the snapshots will be created"
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 722d2f8f420..8beec6c29cf 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -187,6 +187,7 @@ typedef struct {
         gf_boolean_t               restart_done;
         rpcsvc_t                  *uds_rpc; /* RPCSVC for the unix domain socket */
         uint32_t                   base_port;
+        uint32_t                   max_port;
         char                      *snap_bricks_directory;
         gf_store_handle_t         *missed_snaps_list_shandle;
         struct cds_list_head       missed_snaps_list;