From 34899d71f21fd2b4c523b68ffb2d7c655c776641 Mon Sep 17 00:00:00 2001 From: Atin Mukherjee Date: Thu, 31 Mar 2016 11:01:53 +0530 Subject: glusterd: Allocate fresh port on brick (re)start There is no point of using the same port through the entire volume life cycle for a particular bricks process since there is no guarantee that the same port would be free and no other application wouldn't consume it in between the glusterd/volume restart. We hit a race where on glusterd restart the daemon services start followed by brick processes and the time brick process tries to bind with the port which was allocated by glusterd before a restart is been already consumed by some other client like NFS/SHD/... Note : This is a short term solution as here we reduce the race window but don't eliminate it completely. As a long term solution the port allocation has to be done by glusterfsd and the same should be communicated back to glusterd for book keeping Change-Id: Ibbd1e7ca87e51a7cd9cf216b1fe58ef7783aef24 BUG: 1322805 Signed-off-by: Atin Mukherjee Reviewed-on: http://review.gluster.org/13865 Smoke: Gluster Build System NetBSD-regression: NetBSD Build System CentOS-regression: Gluster Build System Reviewed-by: Jeff Darcy --- xlators/mgmt/glusterd/src/glusterd-messages.h | 10 ++++++- xlators/mgmt/glusterd/src/glusterd-store.c | 16 ----------- xlators/mgmt/glusterd/src/glusterd-utils.c | 38 ++++++++++++++++++++++----- 3 files changed, 41 insertions(+), 23 deletions(-) (limited to 'xlators') diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h index ffcfa7773e9..04d630001df 100644 --- a/xlators/mgmt/glusterd/src/glusterd-messages.h +++ b/xlators/mgmt/glusterd/src/glusterd-messages.h @@ -41,7 +41,7 @@ #define GLUSTERD_COMP_BASE GLFS_MSGID_GLUSTERD -#define GLFS_NUM_MESSAGES 573 +#define GLFS_NUM_MESSAGES 574 #define GLFS_MSGID_END (GLUSTERD_COMP_BASE + GLFS_NUM_MESSAGES + 1) /* Messaged with message IDs */ @@ -4633,6 +4633,14 @@ */ #define GD_MSG_SNAP_RESUME_TIER_FAIL (GLUSTERD_COMP_BASE + 573) +/*! + * @messageid + * @diagnosis + * @recommendedaction + * + */ +#define GD_MSG_PMAP_REMOVE_FAIL (GLUSTERD_COMP_BASE + 574) + /*------------*/ #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages" #endif /* !_GLUSTERD_MESSAGES_H_ */ diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c index b1f943074f3..97f5147dd39 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.c +++ b/xlators/mgmt/glusterd/src/glusterd-store.c @@ -2313,13 +2313,6 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo) /* This is required to adhere to the IANA standards */ brickinfo->port = 0; - } else { - /* This is required to have proper ports - assigned to bricks after restart */ - pmap = pmap_registry_get (THIS); - if (pmap->last_alloc <= brickinfo->port) - pmap->last_alloc = - brickinfo->port + 1; } } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT, strlen (GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) { @@ -2329,16 +2322,7 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo) /* This is required to adhere to the IANA standards */ brickinfo->rdma_port = 0; - } else { - /* This is required to have proper ports - assigned to bricks after restart */ - pmap = pmap_registry_get (THIS); - if (pmap->last_alloc <= - brickinfo->rdma_port) - pmap->last_alloc = - brickinfo->rdma_port +1; } - } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED, strlen (GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) { gf_string2int (value, &brickinfo->decommissioned); diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index a0cc5d409c8..55597cd39cf 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -1725,9 +1725,22 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo, if (gf_is_service_running (pidfile, NULL)) goto connect; - port = brickinfo->port; - if (!port) - port = pmap_registry_alloc (THIS); + /* Do a pmap registry remove on the older connected port */ + if (brickinfo->port) { + ret = pmap_registry_remove (this, brickinfo->port, + brickinfo->path, + GF_PMAP_PORT_BRICKSERVER, + NULL); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_PMAP_REMOVE_FAIL, "Failed to remove pmap" + "registry for older signin port %d", + brickinfo->port); + goto out; + } + } + + port = pmap_registry_alloc (THIS); /* Build the exp_path, before starting the glusterfsd even in valgrind mode. Otherwise all the glusterfsd processes start @@ -1790,9 +1803,22 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo, if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) { runner_argprintf (&runner, "%d", port); } else { - rdma_port = brickinfo->rdma_port; - if (!rdma_port) - rdma_port = pmap_registry_alloc (THIS); + /* Do a pmap registry remove on the older connected port */ + if (brickinfo->rdma_port) { + ret = pmap_registry_remove (this, brickinfo->rdma_port, + brickinfo->path, + GF_PMAP_PORT_BRICKSERVER, + NULL); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_PMAP_REMOVE_FAIL, "Failed to " + "remove pmap registry for older signin " + "port %d", brickinfo->rdma_port); + goto out; + } + } + rdma_port = pmap_registry_alloc (THIS); + runner_argprintf (&runner, "%d,%d", port, rdma_port); runner_add_arg (&runner, "--xlator-option"); runner_argprintf (&runner, "%s-server.transport.rdma.listen-port=%d", -- cgit