glusterd: Allocate fresh port on brick (re)start

There is no point of using the same port through the entire volume life cycle
for a particular bricks process since there is no guarantee that the same port
would be free and no other application wouldn't consume it in between the
glusterd/volume restart.

We hit a race where on glusterd restart the daemon services start followed by
brick processes and the time brick process tries to bind with the port which was
allocated by glusterd before a restart is been already consumed by some other
client like NFS/SHD/...

Note : This is a short term solution as here we reduce the race window but don't
eliminate it completely. As a long term solution the port allocation has to be
done by glusterfsd and the same should be communicated back to glusterd for book
keeping

Change-Id: Ibbd1e7ca87e51a7cd9cf216b1fe58ef7783aef24
BUG: 1322805
Signed-off-by: Atin Mukherjee <amukherj@redhat.com>
Reviewed-on: http://review.gluster.org/13865
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>

1 files changed, 32 insertions, 6 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index a0cc5d409c8..55597cd39cf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1725,9 +1725,22 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         if (gf_is_service_running (pidfile, NULL))
                 goto connect;
 
-        port = brickinfo->port;
-        if (!port)
-                port = pmap_registry_alloc (THIS);
+        /* Do a pmap registry remove on the older connected port */
+        if (brickinfo->port) {
+                ret = pmap_registry_remove (this, brickinfo->port,
+                                            brickinfo->path,
+                                            GF_PMAP_PORT_BRICKSERVER,
+                                            NULL);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_PMAP_REMOVE_FAIL, "Failed to remove pmap"
+                                "registry for older signin port %d",
+                                brickinfo->port);
+                        goto out;
+                }
+        }
+
+        port = pmap_registry_alloc (THIS);
 
         /* Build the exp_path, before starting the glusterfsd even in
            valgrind mode. Otherwise all the glusterfsd processes start
@@ -1790,9 +1803,22 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) {
                 runner_argprintf (&runner, "%d", port);
         } else {
-                rdma_port = brickinfo->rdma_port;
-                if (!rdma_port)
-                        rdma_port = pmap_registry_alloc (THIS);
+                /* Do a pmap registry remove on the older connected port */
+                if (brickinfo->rdma_port) {
+                        ret = pmap_registry_remove (this, brickinfo->rdma_port,
+                                                    brickinfo->path,
+                                                    GF_PMAP_PORT_BRICKSERVER,
+                                                    NULL);
+                        if (ret) {
+                                gf_msg (this->name, GF_LOG_ERROR, 0,
+                                        GD_MSG_PMAP_REMOVE_FAIL, "Failed to "
+                                        "remove pmap registry for older signin "
+                                        "port %d", brickinfo->rdma_port);
+                                goto out;
+                        }
+                }
+                rdma_port = pmap_registry_alloc (THIS);
+
                 runner_argprintf (&runner, "%d,%d", port, rdma_port);
                 runner_add_arg (&runner, "--xlator-option");
                 runner_argprintf (&runner, "%s-server.transport.rdma.listen-port=%d",