summaryrefslogtreecommitdiffstats
path: root/xlators/mgmt/glusterd/src
diff options
context:
space:
mode:
authorPrasanna Kumar Kalever <prasanna.kalever@redhat.com>2016-04-27 19:12:19 +0530
committerRaghavendra G <rgowdapp@redhat.com>2016-05-10 00:11:33 -0700
commit610a3f5bcc9f3443da55d857b162c83d50fa3a6b (patch)
treef8f0f053a2fc5d9b9266ae88a657629826a5c9c0 /xlators/mgmt/glusterd/src
parent4c58dd7f03e393b6dd5c01af3e7f4c786ba12e3f (diff)
glusterd: add defence mechanism to avoid brick port clashes
Intro: Currently glusterd maintain the portmap registry which contains ports that are free to use between 49152 - 65535, this registry is initialized once, and updated accordingly as an then when glusterd sees they are been used. Glusterd first checks for a port within the portmap registry and gets a FREE port marked in it, then checks if that port is currently free using a connect() function then passes it to brick process which have to bind on it. Problem: We see that there is a time gap between glusterd checking the port with connect() and brick process actually binding on it. In this time gap it could be so possible that any process would have occupied this port because of which brick will fail to bind and exit. Case 1: To avoid the gluster client process occupying the port supplied by glusterd : we have separated the client port map range with brick port map range more @ http://review.gluster.org/#/c/13998/ Case 2: (Handled by this patch) To avoid the other foreign process occupying the port supplied by glusterd : To handle above situation this patch implements a mechanism to return EADDRINUSE error code to glusterd, upon which a new port is allocated and try to restart the brick process with the newly allocated port. Note: Incase of glusterd restarts i.e. runner_run_nowait() there is no way to handle Case 2, becuase runner_run_nowait() will not wait to get the return/exit code of the executed command (brick process). Hence as of now in such case, we cannot know with what error the brick has failed to connect. This patch also fix the runner_end() to perform some cleanup w.r.t return values. Backport of: > Change-Id: Iec52e7f5d87ce938d173f8ef16aa77fd573f2c5e > BUG: 1322805 > Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com> > Reviewed-on: http://review.gluster.org/14043 > Tested-by: Prasanna Kumar Kalever <pkalever@redhat.com> > Reviewed-by: Atin Mukherjee <amukherj@redhat.com> > Smoke: Gluster Build System <jenkins@build.gluster.com> > NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org> > CentOS-regression: Gluster Build System <jenkins@build.gluster.com> > Reviewed-by: Raghavendra G <rgowdapp@redhat.com> Change-Id: Id7d8351a0082b44310177e714edc0571ad0f7195 BUG: 1333711 Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com> Reviewed-on: http://review.gluster.org/14235 Tested-by: Prasanna Kumar Kalever <pkalever@redhat.com> Smoke: Gluster Build System <jenkins@build.gluster.com> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org> CentOS-regression: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Diffstat (limited to 'xlators/mgmt/glusterd/src')
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-messages.h11
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-utils.c22
2 files changed, 32 insertions, 1 deletions
diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h
index b49d87da1bf..61ba7bd3179 100644
--- a/xlators/mgmt/glusterd/src/glusterd-messages.h
+++ b/xlators/mgmt/glusterd/src/glusterd-messages.h
@@ -41,7 +41,7 @@
#define GLUSTERD_COMP_BASE GLFS_MSGID_GLUSTERD
-#define GLFS_NUM_MESSAGES 573
+#define GLFS_NUM_MESSAGES 575
#define GLFS_MSGID_END (GLUSTERD_COMP_BASE + GLFS_NUM_MESSAGES + 1)
/* Messaged with message IDs */
@@ -4641,6 +4641,15 @@
*/
#define GD_MSG_FILE_NOT_FOUND (GLUSTERD_COMP_BASE + 574)
+/*!
+ * @messageid 106575
+ * @diagnosis Brick failed to start with given port, hence it gets a fresh port
+ * on its own and try to restart the brick with a new port
+ * @recommendedaction Ensure the new port is not blocked by firewall
+ */
+
+#define GD_MSG_RETRY_WITH_NEW_PORT (GLUSTERD_COMP_BASE + 575)
+
/*------------*/
#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
#endif /* !_GLUSTERD_MESSAGES_H_ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 68208312acb..5c28ec942ca 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1808,6 +1808,8 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo,
writing the valgrind log to the same file.
*/
GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);
+
+retry:
runinit (&runner);
if (priv->valgrind) {
@@ -1899,6 +1901,26 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo,
ret = runner_run (&runner);
synclock_lock (&priv->big_lock);
+ if (ret == -EADDRINUSE) {
+ /* retry after getting a new port */
+ gf_msg (this->name, GF_LOG_WARNING, -ret,
+ GD_MSG_SRC_BRICK_PORT_UNAVAIL,
+ "Port %d is used by other process", port);
+
+ port = pmap_registry_alloc (this);
+ if (!port) {
+ gf_msg (this->name, GF_LOG_CRITICAL, 0,
+ GD_MSG_NO_FREE_PORTS,
+ "Couldn't allocate a port");
+ ret = -1;
+ goto out;
+ }
+ gf_msg (this->name, GF_LOG_NOTICE, 0,
+ GD_MSG_RETRY_WITH_NEW_PORT,
+ "Retrying to start brick %s with new port %d",
+ brickinfo->path, port);
+ goto retry;
+ }
} else {
ret = runner_run_nowait (&runner);
}