glusterd: Made dst brick's port info available to all peers

Change-Id: I1f65743a31d95013fdf22cded91c314e9934a3a9
BUG: 816915
Signed-off-by: Krishnan Parthasarathi <kp@gluster.com>
Reviewed-on: http://review.gluster.org/3275
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

3 files changed, 73 insertions, 22 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 94f498d33bc..5948251201a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -193,6 +193,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                                  dict_t *rsp_dict)
 {
         int                                      ret           = 0;
+        int32_t                                  port          = 0;
         char                                    *src_brick     = NULL;
         char                                    *dst_brick     = NULL;
         char                                    *volname       = NULL;
@@ -350,6 +351,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                         }
                 }
                 break;
+
         case GF_REPLACE_OP_PAUSE:
                 if (glusterd_is_rb_paused (volinfo)) {
                         gf_log (this->name, GF_LOG_ERROR, "Replace brick is "
@@ -540,6 +542,28 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                         goto out;
                 }
         }
+
+        if (replace_op == GF_REPLACE_OP_START &&
+            !glusterd_is_local_addr (volinfo->rep_brick.dst_brick->hostname)) {
+                port = pmap_registry_alloc (THIS);
+                if (!port) {
+                        gf_log (THIS->name, GF_LOG_CRITICAL,
+                                "No free ports available");
+                        ret = -1;
+                        goto out;
+                }
+
+                ctx = glusterd_op_get_ctx();
+                ret = dict_set_int32 ((ctx)?ctx:rsp_dict, "dst-brick-port",
+                                      port);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Failed to set dst "
+                                "brick port");
+                        goto out;
+                }
+                volinfo->rep_brick.dst_brick->port = port;
+        }
+
         ret = 0;
 
 out:
@@ -708,9 +732,7 @@ rb_spawn_dst_brick (glusterd_volinfo_t *volinfo,
 
         priv = THIS->private;
 
-        port = pmap_registry_alloc (THIS);
-        brickinfo->port = port;
-
+        port = brickinfo->port;
         GF_ASSERT (port);
 
         runinit (&runner);
@@ -726,6 +748,8 @@ rb_spawn_dst_brick (glusterd_volinfo_t *volinfo,
 
         ret = runner_run (&runner);
         if (ret) {
+                pmap_registry_remove (THIS, 0, brickinfo->path,
+                                      GF_PMAP_PORT_BRICKSERVER, NULL);
                 gf_log ("", GF_LOG_DEBUG,
                         "Could not start glusterfs");
                 goto out;
@@ -1002,7 +1026,7 @@ rb_mountpoint_rmdir (glusterd_volinfo_t *volinfo,
         GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
         ret = rmdir (mntpt);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "rmdir failed, due to %s",
+                gf_log ("", GF_LOG_DEBUG, "rmdir failed, reason: %s",
                         strerror (errno));
                 goto out;
         }
@@ -1049,8 +1073,8 @@ rb_destroy_maintenance_client (glusterd_volinfo_t *volinfo,
 
         ret = unlink (volfile);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "unlink of volfile %s "
-                        "failed", volfile);
+                gf_log ("", GF_LOG_DEBUG, "unlink of %s failed, reason: %s",
+                        volfile, strerror (errno));
                 goto out;
         }
 
@@ -1375,12 +1399,9 @@ rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict,
 
         this = THIS;
 
-        ctx = glusterd_op_get_ctx ();
-        if (ctx) {
-                dict_ret = dict_get_int32 (req_dict, "src-brick-port", &src_port);
-                if (src_port)
-                        src_brickinfo->port = src_port;
-        }
+        dict_ret = dict_get_int32 (req_dict, "src-brick-port", &src_port);
+        if (src_port)
+                src_brickinfo->port = src_port;
 
         if (!glusterd_is_local_addr (src_brickinfo->hostname)) {
                 gf_log ("", GF_LOG_INFO,
@@ -1431,13 +1452,10 @@ rb_update_dstbrick_port (glusterd_brickinfo_t *dst_brickinfo, dict_t *rsp_dict,
         int     dict_ret      = 0;
         int     dst_port      = 0;
 
-        ctx = glusterd_op_get_ctx ();
-        if (ctx) {
-                dict_ret = dict_get_int32 (req_dict, "dst-brick-port", &dst_port);
-                if (dst_port)
-                        dst_brickinfo->port = dst_port;
+        dict_ret = dict_get_int32 (req_dict, "dst-brick-port", &dst_port);
+        if (!dict_ret)
+                dst_brickinfo->port = dst_port;
 
-        }
 
         if (!glusterd_is_local_addr (dst_brickinfo->hostname)) {
                 gf_log ("", GF_LOG_INFO,
@@ -1611,10 +1629,6 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
         }
 
 	if ((GF_REPLACE_OP_START != replace_op)) {
-                ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
-                                               dict, replace_op);
-                if (ret)
-                        goto out;
 
                 /* Set task-id, if available, in op_ctx dict for operations
                  * other than start
@@ -1630,6 +1644,10 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                         }
                 }
 	}
+        ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
+                                       dict, replace_op);
+        if (ret)
+                goto out;
 
         switch (replace_op) {
         case GF_REPLACE_OP_START:
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index bb7c0a76c4a..bbca016ebbc 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -886,6 +886,7 @@ int32_t
 glusterd_store_rbstate_write (int fd, glusterd_volinfo_t *volinfo)
 {
         int     ret             = -1;
+        int     port            = 0;
         char    buf[PATH_MAX]   = {0, };
 
         GF_ASSERT (fd > 0);
@@ -915,11 +916,28 @@ glusterd_store_rbstate_write (int fd, glusterd_volinfo_t *volinfo)
                 if (ret)
                         goto out;
 
+                switch (volinfo->transport_type) {
+                        case GF_TRANSPORT_RDMA:
+                                port = volinfo->rep_brick.dst_brick->rdma_port;
+                                break;
+
+                        case GF_TRANSPORT_TCP:
+                        case GF_TRANSPORT_BOTH_TCP_RDMA:
+                                port = volinfo->rep_brick.dst_brick->port;
+                                break;
+                }
+
+                snprintf (buf, sizeof (buf), "%d", port);
+                ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_PORT,
+                                                 buf);
+                if (ret)
+                        goto out;
                 uuid_unparse (volinfo->rep_brick.rb_id, buf);
                 ret = glusterd_store_save_value (fd, GF_REPLACE_BRICK_TID_KEY,
                                                  buf);
         }
 
+        ret = 0;
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -2177,6 +2195,20 @@ glusterd_store_retrieve_rbstate (char   *volname)
                                                 &volinfo->rep_brick.dst_brick);
                                 if (ret)
                                         goto out;
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_DST_PORT,
+                                             strlen (GLUSTERD_STORE_KEY_RB_DST_PORT))) {
+                                switch (volinfo->transport_type) {
+                                case GF_TRANSPORT_RDMA:
+                                volinfo->rep_brick.dst_brick->rdma_port =
+                                                 atoi (value);
+                                        break;
+
+                                case GF_TRANSPORT_TCP:
+                                case GF_TRANSPORT_BOTH_TCP_RDMA:
+                                volinfo->rep_brick.dst_brick->port =
+                                                atoi (value);
+                                        break;
+                                }
                         } else if (!strncmp (key, GF_REPLACE_BRICK_TID_KEY,
                                              strlen (GF_REPLACE_BRICK_TID_KEY))) {
                                         uuid_parse (value,
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index 03ef0005918..68977dd9ce5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -51,6 +51,7 @@ typedef enum glusterd_store_ver_ac_{
 #define GLUSTERD_STORE_KEY_RB_STATUS      "rb_status"
 #define GLUSTERD_STORE_KEY_RB_SRC_BRICK   "rb_src"
 #define GLUSTERD_STORE_KEY_RB_DST_BRICK   "rb_dst"
+#define GLUSTERD_STORE_KEY_RB_DST_PORT    "rb_port"
 #define GLUSTERD_STORE_KEY_VOL_DEFRAG     "rebalance_status"
 #define GLUSTERD_STORE_KEY_DEFRAG_OP      "rebalance_op"
 #define GLUSTERD_STORE_KEY_USERNAME       "username"