glusterd: replace-brick status was not 'shared' with peer.v3.1.7qa1

Change-Id: Iec33deaff338a6b48e5015785711de3c8bc9fb2e
BUG: 3432
Reviewed-on: http://review.gluster.com/258
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

3 files changed, 251 insertions, 64 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 6da48c3d22b..7b085e26542 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -318,6 +318,7 @@ glusterd_store_create_volume (glusterd_volinfo_t *volinfo)
 {
         int32_t                 ret = -1;
         char                    filepath[PATH_MAX] = {0,};
+        char                    rbstatepath[PATH_MAX] = {0,};
         char                    buf[4096] = {0,};
         glusterd_conf_t         *priv = NULL;
         glusterd_brickinfo_t    *brickinfo = NULL;
@@ -337,6 +338,10 @@ glusterd_store_create_volume (glusterd_volinfo_t *volinfo)
                   GLUSTERD_VOLUME_DIR_PREFIX, volinfo->volname,
                   GLUSTERD_VOLUME_INFO_FILE);
 
+        snprintf (rbstatepath, 1024, "%s/%s/%s/%s", priv->workdir,
+                  GLUSTERD_VOLUME_DIR_PREFIX, volinfo->volname,
+                  GLUSTERD_VOLUME_RBSTATE_FILE);
+
         ret = glusterd_store_handle_new (filepath, &volinfo->shandle);
 
         if (ret) {
@@ -345,6 +350,14 @@ glusterd_store_create_volume (glusterd_volinfo_t *volinfo)
                 goto out;
         }
 
+        ret = glusterd_store_handle_new (rbstatepath, &volinfo->rb_shandle);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to create store"
+                        " handle for path: %s", rbstatepath);
+                goto out;
+        }
+
         snprintf (buf, sizeof (buf), "%d", volinfo->type);
         ret = glusterd_store_save_value (volinfo->shandle,
                                         GLUSTERD_STORE_KEY_VOL_TYPE, buf);
@@ -404,6 +417,28 @@ glusterd_store_create_volume (glusterd_volinfo_t *volinfo)
 
         dict_foreach (volinfo->dict, _setopts, volinfo->shandle);
 
+        snprintf (buf, sizeof (buf), "%d", volinfo->rb_status);
+        ret = glusterd_store_save_value (volinfo->rb_shandle, GLUSTERD_STORE_KEY_RB_STATUS,
+                                         buf);
+        if (ret)
+                goto out;
+
+        if (volinfo->rb_status > GF_RB_STATUS_NONE) {
+                snprintf (buf, sizeof (buf), "%s:%s", volinfo->src_brick->hostname,
+                          volinfo->src_brick->path);
+                ret = glusterd_store_save_value (volinfo->rb_shandle, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                                                 buf);
+                if (ret)
+                        goto out;
+
+                snprintf (buf, sizeof (buf), "%s:%s", volinfo->dst_brick->hostname,
+                          volinfo->dst_brick->path);
+                ret = glusterd_store_save_value (volinfo->rb_shandle, GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                                                 buf);
+                if (ret)
+                        goto out;
+        }
+
         ret = 0;
 
 out:
@@ -643,13 +678,16 @@ out:
 }
 
 int
-glusterd_store_handle_retrieve (char *path, glusterd_store_handle_t **handle)
+glusterd_store_handle_retrieve (char *path, glusterd_store_handle_t **handle,
+                                int *op_errno)
 {
         int32_t                 ret = -1;
         struct stat statbuf = {0};
 
         ret = stat (path, &statbuf);
         if (ret) {
+                if (op_errno)
+                        *op_errno = errno;
                 gf_log ("glusterd", GF_LOG_ERROR, "Unable to retrieve store "
                         "handle for %s, error: %s", path, strerror (errno));
                 goto out;
@@ -754,7 +792,8 @@ glusterd_retrieve_uuid ()
         if (!priv->handle) {
                 snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
                           GLUSTERD_INFO_FILE);
-                ret = glusterd_store_handle_retrieve (path, &handle);
+                ret = glusterd_store_handle_retrieve (path, &handle,
+                                                      NULL);
 
                 if (ret) {
                         gf_log ("", GF_LOG_ERROR, "Unable to get store "
@@ -1053,7 +1092,8 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
 
                 tmpvalue = NULL;
 
-                ret = glusterd_store_handle_retrieve (path, &brickinfo->shandle);
+                ret = glusterd_store_handle_retrieve (path, &brickinfo->shandle,
+                                                      NULL);
 
                 if (ret)
                         goto out;
@@ -1131,6 +1171,106 @@ out:
 
 
 int32_t
+glusterd_store_retrieve_rbstate (glusterd_volinfo_t *volinfo)
+{
+        int32_t                   ret                   = -1;
+        glusterd_store_iter_t     *iter                 = NULL;
+        char                      *key                  = NULL;
+        char                      *value                = NULL;
+        char                      buf[PATH_MAX]         = {0,};
+        char                      volpath[PATH_MAX]     = {0,};
+        char                      rbstatepath[PATH_MAX] = {0,};
+        glusterd_conf_t           *priv                 = NULL;
+        char                      path[PATH_MAX]        = {0,};
+        glusterd_store_op_errno_t op_errno              = GD_STORE_SUCCESS;
+        int                       rb_errno              = 0;
+
+        priv = THIS->private;
+
+        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
+        snprintf (rbstatepath, sizeof (path), "%s/%s", volpath,
+                  GLUSTERD_VOLUME_RBSTATE_FILE);
+
+        ret = glusterd_store_handle_retrieve (rbstatepath,
+                                              &volinfo->rb_shandle,
+                                              &rb_errno);
+
+        if (ret ) {
+                /* Backward compatibility */
+                if (rb_errno == ENOENT) {
+                        gf_log ("", GF_LOG_WARNING, "replace-brick persistent "
+                                "state not present.");
+                        ret = glusterd_store_handle_new (rbstatepath,
+                                                         &volinfo->rb_shandle);
+
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR, "Unable to create store"
+                                        " handle for path: %s", rbstatepath);
+                                goto out;
+                        }
+                        snprintf (buf, sizeof (buf), "%d", volinfo->rb_status);
+                        ret = glusterd_store_save_value (volinfo->rb_shandle,
+                                                         GLUSTERD_STORE_KEY_RB_STATUS,
+                                                         buf);
+                } else {
+                        gf_log ("", GF_LOG_CRITICAL, "Unable to retrieve rbstate");
+                }
+
+                goto out;
+        }
+
+        ret = glusterd_store_iter_new (volinfo->rb_shandle, &iter);
+
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_iter_get_next (iter, &key, &value, &op_errno);
+        if (ret)
+                goto out;
+
+        while (!ret) {
+                if (!strncmp (key, GLUSTERD_STORE_KEY_RB_STATUS,
+                              strlen (GLUSTERD_STORE_KEY_RB_STATUS))) {
+                        volinfo->rb_status = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                                     strlen (GLUSTERD_STORE_KEY_RB_SRC_BRICK))
+                           && volinfo->rb_status > GF_RB_STATUS_NONE) {
+                        ret = glusterd_brickinfo_from_brick (value, &volinfo->src_brick);
+                        if (ret)
+                                goto out;
+
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                                     strlen (GLUSTERD_STORE_KEY_RB_DST_BRICK))
+                           && volinfo->rb_status > GF_RB_STATUS_NONE) {
+                        ret = glusterd_brickinfo_from_brick (value, &volinfo->dst_brick);
+                        if (ret)
+                                goto out;
+                }
+
+                GF_FREE (key);
+                GF_FREE (value);
+                key = NULL;
+                value = NULL;
+
+                ret = glusterd_store_iter_get_next (iter, &key, &value,
+                                                    &op_errno);
+        }
+
+        if (op_errno != GD_STORE_EOF)
+                goto out;
+
+        ret = glusterd_store_iter_destroy (iter);
+
+        if (ret)
+                goto out;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+int32_t
 glusterd_store_retrieve_volume (char    *volname)
 {
         int32_t                   ret                   = -1;
@@ -1141,9 +1281,7 @@ glusterd_store_retrieve_volume (char    *volname)
         char                      volpath[PATH_MAX]     = {0,};
         glusterd_conf_t           *priv                 = NULL;
         char                      path[PATH_MAX]        = {0,};
-        char                      dst_brick[PATH_MAX]   = {0, };
         int                       exists                = 0;
-        int                       dst_port              = -1;
         glusterd_store_op_errno_t op_errno              = GD_STORE_SUCCESS;
 
         ret = glusterd_volinfo_new (&volinfo);
@@ -1159,11 +1297,12 @@ glusterd_store_retrieve_volume (char    *volname)
         snprintf (path, sizeof (path), "%s/%s", volpath,
                   GLUSTERD_VOLUME_INFO_FILE);
 
-        ret = glusterd_store_handle_retrieve (path, &volinfo->shandle);
+        ret = glusterd_store_handle_retrieve (path, &volinfo->shandle, NULL);
 
         if (ret)
                 goto out;
 
+
         ret = glusterd_store_iter_new (volinfo->shandle, &iter);
 
         if (ret)
@@ -1205,39 +1344,6 @@ glusterd_store_retrieve_volume (char    *volname)
                         if (ret)
                                 gf_log ("", GF_LOG_WARNING,
                                         "failed to parse uuid");
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_STATUS,
-                                     strlen (GLUSTERD_STORE_KEY_RB_STATUS))) {
-                        glusterd_set_rb_status (volinfo, atoi (value));
-
-                } else if (volinfo->rb_status > GF_RB_STATUS_NONE &&
-                           !strncmp (key, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
-                                     strlen (GLUSTERD_STORE_KEY_RB_SRC_BRICK))) {
-                        ret = glusterd_brickinfo_from_brick (value,
-                                                             &volinfo->src_brick);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Unable to create"
-                                        " src brickinfo");
-                                goto out;
-                        }
-
-                } else if (volinfo->rb_status > GF_RB_STATUS_NONE &&
-                           !strncmp (key, GLUSTERD_STORE_KEY_RB_DST_BRICK,
-                                     strlen (GLUSTERD_STORE_KEY_RB_DST_BRICK))) {
-                        sscanf (value, "%d:%s", &dst_port, dst_brick);
-                        if (dst_port == -1 || dst_brick[0] == 0) {
-                                gf_log ("", GF_LOG_ERROR, "replace brick: "
-                                        "dst brick info is invalid");
-                                goto out;
-                        }
-                        ret = glusterd_brickinfo_from_brick (dst_brick,
-                                                             &volinfo->dst_brick);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Unable to create"
-                                        " dst brickinfo");
-                                goto out;
-                        }
-                        volinfo->dst_brick->port = dst_port;
-
                 } else {
                         exists = glusterd_check_option_exists (key, NULL);
                         if (exists == -1) {
@@ -1277,6 +1383,11 @@ glusterd_store_retrieve_volume (char    *volname)
         if (ret)
                 goto out;
 
+        ret = glusterd_store_retrieve_rbstate (volinfo);
+
+        if (ret)
+                goto out;
+
         ret = glusterd_store_retrieve_bricks (volinfo);
         if (ret)
                 goto out;
@@ -1357,6 +1468,8 @@ glusterd_store_update_volume (glusterd_volinfo_t *volinfo)
 
         ret = glusterd_store_handle_truncate (volinfo->shandle);
 
+        ret = glusterd_store_handle_truncate (volinfo->rb_shandle);
+
         snprintf (buf, sizeof (buf), "%d", volinfo->type);
         ret = glusterd_store_save_value (volinfo->shandle,
                                         GLUSTERD_STORE_KEY_VOL_TYPE, buf);
@@ -1399,8 +1512,18 @@ glusterd_store_update_volume (glusterd_volinfo_t *volinfo)
         if (ret)
                 goto out;
 
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = glusterd_store_create_brick (volinfo, brickinfo,
+                                                   brick_count);
+                if (ret)
+                        goto out;
+                brick_count++;
+        }
+
+        dict_foreach (volinfo->dict, _setopts, volinfo->shandle);
+
         snprintf (buf, sizeof (buf), "%d", volinfo->rb_status);
-        ret = glusterd_store_save_value (volinfo->shandle, GLUSTERD_STORE_KEY_RB_STATUS,
+        ret = glusterd_store_save_value (volinfo->rb_shandle, GLUSTERD_STORE_KEY_RB_STATUS,
                                          buf);
         if (ret)
                 goto out;
@@ -1408,33 +1531,20 @@ glusterd_store_update_volume (glusterd_volinfo_t *volinfo)
         if (volinfo->rb_status > GF_RB_STATUS_NONE) {
                 snprintf (buf, sizeof (buf), "%s:%s", volinfo->src_brick->hostname,
                           volinfo->src_brick->path);
-                ret = glusterd_store_save_value (volinfo->shandle, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                ret = glusterd_store_save_value (volinfo->rb_shandle, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
                                                  buf);
                 if (ret)
                         goto out;
 
-                snprintf (buf, sizeof (buf), "%d:%s:%s", volinfo->dst_brick->port,
-                          volinfo->dst_brick->hostname, volinfo->dst_brick->path);
-                ret = glusterd_store_save_value (volinfo->shandle, GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                snprintf (buf, sizeof (buf), "%s:%s", volinfo->dst_brick->hostname,
+                          volinfo->dst_brick->path);
+                ret = glusterd_store_save_value (volinfo->rb_shandle, GLUSTERD_STORE_KEY_RB_DST_BRICK,
                                                  buf);
                 if (ret)
                         goto out;
         }
-
-
-        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_store_create_brick (volinfo, brickinfo,
-                                                   brick_count);
-                if (ret)
-                        goto out;
-                brick_count++;
-        }
-
-        dict_foreach (volinfo->dict, _setopts, volinfo->shandle);
-
         ret = 0;
 
-
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
 
@@ -1624,7 +1734,8 @@ glusterd_store_retrieve_peers (xlator_t *this)
 
         while (entry) {
                 snprintf (filepath, PATH_MAX, "%s/%s", path, entry->d_name);
-                ret = glusterd_store_handle_retrieve (filepath, &shandle);
+                ret = glusterd_store_handle_retrieve (filepath, &shandle,
+                                                      NULL);
                 if (ret)
                         goto out;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 02e5fb9307a..0c60ca77676 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1210,12 +1210,14 @@ int32_t
 glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                              dict_t  *dict, int32_t count)
 {
-        int32_t                 ret = -1;
-        char                    key[512] = {0,};
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        int32_t                 i = 1;
-        char                    *volume_id_str = NULL;
-        glusterd_volopt_ctx_t   ctx = {0};
+        int32_t                 ret             = -1;
+        char                    key[512]        = {0,};
+        char                    *src_brick      = NULL;
+        char                    *dst_brick      = NULL;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+        int32_t                 i               = 1;
+        char                    *volume_id_str  = NULL;
+        glusterd_volopt_ctx_t   ctx             = {0};
 
         GF_ASSERT (dict);
         GF_ASSERT (volinfo);
@@ -1271,6 +1273,35 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
+        memset (key, 0, sizeof (key));
+        snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
+        ret = dict_set_int32 (dict, key, volinfo->rb_status);
+        if (ret)
+                goto out;
+
+        if (volinfo->rb_status > GF_RB_STATUS_NONE) {
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                          count);
+                gf_asprintf (&src_brick, "%s:%s",
+                             volinfo->src_brick->hostname,
+                             volinfo->src_brick->path);
+                ret = dict_set_dynstr (dict, key, src_brick);
+                if (ret)
+                        goto out;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                          count);
+                gf_asprintf (&dst_brick, "%s:%s",
+                             volinfo->dst_brick->hostname,
+                             volinfo->dst_brick->path);
+                ret = dict_set_dynstr (dict, key, dst_brick);
+                if (ret)
+                        goto out;
+        }
+
         ctx.dict = dict;
         ctx.count = count;
         ctx.opt_count = 1;
@@ -1486,7 +1517,10 @@ glusterd_import_friend_volume (dict_t *vols, int count)
         glusterd_brickinfo_t    *tmp = NULL;
         int                     new_volinfo = 0;
         int                     i = 1;
+        int                     rb_status = 0;
         char                    *volume_id_str = NULL;
+        char                    *src_brick = NULL;
+        char                    *dst_brick = NULL;
 
         GF_ASSERT (vols);
 
@@ -1551,6 +1585,46 @@ glusterd_import_friend_volume (dict_t *vols, int count)
                 goto out;
         uuid_parse (volume_id_str, volinfo->volume_id);
 
+        memset (key, 0, sizeof (key));
+        snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
+        ret = dict_get_int32 (vols, key, &rb_status);
+        if (ret)
+                goto out;
+        volinfo->rb_status = rb_status;
+
+        if (volinfo->rb_status > GF_RB_STATUS_NONE) {
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                          count);
+                ret = dict_get_str (vols, key, &src_brick);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_brickinfo_from_brick (src_brick,
+                                                     &volinfo->src_brick);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to create"
+                                " src brickinfo");
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                          count);
+                ret = dict_get_str (vols, key, &dst_brick);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_brickinfo_from_brick (dst_brick,
+                                                     &volinfo->dst_brick);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to create"
+                                " dst brickinfo");
+                        goto out;
+                }
+        }
+
         list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
                                    brick_list) {
                 glusterd_delete_volfile (volinfo, brickinfo);
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 009c5567cf9..32849a2bdc0 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -183,6 +183,7 @@ struct glusterd_volinfo_ {
         int                     sub_count;
         int                     port;
         glusterd_store_handle_t *shandle;
+        glusterd_store_handle_t *rb_shandle;
 
         /* Defrag/rebalance related */
         gf_defrag_status_t      defrag_status;
@@ -226,6 +227,7 @@ enum glusterd_vol_comp_status_ {
 #define GLUSTERD_VOLUME_DIR_PREFIX "vols"
 #define GLUSTERD_PEER_DIR_PREFIX "peers"
 #define GLUSTERD_VOLUME_INFO_FILE "info"
+#define GLUSTERD_VOLUME_RBSTATE_FILE "rbstate"
 #define GLUSTERD_BRICK_INFO_DIR "bricks"
 #define GLUSTERD_CKSUM_FILE "cksum"