glusterd: replace-brick status was not 'shared' with peer.v3.3.0qa5

Change-Id: Ia2d89fd919b077232a37debc2aebe1bc72150856
BUG: 3432
Reviewed-on: http://review.gluster.com/285
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

3 files changed, 297 insertions, 76 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 8660b05a11b..f0164849e52 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -567,8 +567,8 @@ glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
         GF_ASSERT (fd > 0);
         GF_ASSERT (volinfo);
 
-        char                    buf[PATH_MAX+20] = {0,};
-        int32_t                 ret              = -1;
+        char                    buf[PATH_MAX] = {0,};
+        int32_t                 ret           = -1;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->type);
         ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TYPE, buf);
@@ -614,28 +614,6 @@ glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
         if (ret)
                 goto out;
 
-        snprintf (buf, sizeof (buf), "%d", volinfo->rb_status);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_STATUS,
-                                         buf);
-        if (ret)
-                goto out;
-
-        if (volinfo->rb_status > GF_RB_STATUS_NONE) {
-                snprintf (buf, sizeof (buf), "%s:%s", volinfo->src_brick->hostname,
-                          volinfo->src_brick->path);
-                ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
-                                                 buf);
-                if (ret)
-                        goto out;
-
-                snprintf (buf, sizeof (buf), "%d:%s:%s", volinfo->dst_brick->port,
-                          volinfo->dst_brick->hostname, volinfo->dst_brick->path);
-                ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_BRICK,
-                                                 buf);
-                if (ret)
-                        goto out;
-        }
-
 out:
         if (ret)
                 gf_log ("", GF_LOG_ERROR, "Unable to write volume values"
@@ -697,6 +675,21 @@ out:
 }
 
 static void
+glusterd_store_rbstatepath_set (glusterd_volinfo_t *volinfo, char *rbstatepath,
+                                size_t len)
+{
+        char    voldirpath[PATH_MAX] = {0,};
+        GF_ASSERT (volinfo);
+        GF_ASSERT (rbstatepath);
+        GF_ASSERT (len >= PATH_MAX);
+
+        glusterd_store_voldirpath_set (volinfo, voldirpath,
+                                       sizeof (voldirpath));
+        snprintf (rbstatepath, len, "%s/%s", voldirpath,
+                  GLUSTERD_VOLUME_RBSTATE_FILE);
+}
+
+static void
 glusterd_store_volfpath_set (glusterd_volinfo_t *volinfo, char *volfpath,
                              size_t len)
 {
@@ -711,6 +704,20 @@ glusterd_store_volfpath_set (glusterd_volinfo_t *volinfo, char *volfpath,
 }
 
 int32_t
+glusterd_store_create_rbstate_shandle_on_absence (glusterd_volinfo_t *volinfo)
+{
+        char            rbstatepath[PATH_MAX] = {0};
+        int32_t         ret                   = 0;
+
+        GF_ASSERT (volinfo);
+
+        glusterd_store_rbstatepath_set (volinfo, rbstatepath, sizeof (rbstatepath));
+        ret = glusterd_store_handle_create_on_absence (&volinfo->rb_shandle,
+                                                       rbstatepath);
+        return ret;
+}
+
+int32_t
 glusterd_store_create_vol_shandle_on_absence (glusterd_volinfo_t *volinfo)
 {
         char            volfpath[PATH_MAX] = {0};
@@ -746,6 +753,73 @@ out:
 }
 
 int32_t
+glusterd_store_rbstate_write (int fd, glusterd_volinfo_t *volinfo)
+{
+        int     ret             = -1;
+        char    buf[PATH_MAX]   = {0, };
+
+        GF_ASSERT (fd > 0);
+        GF_ASSERT (volinfo);
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->rb_status);
+        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_STATUS,
+                                         buf);
+        if (ret)
+                goto out;
+
+        if (volinfo->rb_status > GF_RB_STATUS_NONE) {
+
+                snprintf (buf, sizeof (buf), "%s:%s",
+                          volinfo->src_brick->hostname,
+                          volinfo->src_brick->path);
+                ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                                                 buf);
+                if (ret)
+                        goto out;
+
+                snprintf (buf, sizeof (buf), "%s:%s",
+                          volinfo->dst_brick->hostname,
+                          volinfo->dst_brick->path);
+                ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                                                 buf);
+                if (ret)
+                        goto out;
+        }
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_perform_rbstate_store (glusterd_volinfo_t *volinfo)
+{
+        int                         fd = -1;
+        int32_t                     ret = -1;
+        GF_ASSERT (volinfo);
+
+        fd = glusterd_store_mkstemp (volinfo->rb_shandle);
+        if (fd <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_rbstate_write (fd, volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_rename_tmppath (volinfo->rb_shandle);
+
+out:
+        if (ret && (fd > 0))
+                glusterd_store_unlink_tmppath (volinfo->rb_shandle);
+        if (fd > 0)
+                close (fd);
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
 glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo)
 {
         int                         fd = -1;
@@ -807,9 +881,18 @@ glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t a
         if (ret)
                 goto out;
 
+        ret = glusterd_store_create_rbstate_shandle_on_absence (volinfo);
+        if (ret)
+                goto out;
+
         ret = glusterd_store_perform_volume_store (volinfo);
         if (ret)
                 goto out;
+
+        ret = glusterd_store_perform_rbstate_store (volinfo);
+        if (ret)
+                goto out;
+
         //checksum should be computed at the end
         ret = glusterd_volume_compute_cksum (volinfo);
         if (ret)
@@ -1592,6 +1675,91 @@ out:
 
 
 int32_t
+glusterd_store_retrieve_rbstate (char   *volname)
+{
+        int32_t                   ret                   = -1;
+        glusterd_volinfo_t        *volinfo              = NULL;
+        glusterd_store_iter_t     *iter                 = NULL;
+        char                      *key                  = NULL;
+        char                      *value                = NULL;
+        char                      volpath[PATH_MAX]     = {0,};
+        glusterd_conf_t           *priv                 = NULL;
+        char                      path[PATH_MAX]        = {0,};
+        glusterd_store_op_errno_t op_errno              = GD_STORE_SUCCESS;
+
+        priv = THIS->private;
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get"
+                        "volinfo for %s.", volname);
+                goto out;
+        }
+
+        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
+        snprintf (path, sizeof (path), "%s/%s", volpath,
+                  GLUSTERD_VOLUME_RBSTATE_FILE);
+
+        ret = glusterd_store_handle_retrieve (path, &volinfo->rb_shandle);
+
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_iter_new (volinfo->rb_shandle, &iter);
+
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_iter_get_next (iter, &key, &value, &op_errno);
+        if (ret)
+                goto out;
+
+        while (!ret) {
+                if (!strncmp (key, GLUSTERD_STORE_KEY_RB_STATUS,
+                              strlen (GLUSTERD_STORE_KEY_RB_STATUS))) {
+                        volinfo->rb_status = atoi (value);
+                }
+
+                if (volinfo->rb_status > GF_RB_STATUS_NONE) {
+                        if (!strncmp (key, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                                      strlen (GLUSTERD_STORE_KEY_RB_SRC_BRICK))) {
+                                ret = glusterd_brickinfo_from_brick (value,
+                                                                     &volinfo->src_brick);
+                                if (ret)
+                                        goto out;
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                                             strlen (GLUSTERD_STORE_KEY_RB_DST_BRICK))) {
+                                ret = glusterd_brickinfo_from_brick (value,
+                                                                     &volinfo->dst_brick);
+                                if (ret)
+                                        goto out;
+                        }
+                }
+
+                GF_FREE (key);
+                GF_FREE (value);
+                key = NULL;
+                value = NULL;
+
+                ret = glusterd_store_iter_get_next (iter, &key, &value,
+                                                    &op_errno);
+        }
+
+        if (op_errno != GD_STORE_EOF)
+                goto out;
+
+        ret = glusterd_store_iter_destroy (iter);
+
+        if (ret)
+                goto out;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+int32_t
 glusterd_store_retrieve_volume (char    *volname)
 {
         int32_t                   ret                   = -1;
@@ -1602,9 +1770,7 @@ glusterd_store_retrieve_volume (char    *volname)
         char                      volpath[PATH_MAX]     = {0,};
         glusterd_conf_t           *priv                 = NULL;
         char                      path[PATH_MAX]        = {0,};
-        char                      dst_brick[PATH_MAX]   = {0, };
         int                       exists                = 0;
-        int                       dst_port              = -1;
         glusterd_store_op_errno_t op_errno              = GD_STORE_SUCCESS;
 
         ret = glusterd_volinfo_new (&volinfo);
@@ -1670,39 +1836,6 @@ glusterd_store_retrieve_volume (char    *volname)
                                 gf_log ("", GF_LOG_WARNING,
                                         "failed to parse uuid");
 
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_STATUS,
-                                     strlen (GLUSTERD_STORE_KEY_RB_STATUS))) {
-                        glusterd_set_rb_status (volinfo, atoi (value));
-
-                } else if (volinfo->rb_status > GF_RB_STATUS_NONE &&
-                           !strncmp (key, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
-                                     strlen (GLUSTERD_STORE_KEY_RB_SRC_BRICK))) {
-                        ret = glusterd_brickinfo_from_brick (value,
-                                                             &volinfo->src_brick);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Unable to create"
-                                        " src brickinfo");
-                                goto out;
-                        }
-
-                } else if (volinfo->rb_status > GF_RB_STATUS_NONE &&
-                           !strncmp (key, GLUSTERD_STORE_KEY_RB_DST_BRICK,
-                                     strlen (GLUSTERD_STORE_KEY_RB_DST_BRICK))) {
-                        sscanf (value, "%d:%s", &dst_port, dst_brick);
-                        if (dst_port == -1 || dst_brick[0] == 0) {
-                                gf_log ("", GF_LOG_ERROR, "replace brick: "
-                                        "dst brick info is invalid");
-                                goto out;
-                        }
-                        ret = glusterd_brickinfo_from_brick (dst_brick,
-                                                             &volinfo->dst_brick);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Unable to create"
-                                        " dst brickinfo");
-                                goto out;
-                        }
-                        volinfo->dst_brick->port = dst_port;
-
                 } else if (strstr (key, "slave")) {
                         ret = dict_set_dynstr (volinfo->gsync_slaves, key,
                                                 gf_strdup (value));
@@ -1777,11 +1910,12 @@ out:
 int32_t
 glusterd_store_retrieve_volumes (xlator_t  *this)
 {
-        int32_t         ret = 0;
-        char            path[PATH_MAX] = {0,};
-        glusterd_conf_t *priv = NULL;
-        DIR             *dir = NULL;
-        struct dirent   *entry = NULL;
+        int32_t                ret              = 0;
+        char                   path[PATH_MAX]   = {0,};
+        glusterd_conf_t       *priv             = NULL;
+        DIR                   *dir              = NULL;
+        struct dirent         *entry            = NULL;
+        glusterd_volinfo_t    *volinfo          = NULL;
 
         GF_ASSERT (this);
         priv = this->private;
@@ -1808,6 +1942,16 @@ glusterd_store_retrieve_volumes (xlator_t  *this)
                                 "volume: %s", entry->d_name);
                         goto out;
                 }
+
+                ret = glusterd_store_retrieve_rbstate (entry->d_name);
+                if (ret) {
+                        /* Backward compatibility */
+                        gf_log ("", GF_LOG_INFO, "Creating a new rbstate "
+                                "for volume: %s.", entry->d_name);
+                        ret = glusterd_volinfo_find (entry->d_name, &volinfo);
+                        ret = glusterd_store_create_rbstate_shandle_on_absence (volinfo);
+                        ret = glusterd_store_perform_rbstate_store (volinfo);
+                }
                 glusterd_for_each_entry (entry, dir);
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 278b25c954a..bb7d82ac6ec 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1376,12 +1376,14 @@ int32_t
 glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                              dict_t  *dict, int32_t count)
 {
-        int32_t                 ret = -1;
-        char                    key[512] = {0,};
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        int32_t                 i = 1;
-        char                    *volume_id_str = NULL;
-        glusterd_voldict_ctx_t   ctx = {0};
+        int32_t                 ret             = -1;
+        char                    key[512]        = {0,};
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+        int32_t                 i               = 1;
+        char                    *volume_id_str  = NULL;
+        char                    *src_brick      = NULL;
+        char                    *dst_brick      = NULL;
+        glusterd_voldict_ctx_t   ctx            = {0};
 
         GF_ASSERT (dict);
         GF_ASSERT (volinfo);
@@ -1443,6 +1445,35 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
+        memset (key, 0, sizeof (key));
+        snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
+        ret = dict_set_int32 (dict, key, volinfo->rb_status);
+        if (ret)
+                goto out;
+
+        if (volinfo->rb_status > GF_RB_STATUS_NONE) {
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                          count);
+                gf_asprintf (&src_brick, "%s:%s",
+                             volinfo->src_brick->hostname,
+                             volinfo->src_brick->path);
+                ret = dict_set_dynstr (dict, key, src_brick);
+                if (ret)
+                        goto out;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                          count);
+                gf_asprintf (&dst_brick, "%s:%s",
+                             volinfo->dst_brick->hostname,
+                             volinfo->dst_brick->path);
+                ret = dict_set_dynstr (dict, key, dst_brick);
+                if (ret)
+                        goto out;
+        }
+
         ctx.dict = dict;
         ctx.count = count;
         ctx.opt_count = 1;
@@ -1793,12 +1824,15 @@ int32_t
 glusterd_import_volinfo (dict_t *vols, int count,
                          glusterd_volinfo_t **volinfo)
 {
-        int                ret = -1;
-        char               key[256] = {0};
-        char               *volname = NULL;
-        glusterd_volinfo_t *new_volinfo = NULL;
-        char               *volume_id_str = NULL;
-        char                    msg[2048] = {0};
+        int                ret               = -1;
+        char               key[256]          = {0};
+        char               *volname          = NULL;
+        glusterd_volinfo_t *new_volinfo      = NULL;
+        char               *volume_id_str    = NULL;
+        char               msg[2048]         = {0};
+        char               *src_brick        = NULL;
+        char               *dst_brick        = NULL;
+        int                rb_status         = 0;
 
         GF_ASSERT (vols);
         GF_ASSERT (volinfo);
@@ -1890,6 +1924,47 @@ glusterd_import_volinfo (dict_t *vols, int count,
 
         uuid_parse (volume_id_str, new_volinfo->volume_id);
 
+        memset (key, 0, sizeof (key));
+        snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
+        ret = dict_get_int32 (vols, key, &rb_status);
+        if (ret)
+                goto out;
+        new_volinfo->rb_status = rb_status;
+
+        if (new_volinfo->rb_status > GF_RB_STATUS_NONE) {
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                          count);
+                ret = dict_get_str (vols, key, &src_brick);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_brickinfo_from_brick (src_brick,
+                                                     &new_volinfo->src_brick);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to create"
+                                " src brickinfo");
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                          count);
+                ret = dict_get_str (vols, key, &dst_brick);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_brickinfo_from_brick (dst_brick,
+                                                     &new_volinfo->dst_brick);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to create"
+                                " dst brickinfo");
+                        goto out;
+                }
+        }
+
+
         ret = glusterd_import_friend_volume_opts (vols, count, new_volinfo);
         if (ret)
                 goto out;
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 4362984fef4..e16d6209b0e 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -192,6 +192,7 @@ struct glusterd_volinfo_ {
         int                     replica_count;
         int                     port;
         glusterd_store_handle_t *shandle;
+        glusterd_store_handle_t *rb_shandle;
 
         /* Defrag/rebalance related */
         gf_defrag_status_t      defrag_status;
@@ -242,6 +243,7 @@ enum glusterd_vol_comp_status_ {
 #define GLUSTERD_VOLUME_DIR_PREFIX "vols"
 #define GLUSTERD_PEER_DIR_PREFIX "peers"
 #define GLUSTERD_VOLUME_INFO_FILE "info"
+#define GLUSTERD_VOLUME_RBSTATE_FILE "rbstate"
 #define GLUSTERD_BRICK_INFO_DIR "bricks"
 #define GLUSTERD_CKSUM_FILE "cksum"