From f502e28e8b416f80bd9506ac204948681610b305 Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Tue, 10 Dec 2013 11:34:06 +0530 Subject: glusterd: Save/restore/sync rebalance dict A dictionary was added to store additional information of a rebalance process, like the bricks being removed in case of a rebalance started by remove-brick. This dictionary wasn't being stored/restored or synced during volume sync, leading to errors like a volume status command failing. These issues have been fixed in this patch. The rebalance dict is now stored/restored and also exported/imported during volume sync. Also, this makes sure that the rebalance dict is only create on remove-brick start. This adds a bricks decommissioned status to the information imported/exported during volume sync. Change-Id: I56fed23dc2de80a96648055fe705e9c3ffd55227 BUG: 1040809 Signed-off-by: Kaushal M Reviewed-on: http://review.gluster.org/6492 Tested-by: Gluster Build System Reviewed-by: Krishnan Parthasarathi Reviewed-by: Vijay Bellur --- xlators/mgmt/glusterd/src/glusterd-brick-ops.c | 60 ++++++------ xlators/mgmt/glusterd/src/glusterd-store.c | 100 +++++++++++++++----- xlators/mgmt/glusterd/src/glusterd-utils.c | 125 ++++++++++++++++++++----- 3 files changed, 208 insertions(+), 77 deletions(-) (limited to 'xlators/mgmt') diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c index d832cdf24..26d608a2f 100644 --- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c +++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c @@ -1842,19 +1842,22 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr) goto out; } - /* Save the list of bricks for later usage. Right now this is required - * for displaying the task parameters with task status in volume status. + /* Save the list of bricks for later usage only on starting a + * remove-brick. Right now this is required for displaying the task + * parameters with task status in volume status. */ - bricks_dict = dict_new (); - if (!bricks_dict) { - ret = -1; - goto out; - } - ret = dict_set_int32 (bricks_dict, "count", count); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, - "Failed to save remove-brick count"); - goto out; + if (GF_OP_CMD_START == cmd) { + bricks_dict = dict_new (); + if (!bricks_dict) { + ret = -1; + goto out; + } + ret = dict_set_int32 (bricks_dict, "count", count); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, + "Failed to save remove-brick count"); + goto out; + } } while ( i <= count) { snprintf (key, 256, "brick%d", i); @@ -1865,20 +1868,22 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr) goto out; } - brick_tmpstr = gf_strdup (brick); - if (!brick_tmpstr) { - ret = -1; - gf_log (this->name, GF_LOG_ERROR, - "Failed to duplicate brick name"); - goto out; - } - ret = dict_set_dynstr (bricks_dict, key, brick_tmpstr); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, - "Failed to add brick to dict"); - goto out; + if (GF_OP_CMD_START == cmd) { + brick_tmpstr = gf_strdup (brick); + if (!brick_tmpstr) { + ret = -1; + gf_log (this->name, GF_LOG_ERROR, + "Failed to duplicate brick name"); + goto out; + } + ret = dict_set_dynstr (bricks_dict, key, brick_tmpstr); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, + "Failed to add brick to dict"); + goto out; + } + brick_tmpstr = NULL; } - brick_tmpstr = NULL; ret = glusterd_op_perform_remove_brick (volinfo, brick, force, &need_rebalance); @@ -1886,6 +1891,9 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr) goto out; i++; } + if (GF_OP_CMD_START == cmd) + volinfo->rebal.dict = dict_ref (bricks_dict); + ret = dict_get_int32 (dict, "replica-count", &replica_count); if (!ret) { gf_log (this->name, GF_LOG_INFO, @@ -1910,8 +1918,6 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr) } } } - volinfo->rebal.dict = bricks_dict; - bricks_dict = NULL; ret = glusterd_create_volfiles_and_notify_services (volinfo); if (ret) { diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c index 5b51aabad..2ba2548d2 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.c +++ b/xlators/mgmt/glusterd/src/glusterd-store.c @@ -883,6 +883,19 @@ out: return ret; } +int +_gd_store_rebalance_dict (dict_t *dict, char *key, data_t *value, void *data) +{ + int ret = -1; + int fd = 0; + + fd = *(int *)data; + + ret = gf_store_save_value (fd, key, value->data); + + return ret; +} + int32_t glusterd_store_node_state_write (int fd, glusterd_volinfo_t *volinfo) { @@ -907,9 +920,14 @@ glusterd_store_node_state_write (int fd, glusterd_volinfo_t *volinfo) if (ret) goto out; - if (volinfo->rebal.defrag_cmd) { - uuid_unparse (volinfo->rebal.rebalance_id, buf); - ret = gf_store_save_value (fd, GF_REBALANCE_TID_KEY, buf); + uuid_unparse (volinfo->rebal.rebalance_id, buf); + ret = gf_store_save_value (fd, GF_REBALANCE_TID_KEY, buf); + if (ret) + goto out; + + if (volinfo->rebal.dict) { + dict_foreach (volinfo->rebal.dict, _gd_store_rebalance_dict, + &fd); } out: gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret); @@ -1726,17 +1744,22 @@ out: int32_t glusterd_store_retrieve_node_state (char *volname) { - int32_t ret = -1; - glusterd_volinfo_t *volinfo = NULL; - gf_store_iter_t *iter = NULL; - char *key = NULL; - char *value = NULL; - char volpath[PATH_MAX] = {0,}; - glusterd_conf_t *priv = NULL; - char path[PATH_MAX] = {0,}; - gf_store_op_errno_t op_errno = GD_STORE_SUCCESS; + int32_t ret = -1; + glusterd_volinfo_t *volinfo = NULL; + gf_store_iter_t *iter = NULL; + char *key = NULL; + char *value = NULL; + char *dup_value = NULL; + char volpath[PATH_MAX] = {0,}; + glusterd_conf_t *priv = NULL; + char path[PATH_MAX] = {0,}; + gf_store_op_errno_t op_errno = GD_STORE_SUCCESS; + dict_t *tmp_dict = NULL; + xlator_t *this = NULL; - priv = THIS->private; + this = THIS; + GF_ASSERT (this); + priv = this->private; ret = glusterd_volinfo_find (volname, &volinfo); if (ret) { @@ -1766,16 +1789,35 @@ glusterd_store_retrieve_node_state (char *volname) if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DEFRAG, strlen (GLUSTERD_STORE_KEY_VOL_DEFRAG))) { volinfo->rebal.defrag_cmd = atoi (value); - } - - if (volinfo->rebal.defrag_cmd) { - if (!strncmp (key, GF_REBALANCE_TID_KEY, - strlen (GF_REBALANCE_TID_KEY))) - uuid_parse (value, volinfo->rebal.rebalance_id); - - if (!strncmp (key, GLUSTERD_STORE_KEY_DEFRAG_OP, - strlen (GLUSTERD_STORE_KEY_DEFRAG_OP))) - volinfo->rebal.op = atoi (value); + } else if (!strncmp (key, GF_REBALANCE_TID_KEY, + strlen (GF_REBALANCE_TID_KEY))) { + uuid_parse (value, volinfo->rebal.rebalance_id); + } else if (!strncmp (key, GLUSTERD_STORE_KEY_DEFRAG_OP, + strlen (GLUSTERD_STORE_KEY_DEFRAG_OP))) { + volinfo->rebal.op = atoi (value); + } else { + if (!tmp_dict) { + tmp_dict = dict_new (); + if (!tmp_dict) { + ret = -1; + goto out; + } + } + dup_value = gf_strdup (value); + if (!dup_value) { + ret = -1; + gf_log (this->name, GF_LOG_ERROR, + "Failed to strdup value string"); + goto out; + } + ret = dict_set_str (tmp_dict, key, dup_value); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, + "Error setting data in rebal " + "dict."); + goto out; + } + dup_value = NULL; } GF_FREE (key); @@ -1785,9 +1827,13 @@ glusterd_store_retrieve_node_state (char *volname) ret = gf_store_iter_get_next (iter, &key, &value, &op_errno); } + if (tmp_dict) + volinfo->rebal.dict = dict_ref (tmp_dict); - if (op_errno != GD_STORE_EOF) + if (op_errno != GD_STORE_EOF) { + ret = -1; goto out; + } ret = gf_store_iter_destroy (iter); @@ -1795,6 +1841,12 @@ glusterd_store_retrieve_node_state (char *volname) goto out; out: + if (dup_value) + GF_FREE (dup_value); + if (ret && volinfo->rebal.dict) + dict_unref (volinfo->rebal.dict); + if (tmp_dict) + dict_unref (tmp_dict); gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret); return ret; diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index 8cf9f790f..f398e49dc 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -1956,20 +1956,18 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo, if (ret) goto out; - if (volinfo->rebal.defrag_cmd) { - rebalance_id_str = gf_strdup (uuid_utoa - (volinfo->rebal.rebalance_id)); - if (!rebalance_id_str) { - ret = -1; - goto out; - } - memset (key, 0, sizeof (key)); - snprintf (key, 256, "volume%d.rebalance-id", count); - ret = dict_set_dynstr (dict, key, rebalance_id_str); - if (ret) - goto out; - rebalance_id_str = NULL; + rebalance_id_str = gf_strdup (uuid_utoa + (volinfo->rebal.rebalance_id)); + if (!rebalance_id_str) { + ret = -1; + goto out; } + memset (key, 0, sizeof (key)); + snprintf (key, 256, "volume%d.rebalance-id", count); + ret = dict_set_dynstr (dict, key, rebalance_id_str); + if (ret) + goto out; + rebalance_id_str = NULL; memset (key, 0, sizeof (key)); snprintf (key, sizeof (key), "volume%d.rebalance-op", count); @@ -1977,6 +1975,23 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo, if (ret) goto out; + if (volinfo->rebal.dict) { + snprintf (prefix, sizeof (prefix), "volume%d", count); + ctx.dict = dict; + ctx.prefix = prefix; + ctx.opt_count = 1; + ctx.key_name = "rebal-dict-key"; + ctx.val_name = "rebal-dict-value"; + + dict_foreach (volinfo->rebal.dict, _add_dict_to_prdict, &ctx); + ctx.opt_count--; + memset (key, 0, sizeof (key)); + snprintf (key, sizeof (key), "volume%d.rebal-dict-count", count); + ret = dict_set_int32 (dict, key, ctx.opt_count); + if (ret) + goto out; + } + memset (key, 0, sizeof (key)); snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count); ret = dict_set_int32 (dict, key, volinfo->rep_brick.rb_status); @@ -2066,6 +2081,13 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo, if (ret) goto out; + memset (key, 0, sizeof (key)); + snprintf (key, sizeof (key), "volume%d.brick%d.decommissioned", + count, i); + ret = dict_set_int32 (dict, key, brickinfo->decommissioned); + if (ret) + goto out; + i++; } @@ -2733,6 +2755,7 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count, int ret = -1; char *hostname = NULL; char *path = NULL; + int decommissioned = 0; glusterd_brickinfo_t *new_brickinfo = NULL; char msg[2048] = {0}; @@ -2758,12 +2781,22 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count, goto out; } + memset (key, 0, sizeof (key)); + snprintf (key, sizeof (key), "volume%d.brick%d.decommissioned", + vol_count, brick_count); + ret = dict_get_int32 (vols, key, &decommissioned); + if (ret) { + /* For backward compatibility */ + ret = 0; + } + ret = glusterd_brickinfo_new (&new_brickinfo); if (ret) goto out; strcpy (new_brickinfo->path, path); strcpy (new_brickinfo->hostname, hostname); + new_brickinfo->decommissioned = decommissioned; //peerinfo might not be added yet (void) glusterd_resolve_brick (new_brickinfo); ret = 0; @@ -2908,6 +2941,43 @@ out: return ret; } +int +gd_import_friend_volume_rebal_dict (dict_t *dict, int count, + glusterd_volinfo_t *volinfo) +{ + int ret = -1; + char key[256] = {0,}; + int dict_count = 0; + char prefix[64] = {0}; + + GF_ASSERT (dict); + GF_ASSERT (volinfo); + + memset (key, 0, sizeof (key)); + snprintf (key, sizeof (key), "volume%d.rebal-dict-count", count); + ret = dict_get_int32 (dict, key, &dict_count); + if (ret) { + /* Older peers will not have this dict */ + ret = 0; + goto out; + } + + volinfo->rebal.dict = dict_new (); + if(!volinfo->rebal.dict) { + ret = -1; + goto out; + } + + snprintf (prefix, sizeof (prefix), "volume%d", count); + ret = import_prdict_dict (dict, volinfo->rebal.dict, "rebal-dict-key", + "rebal-dict-value", dict_count, prefix); +out: + if (ret && volinfo->rebal.dict) + dict_unref (volinfo->rebal.dict); + gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret); + return ret; +} + int32_t glusterd_import_volinfo (dict_t *vols, int count, glusterd_volinfo_t **volinfo) @@ -3071,19 +3141,16 @@ glusterd_import_volinfo (dict_t *vols, int count, goto out; } - if (new_volinfo->rebal.defrag_cmd) { - memset (key, 0, sizeof (key)); - snprintf (key, sizeof (key), "volume%d.rebalance-id", count); - ret = dict_get_str (vols, key, &rebalance_id_str); - if (ret) { - /* This is not present in older glusterfs versions, - * so don't error out - */ - ret = 0; - } else { - uuid_parse (rebalance_id_str, - new_volinfo->rebal.rebalance_id); - } + memset (key, 0, sizeof (key)); + snprintf (key, sizeof (key), "volume%d.rebalance-id", count); + ret = dict_get_str (vols, key, &rebalance_id_str); + if (ret) { + /* This is not present in older glusterfs versions, + * so don't error out + */ + ret = 0; + } else { + uuid_parse (rebalance_id_str, new_volinfo->rebal.rebalance_id); } memset (key, 0, sizeof (key)); @@ -3095,6 +3162,12 @@ glusterd_import_volinfo (dict_t *vols, int count, */ ret = 0; } + ret = gd_import_friend_volume_rebal_dict (vols, count, new_volinfo); + if (ret) { + snprintf (msg, sizeof (msg), "Failed to import rebalance dict " + "for volume."); + goto out; + } memset (key, 0, sizeof (key)); snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count); -- cgit