From 17454dfea9f3c4d47fcf0b5370a6155f639c8aeb Mon Sep 17 00:00:00 2001 From: Ravishankar N Date: Tue, 25 Feb 2014 09:17:18 +0530 Subject: glusterd: persistent client xlator/ afr changelog names -Add a unique brick-id field to glusterd_brickinfo_t -Persist the id to the brickinfo file -Use the brick-id as the client xlator name during vol create, add-brick and replace-brick operations. -For older volumes,generate the id in-memory during glusterd restore but defer writing it to the brickinfo file until the next volume set operation. -send and receive the brick-ids during peer probe. Feature page: www.gluster.org/community/documentation/index.php/Features/persistent-AFR-changelog-xattributes Related patch: http://review.gluster.org/#/c/7122 Change-Id: Ib7f1570004e33f4144476410eec2b84df4e41448 BUG: 1066778 Signed-off-by: Ravishankar N Reviewed-on: http://review.gluster.org/7155 Tested-by: Gluster Build System Reviewed-by: Pranith Kumar Karampuri Reviewed-by: Kaushal M Reviewed-by: Vijay Bellur --- xlators/mgmt/glusterd/src/glusterd-brick-ops.c | 7 ++++ xlators/mgmt/glusterd/src/glusterd-replace-brick.c | 3 ++ xlators/mgmt/glusterd/src/glusterd-store.c | 21 +++++++++- xlators/mgmt/glusterd/src/glusterd-store.h | 1 + xlators/mgmt/glusterd/src/glusterd-utils.c | 46 ++++++++++++++++++++++ xlators/mgmt/glusterd/src/glusterd-utils.h | 7 ++++ xlators/mgmt/glusterd/src/glusterd-volgen.c | 2 +- xlators/mgmt/glusterd/src/glusterd-volume-ops.c | 7 ++++ xlators/mgmt/glusterd/src/glusterd.h | 1 + 9 files changed, 92 insertions(+), 3 deletions(-) (limited to 'xlators') diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c index 5ab23f2d9..f15ec7b18 100644 --- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c +++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c @@ -995,6 +995,7 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count, gf_boolean_t restart_needed = 0; char msg[1024] __attribute__((unused)) = {0, }; int caps = 0; + int brickid = 0; GF_ASSERT (volinfo); @@ -1022,11 +1023,17 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count, "type is set %d, need to change it", type); } + brickid = glusterd_get_next_available_brickid (volinfo); + if (brickid < 0) + goto out; while ( i <= count) { ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo); if (ret) goto out; + GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (brickinfo, volinfo, + brickid++); + ret = glusterd_resolve_brick (brickinfo); if (ret) goto out; diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c index 9685cb374..e78eff44d 100644 --- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c +++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c @@ -1516,6 +1516,9 @@ glusterd_op_perform_replace_brick (glusterd_volinfo_t *volinfo, if (ret) goto out; + strncpy (new_brickinfo->brick_id, old_brickinfo->brick_id, + sizeof (new_brickinfo->brick_id)); + list_add_tail (&new_brickinfo->brick_list, &old_brickinfo->brick_list); diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c index 0ee430969..37cf98894 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.c +++ b/xlators/mgmt/glusterd/src/glusterd-store.c @@ -241,6 +241,11 @@ glusterd_store_brickinfo_write (int fd, glusterd_brickinfo_t *brickinfo) if (ret) goto out; + ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_ID, + brickinfo->brick_id); + if (ret) + goto out; + if (!brickinfo->vg[0]) goto out; @@ -1493,7 +1498,6 @@ out: return ret; } - int32_t glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo) { @@ -1511,6 +1515,7 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo) gf_store_iter_t *tmpiter = NULL; char *tmpvalue = NULL; struct pmap_registry *pmap = NULL; + int brickid = 0; gf_store_op_errno_t op_errno = GD_STORE_SUCCESS; GF_ASSERT (volinfo); @@ -1606,6 +1611,9 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo) strlen (GLUSTERD_STORE_KEY_BRICK_VGNAME))) { strncpy (brickinfo->vg, value, sizeof (brickinfo->vg)); + } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_ID)) { + strncpy (brickinfo->brick_id, value, + sizeof (brickinfo->brick_id)); } else { gf_log ("", GF_LOG_ERROR, "Unknown key: %s", key); @@ -1620,13 +1628,22 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo) &op_errno); } - if (op_errno != GD_STORE_EOF) + if (op_errno != GD_STORE_EOF) { + gf_log ("", GF_LOG_ERROR, "Error parsing brickinfo: " + "op_errno=%d", op_errno); goto out; + } ret = gf_store_iter_destroy (iter); if (ret) goto out; + if (brickinfo->brick_id[0] == '\0') { + /* This is an old volume upgraded to op_version 4 */ + GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (brickinfo, volinfo, + brickid++); + } + list_add_tail (&brickinfo->brick_list, &volinfo->bricks); brick_count++; } diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h index fadea8b2f..955abb09f 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.h +++ b/xlators/mgmt/glusterd/src/glusterd-store.h @@ -65,6 +65,7 @@ typedef enum glusterd_store_ver_ac_{ #define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT "rdma.listen-port" #define GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED "decommissioned" #define GLUSTERD_STORE_KEY_BRICK_VGNAME "vg" +#define GLUSTERD_STORE_KEY_BRICK_ID "brick-id" #define GLUSTERD_STORE_KEY_PEER_UUID "uuid" #define GLUSTERD_STORE_KEY_PEER_HOSTNAME "hostname" diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index 6393c554e..0bec8c06b 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -671,6 +671,30 @@ out: return ret; } +int +glusterd_get_next_available_brickid (glusterd_volinfo_t *volinfo) +{ + glusterd_brickinfo_t *brickinfo = NULL; + char *token = NULL; + int brickid = 0; + int max_brickid = -1; + int ret = -1; + + list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) { + token = strrchr (brickinfo->brick_id, '-'); + ret = gf_string2int32 (++token, &brickid); + if (ret < 0) { + gf_log (THIS->name, GF_LOG_ERROR, + "Unable to generate brick ID"); + return ret; + } + if (brickid > max_brickid) + max_brickid = brickid; + } + + return max_brickid + 1 ; +} + int32_t glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo) { @@ -2131,6 +2155,13 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo, if (ret) goto out; + memset (key, 0, sizeof (key)); + snprintf (key, sizeof (key), "volume%d.brick%d.brick_id", + count, i); + ret = dict_set_str (dict, key, brickinfo->brick_id); + if (ret) + goto out; + i++; } @@ -2805,6 +2836,7 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count, int ret = -1; char *hostname = NULL; char *path = NULL; + char *brick_id = NULL; int decommissioned = 0; glusterd_brickinfo_t *new_brickinfo = NULL; char msg[2048] = {0}; @@ -2831,6 +2863,11 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count, goto out; } + memset (key, 0, sizeof (key)); + snprintf (key, sizeof (key), "volume%d.brick%d.brick_id", + vol_count, brick_count); + ret = dict_get_str (vols, key, &brick_id); + memset (key, 0, sizeof (key)); snprintf (key, sizeof (key), "volume%d.brick%d.decommissioned", vol_count, brick_count); @@ -2847,6 +2884,8 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count, strcpy (new_brickinfo->path, path); strcpy (new_brickinfo->hostname, hostname); new_brickinfo->decommissioned = decommissioned; + if (brick_id) + strcpy (new_brickinfo->brick_id, brick_id); //peerinfo might not be added yet (void) glusterd_resolve_brick (new_brickinfo); ret = 0; @@ -2864,6 +2903,7 @@ glusterd_import_bricks (dict_t *vols, int32_t vol_count, { int ret = -1; int brick_count = 1; + int brickid = 0; glusterd_brickinfo_t *new_brickinfo = NULL; GF_ASSERT (vols); @@ -2875,6 +2915,12 @@ glusterd_import_bricks (dict_t *vols, int32_t vol_count, &new_brickinfo); if (ret) goto out; + if (new_brickinfo->brick_id[0] == '\0') + /*We were probed from a peer having op-version + less than GD_OP_VER_PERSISTENT_AFR_XATTRS*/ + GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (new_brickinfo, + new_volinfo, + brickid++); list_add_tail (&new_brickinfo->brick_list, &new_volinfo->bricks); brick_count++; } diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h index cd22b2960..aebf5fcef 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.h +++ b/xlators/mgmt/glusterd/src/glusterd-utils.h @@ -29,6 +29,10 @@ #include "protocol-common.h" #define GLUSTERD_SOCK_DIR "/var/run" +#define GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(brickinfo, volinfo, brickid) do {\ + sprintf (brickinfo->brick_id, "%s-client-%d",\ + volinfo->volname, brickid);\ +} while (0) struct glusterd_lock_ { uuid_t owner; @@ -124,6 +128,9 @@ int32_t glusterd_service_stop(const char *service, char *pidfile, int sig, gf_boolean_t force_kill); +int +glusterd_get_next_available_brickid (glusterd_volinfo_t *volinfo); + int32_t glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo); diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c index 001825941..9012003c9 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.c +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c @@ -2151,7 +2151,7 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, list_for_each_entry (brick, &volinfo->bricks, brick_list) { ret = -1; xl = volgen_graph_add_nolink (graph, "protocol/client", - "%s-client-%d", volname, i); + "%s", brick->brick_id); if (!xl) goto out; ret = xlator_set_option (xl, "remote-host", brick->hostname); diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c index 4acea7686..135faa40a 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c @@ -1488,6 +1488,7 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr) char *username = NULL; char *password = NULL; int caps = 0; + int brickid = 0; char msg[1024] __attribute__((unused)) = {0, }; this = THIS; @@ -1653,11 +1654,17 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr) brick = strtok_r (brick_list+1, " \n", &saveptr); caps = CAPS_BD | CAPS_THIN | CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT; + brickid = glusterd_get_next_available_brickid (volinfo); + if (brickid < 0) + goto out; while ( i <= count) { ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo); if (ret) goto out; + GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (brickinfo, volinfo, + brickid++); + ret = glusterd_resolve_brick (brickinfo); if (ret) { gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK, diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index df53327cb..0694f7386 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -175,6 +175,7 @@ typedef enum gf_brick_status { struct glusterd_brickinfo { char hostname[1024]; char path[PATH_MAX]; + char brick_id[1024];/*Client xlator name, AFR changelog name*/ struct list_head brick_list; uuid_t uuid; int port; -- cgit