From 927483b6fbf2a8e731f9e34dd70dc4179accc079 Mon Sep 17 00:00:00 2001 From: Jeff Darcy Date: Tue, 25 Mar 2014 16:48:37 +0000 Subject: nsr: use different etcd keys for each NSR subvolume This is necessary both for separate volumes using NSR and for DHT volumes composed of multiple NSR subvolumes. Change-Id: Ia269d70b535cc26900f8b6e7f22706087746fbe7 Signed-off-by: Jeff Darcy --- xlators/cluster/nsr-recon/src/recon_driver.c | 1 + xlators/cluster/nsr-server/src/leader.c | 12 +++---- xlators/cluster/nsr-server/src/nsr-internal.h | 5 +-- xlators/cluster/nsr-server/src/nsr.c | 51 ++++++++++++++++++++++----- xlators/cluster/nsr-server/src/recon_notify.c | 2 +- 5 files changed, 53 insertions(+), 18 deletions(-) (limited to 'xlators/cluster') diff --git a/xlators/cluster/nsr-recon/src/recon_driver.c b/xlators/cluster/nsr-recon/src/recon_driver.c index b02abf2bb..e0327f81d 100644 --- a/xlators/cluster/nsr-recon/src/recon_driver.c +++ b/xlators/cluster/nsr-recon/src/recon_driver.c @@ -3066,6 +3066,7 @@ nsr_reconciliation_driver(void *arg) nsr_driver_log (this->name, GF_LOG_INFO, "finished recon work as joiner \n"); + break; default: nsr_driver_log (this->name, GF_LOG_ERROR, diff --git a/xlators/cluster/nsr-server/src/leader.c b/xlators/cluster/nsr-server/src/leader.c index 944c85cdc..02a2609c8 100644 --- a/xlators/cluster/nsr-server/src/leader.c +++ b/xlators/cluster/nsr-server/src/leader.c @@ -30,7 +30,6 @@ #include "../../nsr-recon/src/recon_driver.h" #include "../../nsr-recon/src/recon_xlator.h" -#define NSR_KEY "xyzzy" #define NSR_TTL 5 static void @@ -39,20 +38,19 @@ nsr_set_leader (xlator_t *this, etcd_session etcd) long term = 0; etcd_result res; nsr_private_t *priv = this->private; - char *term_key = priv->term_uuid; char n_t[sizeof(long)+1]; char *text = NULL; gf_log (this->name, GF_LOG_INFO, "Just became leader"); - text = etcd_get(etcd, priv->term_uuid); + text = etcd_get(etcd, priv->term_key); if(text == NULL) { term = 0; } else { term = strtol(text, NULL, 10); } sprintf(n_t,"%ld",term+1); - res = etcd_set(etcd, term_key,n_t,text,0); + res = etcd_set(etcd, priv->term_key,n_t,text,0); if(res != ETCD_OK) { gf_log (this->name, GF_LOG_ERROR, "failed to set term"); return; @@ -84,7 +82,7 @@ nsr_leader_thread (void *arg) char *index_out = NULL; gf_log (this->name, GF_LOG_INFO, - "calling glfs_open_str on servers %s", priv->etcd_servers); + "calling etcd_open_str on servers %s", priv->etcd_servers); priv->etcd = etcd_open_str(priv->etcd_servers); if (!(priv->etcd)) { @@ -98,7 +96,7 @@ nsr_leader_thread (void *arg) for (;;) { /* Not leader yet. Try to become leader. */ for (;;) { - res = etcd_lock (priv->etcd, NSR_KEY, NSR_TTL, + res = etcd_lock (priv->etcd, priv->leader_key, NSR_TTL, index_in, &index_out); if (res == ETCD_OK) { break; @@ -113,7 +111,7 @@ nsr_leader_thread (void *arg) index_in = index_out; index_out = NULL; for (;;) { - res = etcd_lock (priv->etcd, NSR_KEY, NSR_TTL, + res = etcd_lock (priv->etcd, priv->leader_key, NSR_TTL, index_in, &index_out); if (index_out && (index_in != index_out)) { if (index_in) { diff --git a/xlators/cluster/nsr-server/src/nsr-internal.h b/xlators/cluster/nsr-server/src/nsr-internal.h index 7825973f7..fc612c136 100644 --- a/xlators/cluster/nsr-server/src/nsr-internal.h +++ b/xlators/cluster/nsr-server/src/nsr-internal.h @@ -35,8 +35,9 @@ typedef struct _nsr_recon_notify_ev_s { typedef struct { char *etcd_servers; - char *vol_uuid; - char *term_uuid; + char *subvol_uuid; + char *leader_key; + char *term_key; char *brick_uuid; gf_boolean_t leader; uint8_t up_children; diff --git a/xlators/cluster/nsr-server/src/nsr.c b/xlators/cluster/nsr-server/src/nsr.c index 214c54f18..eda9e555a 100644 --- a/xlators/cluster/nsr-server/src/nsr.c +++ b/xlators/cluster/nsr-server/src/nsr.c @@ -500,6 +500,25 @@ out: extern void *nsr_leader_thread (void *); +void +nsr_deallocate_priv (nsr_private_t *priv) +{ + if (!priv) { + return; + } + + if (priv->leader_key) { + GF_FREE(priv->leader_key); + } + + if (priv->term_key) { + GF_FREE(priv->term_key); + } + + GF_FREE(priv); +} + + int32_t nsr_init (xlator_t *this) { @@ -571,12 +590,22 @@ nsr_init (xlator_t *this) gf_log (this->name, GF_LOG_ERROR, "etcd servers not generated. ???"); goto err; } - priv->vol_uuid = "temporary"; + + GF_OPTION_INIT ("subvol-uuid", priv->subvol_uuid, str, err); + gf_log (this->name, GF_LOG_INFO, "subvol_uuid = %s", priv->subvol_uuid); + if (gf_asprintf(&priv->leader_key,"%s:leader",priv->subvol_uuid) <= 0) { + gf_log (this->name, GF_LOG_ERROR, + "could not generate leader key"); + goto err; + } + if (gf_asprintf(&priv->term_key,"%s:term",priv->subvol_uuid) <= 0) { + gf_log (this->name, GF_LOG_ERROR, + "could not generate term key"); + goto err; + } uuid_generate(tmp_uuid); priv->brick_uuid = strdup(uuid_utoa(tmp_uuid)); - priv->term_uuid = "nsr-term"; - gf_log (this->name, GF_LOG_INFO, - "brick_uuid = %s\n", priv->brick_uuid); + gf_log (this->name, GF_LOG_INFO, "brick_uuid = %s\n", priv->brick_uuid); GF_OPTION_INIT ("my-name", my_name, str, err); if (!my_name) { @@ -707,7 +736,10 @@ nsr_init (xlator_t *this) sleep(1); } - (void)pthread_create(&kid,NULL,nsr_leader_thread,this); + if (pthread_create(&kid,NULL,nsr_leader_thread,this) != 0) { + gf_log (this->name, GF_LOG_ERROR, + "failed to start leader thread"); + } while (priv->leader_inited == 0) { sleep(1); } @@ -726,9 +758,7 @@ nsr_init (xlator_t *this) return 0; err: - if (priv) { - GF_FREE(priv); - } + nsr_deallocate_priv(priv); return -1; } @@ -736,6 +766,7 @@ err: void nsr_fini (xlator_t *this) { + nsr_deallocate_priv(this->private); } class_methods_t class_methods = { @@ -765,5 +796,9 @@ struct volume_options options[] = { .type = GF_OPTION_TYPE_STR, .description = "list of comma seperated etc servers" }, + { .key = {"subvol-uuid"}, + .type = GF_OPTION_TYPE_STR, + .description = "UUID for this NSR (sub)volume" + }, { .key = {NULL} }, }; diff --git a/xlators/cluster/nsr-server/src/recon_notify.c b/xlators/cluster/nsr-server/src/recon_notify.c index 7397192ae..24f7cf2de 100644 --- a/xlators/cluster/nsr-server/src/recon_notify.c +++ b/xlators/cluster/nsr-server/src/recon_notify.c @@ -234,7 +234,7 @@ nsr_setup_recon (xlator_t *this) if (priv->nsr_recon_start == _gf_false) return 0; - ctx->fs = glfs_new(priv->vol_uuid); + ctx->fs = glfs_new(priv->subvol_uuid); if (!ctx->fs) { ret = 1; gf_log (this->name, GF_LOG_ERROR, "failed to initialise glfs \n"); -- cgit