From 306f29a8f49dc7358e195cd0a80531776ff679f7 Mon Sep 17 00:00:00 2001 From: Raghavan P Date: Wed, 22 Jan 2014 14:53:28 +0530 Subject: NSR reconciliation: Adding index for each op We need to have the leader generate an index number for each changelog entry. Change-Id: If451dc00a21dbc481771b713ac75fd0af6887014 Signed-off-by: Raghavan P --- xlators/cluster/nsr-server/src/all-templates.c | 10 ++++++++++ xlators/cluster/nsr-server/src/leader.c | 4 ++-- xlators/cluster/nsr-server/src/nsr-internal.h | 2 ++ xlators/cluster/nsr-server/src/nsr.c | 2 ++ xlators/cluster/nsr-server/src/recon_notify.c | 1 + .../changelog/src/policy/changelog-policy-replication.c | 7 +++++-- 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/xlators/cluster/nsr-server/src/all-templates.c b/xlators/cluster/nsr-server/src/all-templates.c index 5a62ae507..3403aed8c 100644 --- a/xlators/cluster/nsr-server/src/all-templates.c +++ b/xlators/cluster/nsr-server/src/all-templates.c @@ -58,6 +58,7 @@ nsr_$NAME$ (call_frame_t *frame, xlator_t *this, int op_errno = ENOMEM; int from_leader; int from_recon; + uint32_t ti = 0; local = mem_get0(this->local_pool); if (!local) { @@ -112,6 +113,15 @@ nsr_$NAME$ (call_frame_t *frame, xlator_t *this, goto err; } + LOCK(&priv->index_lock); + ti = ++(priv->index); + UNLOCK(&priv->index_lock); + if (dict_set_int32(xdata,NSR_INDEX_XATTR,ti) != 0) { + gf_log (this->name, GF_LOG_ERROR, + "failed to set index"); + goto err; + } + local->stub = fop_$NAME$_stub (frame,nsr_$NAME$_continue, $ARGS_SHORT$); if (!local->stub) { diff --git a/xlators/cluster/nsr-server/src/leader.c b/xlators/cluster/nsr-server/src/leader.c index 319f99317..645f68bf6 100644 --- a/xlators/cluster/nsr-server/src/leader.c +++ b/xlators/cluster/nsr-server/src/leader.c @@ -79,13 +79,13 @@ nsr_set_leader (xlator_t *this, etcd_session etcd) } priv->leader = _gf_true; + priv->current_term = term + 1; + if (priv->nsr_recon_start == _gf_false) { atomic_fetch_and(&(priv->fence_io), 0); return; } - priv->current_term = term + 1; - // Move this inside recon notify??? atomic_fetch_or(&(priv->fence_io), 1); diff --git a/xlators/cluster/nsr-server/src/nsr-internal.h b/xlators/cluster/nsr-server/src/nsr-internal.h index 4382f5426..7825973f7 100644 --- a/xlators/cluster/nsr-server/src/nsr-internal.h +++ b/xlators/cluster/nsr-server/src/nsr-internal.h @@ -56,6 +56,8 @@ typedef struct { gf_boolean_t nsr_recon_start; void * recon_ctx; volatile uint32_t ops_in_flight; + uint32_t index; + gf_lock_t index_lock; } nsr_private_t; typedef struct { diff --git a/xlators/cluster/nsr-server/src/nsr.c b/xlators/cluster/nsr-server/src/nsr.c index d4b78c218..b4618bb62 100644 --- a/xlators/cluster/nsr-server/src/nsr.c +++ b/xlators/cluster/nsr-server/src/nsr.c @@ -138,6 +138,7 @@ nsr_mark_fd_dirty (xlator_t *this, nsr_local_t *local) } #define NSR_TERM_XATTR "trusted.nsr.term" +#define NSR_INDEX_XATTR "trusted.nsr.index" #define RECON_TERM_XATTR "trusted.nsr.recon-term" #define RECON_INDEX_XATTR "trusted.nsr.recon-index" #define NSR_REP_COUNT_XATTR "trusted.nsr.rep-count" @@ -533,6 +534,7 @@ nsr_init (xlator_t *this) } LOCK_INIT(&priv->dirty_lock); + LOCK_INIT(&priv->index_lock); INIT_LIST_HEAD(&priv->dirty_fds); this->private = priv; diff --git a/xlators/cluster/nsr-server/src/recon_notify.c b/xlators/cluster/nsr-server/src/recon_notify.c index 9cf2fce5d..7a0de85b1 100644 --- a/xlators/cluster/nsr-server/src/recon_notify.c +++ b/xlators/cluster/nsr-server/src/recon_notify.c @@ -133,6 +133,7 @@ nsr_recon_set_leader (xlator_t *this) // TBD - error handling ctx->last_reconciled_term = priv->current_term; + priv->index = 0; // reset changelog index atomic_fetch_and(&(priv->fence_io), 0); return; diff --git a/xlators/features/changelog/src/policy/changelog-policy-replication.c b/xlators/features/changelog/src/policy/changelog-policy-replication.c index 2c54d2cff..5c6fca4e0 100644 --- a/xlators/features/changelog/src/policy/changelog-policy-replication.c +++ b/xlators/features/changelog/src/policy/changelog-policy-replication.c @@ -112,6 +112,7 @@ uuid_fn (void *data, char *buffer, gf_boolean_t encode) /* TBD: move declarations here and nsr.c into a common place */ #define NSR_TERM_XATTR "trusted.nsr.term" +#define NSR_INDEX_XATTR "trusted.nsr.index" #define RECON_TERM_XATTR "trusted.nsr.recon-term" #define RECON_INDEX_XATTR "trusted.nsr.recon-index" @@ -121,6 +122,7 @@ changelog_fix_term(xlator_t *this, dict_t *xdata) { int32_t old_term, new_term; + uint32_t index; changelog_priv_t *priv = this->private; int ret = 0; char nfile[PATH_MAX] = {0,}; @@ -130,7 +132,8 @@ changelog_fix_term(xlator_t *this, // If coming via the regular IO path, we should get the dict "nsr-term" // If coming via reconciliation, we should get the dicts "nsr-recon-term" // that indicates the term and "nsr-recon-index" for the index - if (dict_get_int32(xdata,NSR_TERM_XATTR,&new_term) == 0) { + if ((dict_get_int32(xdata,NSR_TERM_XATTR,&new_term) == 0) && + (dict_get_int32(xdata, NSR_INDEX_XATTR, &index) == 0)) { old_term = priv->term; if (old_term != new_term) { @@ -147,7 +150,7 @@ changelog_fix_term(xlator_t *this, return _gf_false; } local->nr_bytes = 0; - local->lu.val = get_index (priv); + local->lu.val = index; } else if ((dict_get_int32(xdata, RECON_TERM_XATTR, &recon_term) == 0) && (dict_get_int32(xdata, RECON_INDEX_XATTR, &recon_index) == 0)) { -- cgit