From d7c30458c87f1341295f764e1a2a539dc4248ecc Mon Sep 17 00:00:00 2001 From: Ravishankar N Date: Wed, 15 Apr 2015 22:22:08 +0530 Subject: afr: add arbitration support Backport of http://review.gluster.org/#/c/10258/ Add logic in afr to work in conjunction with the arbiter xlator when a replica 3 arbiter volume is created. More specifically, this patch: * Enables full locks for afr data transaction for such volumes. * Removes the upfront marking of pending xattrs at the time of pre-op and defer it to post-op. (This is an arbiter independent change and is made for all afr transactions.) * After pre-op stage, check if we can proceed with the fop stage without ending up in split-brain by examining the changelog xattrs. * Unwinds the fop with failure if only one source was available at the time of pre-op and the fop happened to fail on particular source brick. * Skips data self-heal if arbiter brick is the only source available. * Adds the arbiter-count option to the shd graph. This patch is a part of the arbiter logic implementation for 3 way AFR details of which can be found at http://review.gluster.org/#/c/9656/ Change-Id: I9603db9d04de5626eb2f4d8d959ef5b46113561d BUG: 1217689 Signed-off-by: Ravishankar N Reviewed-on: http://review.gluster.org/10514 Tested-by: Gluster Build System Reviewed-by: Pranith Kumar Karampuri --- xlators/cluster/afr/src/afr-transaction.c | 183 +++++++++++++++++++++++++----- 1 file changed, 152 insertions(+), 31 deletions(-) (limited to 'xlators/cluster/afr/src/afr-transaction.c') diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c index d2111060035..a2023884465 100644 --- a/xlators/cluster/afr/src/afr-transaction.c +++ b/xlators/cluster/afr/src/afr-transaction.c @@ -15,7 +15,7 @@ #include "afr.h" #include "afr-transaction.h" - +#include "afr-self-heal.h" #include gf_boolean_t @@ -139,14 +139,130 @@ __mark_all_success (call_frame_t *frame, xlator_t *this) } } +void +afr_compute_pre_op_sources (call_frame_t *frame, xlator_t *this) +{ + afr_local_t *local = NULL; + afr_private_t *priv = NULL; + afr_transaction_type type = -1; + dict_t *xdata = NULL; + int **matrix = NULL; + int idx = -1; + int i = 0; + int j = 0; + + priv = this->private; + local = frame->local; + type = local->transaction.type; + idx = afr_index_for_transaction_type (type); + matrix = ALLOC_MATRIX (priv->child_count, int); + + for (i = 0; i < priv->child_count; i++) { + if (!local->transaction.pre_op_xdata[i]) + continue; + xdata = local->transaction.pre_op_xdata[i]; + afr_selfheal_fill_matrix (this, matrix, i, idx, xdata); + } + + memset (local->transaction.pre_op_sources, 1, priv->child_count); + + /*If lock or pre-op failed on a brick, it is not a source. */ + for (i = 0; i < priv->child_count; i++) { + if (local->transaction.failed_subvols[i]) + local->transaction.pre_op_sources[i] = 0; + } + + /* If brick is blamed by others, it is not a source. */ + for (i = 0; i < priv->child_count; i++) + for (j = 0; j < priv->child_count; j++) + if (matrix[i][j] != 0) + local->transaction.pre_op_sources[j] = 0; + + /*We don't need the xattrs any more. */ + for (i = 0; i < priv->child_count; i++) + if (local->transaction.pre_op_xdata[i]) { + dict_unref (local->transaction.pre_op_xdata[i]); + local->transaction.pre_op_xdata[i] = NULL; + } +} + +void +afr_txn_arbitrate_fop_cbk (call_frame_t *frame, xlator_t *this) +{ + afr_local_t *local = NULL; + afr_private_t *priv = NULL; + gf_boolean_t fop_failed = _gf_false; + unsigned char *pre_op_sources = NULL; + int i = 0; + + local = frame->local; + priv = this->private; + pre_op_sources = local->transaction.pre_op_sources; + + if (priv->arbiter_count != 1 || local->op_ret < 0) + return; + + /* If the fop failed on the brick, it is not a source. */ + for (i = 0; i < priv->child_count; i++) + if (local->transaction.failed_subvols[i]) + pre_op_sources[i] = 0; + + switch (AFR_COUNT (pre_op_sources, priv->child_count)) { + case 1: + if (pre_op_sources[ARBITER_BRICK_INDEX]) + fop_failed = _gf_true; + break; + case 0: + fop_failed = _gf_true; + break; + } + + if (fop_failed) { + local->op_ret = -1; + local->op_errno = ENOTCONN; + } + + return; +} + +void +afr_txn_arbitrate_fop (call_frame_t *frame, xlator_t *this) +{ + afr_local_t *local = NULL; + afr_private_t *priv = NULL; + int pre_op_sources_count = 0; + + priv = this->private; + local = frame->local; + + afr_compute_pre_op_sources (frame, this); + pre_op_sources_count = AFR_COUNT (local->transaction.pre_op_sources, + priv->child_count); + + /* If arbiter is the only source, do not proceed. */ + if (pre_op_sources_count < 2 && + local->transaction.pre_op_sources[ARBITER_BRICK_INDEX]) { + local->internal_lock.lock_cbk = local->transaction.done; + local->op_ret = -1; + local->op_errno = ENOTCONN; + afr_restore_lk_owner (frame); + afr_unlock (frame, this); + } else { + local->transaction.fop (frame, this); + } + + return; +} int afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this) { afr_local_t *local = NULL; + afr_private_t *priv = NULL; fd_t *fd = NULL; local = frame->local; + priv = this->private; fd = local->fd; /* Perform fops with the lk-owner from top xlator. @@ -172,12 +288,15 @@ afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this) */ if (fd) afr_delayed_changelog_wake_up (this, fd); - local->transaction.fop (frame, this); + if (priv->arbiter_count == 1) { + afr_txn_arbitrate_fop (frame, this); + } else { + local->transaction.fop (frame, this); + } return 0; } - static int __changelog_enabled (afr_private_t *priv, afr_transaction_type type) { @@ -372,11 +491,16 @@ afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this) { afr_private_t *priv = NULL; afr_local_t *local = NULL; + int pre_op_count = 0; int i = 0; local = frame->local; priv = this->private; + pre_op_count = AFR_COUNT (local->transaction.pre_op, priv->child_count); + if (pre_op_count < priv->child_count) + return _gf_false; + for (i = 0; i < priv->child_count; i++) { if (local->transaction.failed_subvols[i]) return _gf_false; @@ -591,9 +715,6 @@ afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this) else need_undirty = _gf_true; - //If the fop fails on all the subvols then pending markers are placed - //for every subvol on all subvolumes. Which is nothing but split-brain. - //Avoid this by not doing post-op in case of failures. if (local->op_ret < 0) { afr_changelog_post_op_done (frame, this); goto out; @@ -846,12 +967,22 @@ afr_changelog_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, int op_errno, dict_t *xattr, dict_t *xdata) { afr_local_t *local = NULL; + afr_private_t *priv = NULL; int call_count = -1; + int child_index = -1; local = frame->local; + priv = this->private; + child_index = (long) cookie; if (op_ret == -1) - afr_transaction_fop_failed (frame, this, (long) cookie); + afr_transaction_fop_failed (frame, this, child_index); + + if (priv->arbiter_count == 1 && !op_ret) { + if (xattr) + local->transaction.pre_op_xdata[child_index] = + dict_ref (xattr); + } call_count = afr_frame_return (frame); @@ -964,7 +1095,6 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this) afr_local_t *local = NULL; afr_internal_lock_t *int_lock = NULL; unsigned char *locked_nodes = NULL; - unsigned char *pending_subvols = NULL; int idx = -1; gf_boolean_t pre_nop = _gf_true; dict_t *xdata_req = NULL; @@ -975,15 +1105,13 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this) locked_nodes = afr_locked_nodes_get (local->transaction.type, int_lock); - pending_subvols = alloca0 (priv->child_count); - for (i = 0; i < priv->child_count; i++) { if (locked_nodes[i]) { local->transaction.pre_op[i] = 1; call_count++; } else { - pending_subvols[i] = 1; - } + local->transaction.failed_subvols[i] = 1; + } } /* This condition should not be met with present code, as @@ -1009,28 +1137,21 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this) goto err; } - pre_nop = _gf_true; - if (afr_changelog_pre_op_inherit (frame, this)) goto next; - if (call_count < priv->child_count) { - /* For subvols we are not performing operation on, - mark them as pending up-front along with the FOP - so that we can safely defer unmarking dirty until - later. - */ - for (i = 0; i < priv->child_count; i++) { - if (pending_subvols[i]) - local->pending[i][idx] = hton32(1); - } - ret = afr_set_pending_dict (priv, xdata_req, - local->pending); - if (ret < 0) { - op_errno = ENOMEM; - goto err; - } - pre_nop = _gf_false; + if (call_count < priv->child_count) + pre_nop = _gf_false; + + /* Set an all-zero pending changelog so that in the cbk, we can get the + * current on-disk values. In a replica 3 volume with arbiter enabled, + * these values are needed to arrive at a go/ no-go of the fop phase to + * avoid ending up in split-brain.*/ + + ret = afr_set_pending_dict (priv, xdata_req, local->pending); + if (ret < 0) { + op_errno = ENOMEM; + goto err; } if (call_count > 1 && -- cgit