afr: add arbitration support

Add logic in afr to work in conjunction with the arbiter xlator when a
replica 3 arbiter volume is created. More specifically, this patch:

* Enables full locks for afr data transaction for such volumes.
* Removes the upfront marking of pending xattrs at the time of pre-op
  and defer it to post-op. (This is an arbiter independent change and is made for all afr transactions.)
* After pre-op stage, check if we can proceed with the fop stage without
  ending up in split-brain by examining the changelog xattrs.
* Unwinds the fop with failure if only one source was available at the
   time of pre-op and the fop happened to fail on particular source brick.
* Skips data self-heal if arbiter brick is the only source available.
* Adds the arbiter-count option to the shd graph.

This patch is a part of the arbiter logic implementation for 3 way AFR
details of which can be found at http://review.gluster.org/#/c/9656/

Change-Id: I9603db9d04de5626eb2f4d8d959ef5b46113561d
BUG: 1199985
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-on: http://review.gluster.org/10258
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>

1 files changed, 152 insertions, 31 deletions

diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index d2111060035..a2023884465 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -15,7 +15,7 @@
 
 #include "afr.h"
 #include "afr-transaction.h"
-
+#include "afr-self-heal.h"
 #include <signal.h>
 
 gf_boolean_t
@@ -139,14 +139,130 @@ __mark_all_success (call_frame_t *frame, xlator_t *this)
 	}
 }
 
+void
+afr_compute_pre_op_sources (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        afr_transaction_type type = -1;
+        dict_t *xdata = NULL;
+        int **matrix = NULL;
+        int idx = -1;
+        int i = 0;
+        int j = 0;
+
+        priv = this->private;
+        local = frame->local;
+        type = local->transaction.type;
+        idx = afr_index_for_transaction_type (type);
+        matrix = ALLOC_MATRIX (priv->child_count, int);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->transaction.pre_op_xdata[i])
+                        continue;
+                xdata = local->transaction.pre_op_xdata[i];
+                afr_selfheal_fill_matrix (this, matrix, i, idx, xdata);
+        }
+
+        memset (local->transaction.pre_op_sources, 1, priv->child_count);
+
+        /*If lock or pre-op failed on a brick, it is not a source. */
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.failed_subvols[i])
+                        local->transaction.pre_op_sources[i] = 0;
+        }
+
+        /* If brick is blamed by others, it is not a source. */
+        for (i = 0; i < priv->child_count; i++)
+                for (j = 0; j < priv->child_count; j++)
+                        if (matrix[i][j] != 0)
+                                local->transaction.pre_op_sources[j] = 0;
+
+        /*We don't need the xattrs any more. */
+        for (i = 0; i < priv->child_count; i++)
+                if (local->transaction.pre_op_xdata[i]) {
+                        dict_unref (local->transaction.pre_op_xdata[i]);
+                        local->transaction.pre_op_xdata[i] = NULL;
+                }
+}
+
+void
+afr_txn_arbitrate_fop_cbk (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        gf_boolean_t fop_failed = _gf_false;
+        unsigned char *pre_op_sources = NULL;
+        int i = 0;
+
+        local = frame->local;
+        priv  = this->private;
+        pre_op_sources = local->transaction.pre_op_sources;
+
+        if (priv->arbiter_count != 1 || local->op_ret < 0)
+                return;
+
+        /* If the fop failed on the brick, it is not a source. */
+        for (i = 0; i < priv->child_count; i++)
+                if (local->transaction.failed_subvols[i])
+                        pre_op_sources[i] = 0;
+
+        switch (AFR_COUNT (pre_op_sources, priv->child_count)) {
+        case 1:
+                if (pre_op_sources[ARBITER_BRICK_INDEX])
+                        fop_failed = _gf_true;
+                break;
+        case 0:
+                fop_failed = _gf_true;
+                break;
+        }
+
+        if (fop_failed) {
+                local->op_ret = -1;
+                local->op_errno = ENOTCONN;
+        }
+
+        return;
+}
+
+void
+afr_txn_arbitrate_fop (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int pre_op_sources_count = 0;
+
+        priv = this->private;
+        local = frame->local;
+
+        afr_compute_pre_op_sources (frame, this);
+        pre_op_sources_count = AFR_COUNT (local->transaction.pre_op_sources,
+                                          priv->child_count);
+
+        /* If arbiter is the only source, do not proceed. */
+        if (pre_op_sources_count < 2 &&
+            local->transaction.pre_op_sources[ARBITER_BRICK_INDEX]) {
+                local->internal_lock.lock_cbk = local->transaction.done;
+                local->op_ret = -1;
+                local->op_errno =  ENOTCONN;
+                afr_restore_lk_owner (frame);
+                afr_unlock (frame, this);
+        } else {
+                local->transaction.fop (frame, this);
+        }
+
+        return;
+}
 
 int
 afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t     *local = NULL;
+        afr_private_t   *priv = NULL;
         fd_t            *fd   = NULL;
 
         local = frame->local;
+        priv = this->private;
         fd    = local->fd;
 
         /*  Perform fops with the lk-owner from top xlator.
@@ -172,12 +288,15 @@ afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
         */
         if (fd)
                 afr_delayed_changelog_wake_up (this, fd);
-        local->transaction.fop (frame, this);
+        if (priv->arbiter_count == 1) {
+                afr_txn_arbitrate_fop (frame, this);
+        } else {
+                local->transaction.fop (frame, this);
+        }
 
 	return 0;
 }
 
-
 static int
 __changelog_enabled (afr_private_t *priv, afr_transaction_type type)
 {
@@ -372,11 +491,16 @@ afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
+        int pre_op_count = 0;
         int i = 0;
 
         local = frame->local;
 	priv = this->private;
 
+        pre_op_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
+        if (pre_op_count < priv->child_count)
+                return _gf_false;
+
         for (i = 0; i < priv->child_count; i++) {
                 if (local->transaction.failed_subvols[i])
                         return _gf_false;
@@ -591,9 +715,6 @@ afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this)
 	else
 		need_undirty = _gf_true;
 
-        //If the fop fails on all the subvols then pending markers are placed
-        //for every subvol on all subvolumes. Which is nothing but split-brain.
-        //Avoid this by not doing post-op in case of failures.
         if (local->op_ret < 0) {
                 afr_changelog_post_op_done (frame, this);
                 goto out;
@@ -846,12 +967,22 @@ afr_changelog_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
         int call_count = -1;
+        int child_index = -1;
 
         local = frame->local;
+        priv = this->private;
+        child_index = (long) cookie;
 
 	if (op_ret == -1)
-		afr_transaction_fop_failed (frame, this, (long) cookie);
+		afr_transaction_fop_failed (frame, this, child_index);
+
+        if (priv->arbiter_count == 1 && !op_ret) {
+                if (xattr)
+                        local->transaction.pre_op_xdata[child_index] =
+                                                               dict_ref (xattr);
+        }
 
 	call_count = afr_frame_return (frame);
 
@@ -964,7 +1095,6 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
         afr_local_t *local = NULL;
         afr_internal_lock_t *int_lock = NULL;
         unsigned char       *locked_nodes = NULL;
-	unsigned char       *pending_subvols = NULL;
 	int idx = -1;
 	gf_boolean_t pre_nop = _gf_true;
 	dict_t *xdata_req = NULL;
@@ -975,15 +1105,13 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
 
         locked_nodes = afr_locked_nodes_get (local->transaction.type, int_lock);
 
-	pending_subvols = alloca0 (priv->child_count);
-
 	for (i = 0; i < priv->child_count; i++) {
 		if (locked_nodes[i]) {
 			local->transaction.pre_op[i] = 1;
 			call_count++;
 		} else {
-			pending_subvols[i] = 1;
-		}
+                        local->transaction.failed_subvols[i] = 1;
+                }
 	}
 
         /* This condition should not be met with present code, as
@@ -1009,28 +1137,21 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
 		goto err;
 	}
 
-	pre_nop = _gf_true;
-
 	if (afr_changelog_pre_op_inherit (frame, this))
 		goto next;
 
-	if (call_count < priv->child_count) {
-		/* For subvols we are not performing operation on,
-		   mark them as pending up-front along with the FOP
-		   so that we can safely defer unmarking dirty until
-		   later.
-		*/
-		for (i = 0; i < priv->child_count; i++) {
-			if (pending_subvols[i])
-				local->pending[i][idx] = hton32(1);
-		}
-		ret = afr_set_pending_dict (priv, xdata_req,
-					    local->pending);
-		if (ret < 0) {
-			op_errno = ENOMEM;
-			goto err;
-		}
-		pre_nop = _gf_false;
+        if (call_count < priv->child_count)
+                pre_nop = _gf_false;
+
+        /* Set an all-zero pending changelog so that in the cbk, we can get the
+         * current on-disk values. In a replica 3 volume with arbiter enabled,
+         * these values are needed to arrive at a go/ no-go of the fop phase to
+         * avoid ending up in split-brain.*/
+
+        ret = afr_set_pending_dict (priv, xdata_req, local->pending);
+	if (ret < 0) {
+		op_errno = ENOMEM;
+		goto err;
 	}
 
 	if (call_count > 1 &&