cluster/afr: Choose local child as source if possible

It is better to choose local brick as source if possible to prevent
over the wire read thus saving on bandwidth. Also changed code to not
attempt data-heal if 'source' is selected as arbiter.

Change-Id: I9a328d0198422280b13a30ab99545370a301dfea
BUG: 1314150
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/13585
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Krutika Dhananjay <kdhananj@redhat.com>
Tested-by: Krutika Dhananjay <kdhananj@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>

9 files changed, 57 insertions, 31 deletions

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 7f6bf74abd9..e0f530f999a 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1886,6 +1886,7 @@ afr_local_discovery_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
          * the slowest local subvolume is far preferable to a remote one.
          */
         if (is_local) {
+                priv->local[child_index] = 1;
                 /* Don't set arbiter as read child. */
                 if (AFR_IS_ARBITER_BRICK(priv, child_index))
                         goto out;
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 74e1a444069..f12ce81b0ee 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1532,3 +1532,32 @@ afr_throttled_selfheal (call_frame_t *frame, xlator_t *this)
                                       "pending, background self-heal rejected.");
         }
 }
+
+int
+afr_choose_source_by_policy (afr_private_t *priv, unsigned char *sources,
+                             afr_transaction_type type)
+{
+        int source = -1;
+        int i      = 0;
+
+        /* Give preference to local child to save on bandwidth */
+        for (i = 0; i < priv->child_count; i++) {
+                if (priv->local[i] && sources[i]) {
+                        if ((type == AFR_DATA_TRANSACTION) &&
+                            AFR_IS_ARBITER_BRICK (priv, i))
+                                continue;
+
+                        source = i;
+                        goto out;
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (sources[i]) {
+                        source = i;
+                        goto out;
+                }
+        }
+out:
+        return source;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 27d2849c158..a2dddafe5c0 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -572,7 +572,6 @@ __afr_selfheal_data_finalize_source (call_frame_t *frame, xlator_t *this,
 				     struct afr_reply *replies,
                                      uint64_t *witness)
 {
-	int i = 0;
 	afr_private_t *priv = NULL;
 	int source = -1;
 	int sources_count = 0;
@@ -609,13 +608,9 @@ __afr_selfheal_data_finalize_source (call_frame_t *frame, xlator_t *this,
 
 out:
         afr_mark_active_sinks (this, sources, locked_on, healed_sinks);
+        source = afr_choose_source_by_policy (priv, sources,
+                                              AFR_DATA_TRANSACTION);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i]) {
-                        source = i;
-                        break;
-                }
-        }
 	return source;
 }
 
@@ -729,8 +724,7 @@ __afr_selfheal_data (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 		source = ret;
 
-                if (AFR_IS_ARBITER_BRICK(priv, source) &&
-                    AFR_COUNT (sources, priv->child_count) == 1) {
+                if (AFR_IS_ARBITER_BRICK(priv, source)) {
                         did_sh = _gf_false;
                         goto unlock;
                 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 42a6c0453b1..e4d616ea20a 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -356,7 +356,6 @@ __afr_selfheal_entry_finalize_source (xlator_t *this, unsigned char *sources,
                                       struct afr_reply *replies,
                                       uint64_t *witness)
 {
-	int i = 0;
 	afr_private_t *priv = NULL;
 	int source = -1;
 	int sources_count = 0;
@@ -373,13 +372,8 @@ __afr_selfheal_entry_finalize_source (xlator_t *this, unsigned char *sources,
 		return -1;
 	}
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sources[i]) {
-			source = i;
-			break;
-		}
-	}
-
+        source = afr_choose_source_by_policy (priv, sources,
+                                              AFR_ENTRY_TRANSACTION);
 	return source;
 }
 
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index 24b139ad216..65c25abcb4a 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -204,7 +204,7 @@ __afr_selfheal_metadata_finalize_source (call_frame_t *frame, xlator_t *this,
 {
 	int i = 0;
 	afr_private_t *priv = NULL;
-	struct iatt first = {0, };
+	struct iatt srcstat = {0, };
 	int source = -1;
 	int sources_count = 0;
 
@@ -257,23 +257,17 @@ __afr_selfheal_metadata_finalize_source (call_frame_t *frame, xlator_t *this,
         if (afr_dict_contains_heal_op(frame))
                 return -EIO;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!sources[i])
-			continue;
-		if (source == -1) {
-			source = i;
-			first = replies[i].poststat;
-                        break;
-		}
-	}
+        source = afr_choose_source_by_policy (priv, sources,
+                                              AFR_METADATA_TRANSACTION);
+        srcstat = replies[source].poststat;
 
 	for (i = 0; i < priv->child_count; i++) {
 		if (!sources[i] || i == source)
 			continue;
-		if (!IA_EQUAL (first, replies[i].poststat, type) ||
-		    !IA_EQUAL (first, replies[i].poststat, uid) ||
-		    !IA_EQUAL (first, replies[i].poststat, gid) ||
-		    !IA_EQUAL (first, replies[i].poststat, prot)) {
+		if (!IA_EQUAL (srcstat, replies[i].poststat, type) ||
+		    !IA_EQUAL (srcstat, replies[i].poststat, uid) ||
+		    !IA_EQUAL (srcstat, replies[i].poststat, gid) ||
+		    !IA_EQUAL (srcstat, replies[i].poststat, prot)) {
                         gf_msg_debug (this->name, 0, "%s: iatt mismatch "
                                       "for source(%d) vs (%d)",
                                       uuid_utoa
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index b298fa130c3..b0f545f2816 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -266,4 +266,7 @@ afr_selfheal_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 afr_locked_fill (call_frame_t *frame, xlator_t *this,
                  unsigned char *locked_on);
+int
+afr_choose_source_by_policy (afr_private_t *priv, unsigned char *sources,
+                             afr_transaction_type type);
 #endif /* !_AFR_SELFHEAL_H */
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index c539e117607..228ea388662 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -516,14 +516,17 @@ afr_shd_index_healer (void *data)
 	struct subvol_healer *healer = NULL;
 	xlator_t *this = NULL;
 	int ret = 0;
+	afr_private_t *priv = NULL;
 
 	healer = data;
 	THIS = this = healer->this;
+	priv = this->private;
 
 	for (;;) {
 		afr_shd_healer_wait (healer);
 
 		ASSERT_LOCAL(this, healer);
+		priv->local[healer->subvol] = healer->local;
 
 		do {
 		        gf_msg_debug (this->name, 0,
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 535f3e5b946..2c7e69aae3b 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -387,6 +387,13 @@ init (xlator_t *this)
 
         priv->wait_count = 1;
 
+        priv->local = GF_CALLOC (sizeof (unsigned char), child_count,
+                                 gf_afr_mt_char);
+        if (!priv->local) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
         priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
                                     gf_afr_mt_char);
         if (!priv->child_up) {
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 7bb5aa8ded2..41ffa4ecdb2 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -61,6 +61,7 @@ typedef struct _afr_private {
         inode_t *root_inode;
 
         unsigned char *child_up;
+        unsigned char *local;
 
         char **pending_key;