cluster/afr: For entry self heal, mark all source bricks

Problem:
Whenever a new brick is added into a replicate volume, all
source bricks are not marked as source. Only one of them is
marked as source. Here marked as source refers to adding
extended attribute at the backend of a file corresponding to
the newly added brick. As well as source bricks should point
to the newly added brick so that heal can be triggered.

Fix:
All source bricks will now point to newly added bricks and heal
can be triggered based on the extended attributes.

Change-Id: I318e1f779a380c16c448a2d05c0140d8e4647fd4
BUG: 1037501
Signed-off-by: Venkatesh Somyajulu <vsomyaju@redhat.com>
Reviewed-on: http://review.gluster.org/6540
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

2 files changed, 269 insertions, 13 deletions

diff --git a/tests/bugs/bug-1037501.t b/tests/bugs/bug-1037501.t
new file mode 100755
index 00000000000..5470d05632c
--- /dev/null
+++ b/tests/bugs/bug-1037501.t
@@ -0,0 +1,220 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+function write_file()
+{
+	path="$1"; shift
+	echo "$*" > "$path"
+}
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+## Start and create a volume
+mkdir -p ${B0}/${V0}-0
+mkdir -p ${B0}/${V0}-1
+mkdir -p ${B0}/${V0}-2
+TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}-{0,1,2}
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Make sure io-cache and write-behind don't interfere.
+TEST $CLI volume set $V0 data-self-heal off;
+
+## Make sure automatic self-heal doesn't perturb our results.
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+
+TEST $CLI volume set $V0 background-self-heal-count 0
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Mount native
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
+
+TEST `echo "TEST-FILE" > $M0/File`
+TEST `mkdir $M0/Dir`
+TEST `ln  $M0/File $M0/Link`
+TEST `mknod $M0/FIFO p`
+
+TEST $CLI volume add-brick $V0 replica 4 $H0:$B0/$V0-3 force
+TEST $CLI volume add-brick $V0 replica 5 $H0:$B0/$V0-4 force
+TEST $CLI volume add-brick $V0 replica 6 $H0:$B0/$V0-5 force
+
+sleep 10
+
+TEST ls $M0/
+
+
+function compare()
+{
+	var=-1;
+	if [ $1 == $2 ]; then
+		var=0;
+	else
+		var=-1;
+	fi
+
+	echo $var
+}
+
+
+var1=`getfattr -d -m . $B0/$V0-0/File -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000020000000200000000"
+
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/File -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/File -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/File -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/File -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/File -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/File -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/File -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/File -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/Dir -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/Dir -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/Dir -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/Dir -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/Dir -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/Dir -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/Dir -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/Dir -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/Dir -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+
+var1=`getfattr -d -m . $B0/$V0-0/Link -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000020000000200000000"
+
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/Link -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/Link -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/Link -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/Link -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/Link -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/Link -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/Link -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/Link -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+
+
+
+var1=`getfattr -d -m . $B0/$V0-0/FIFO -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000000000000100000000"
+
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/FIFO -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/FIFO -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/FIFO -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/FIFO -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/FIFO -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/FIFO -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/FIFO -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/FIFO -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+cleanup;
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 53491a1d74c..0ca06aaa382 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -1002,6 +1002,7 @@ afr_sh_entry_impunge_xattrop_cbk (call_frame_t *impunge_frame, void *cookie,
         afr_private_t   *priv = NULL;
         afr_local_t     *impunge_local = NULL;
         int              child_index = 0;
+        int              call_count = -1;
 
         priv          = this->private;
         impunge_local = impunge_frame->local;
@@ -1012,16 +1013,26 @@ afr_sh_entry_impunge_xattrop_cbk (call_frame_t *impunge_frame, void *cookie,
                 gf_log (this->name, GF_LOG_INFO,
                         "%s: failed to perform xattrop on %s (%s)",
                         impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-                goto out;
+                        priv->children[child_index]->name, strerror (op_errno));
+
+                        LOCK (&impunge_frame->lock);
+                        {
+                                impunge_local->op_ret = -1;
+                                impunge_local->op_errno = op_errno;
+                        }
+                        UNLOCK (&impunge_frame->lock);
         }
 
-        afr_sh_entry_impunge_setattr (impunge_frame, this);
-        return 0;
-out:
-        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                        -1, op_errno);
+        call_count = afr_frame_return (impunge_frame);
+
+        if (call_count == 0) {
+                if (impunge_local->op_ret == 0) {
+                        afr_sh_entry_impunge_setattr (impunge_frame, this);
+                } else {
+                        afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                                -1, impunge_local->op_errno);
+                }
+        }
         return 0;
 }
 
@@ -1035,11 +1046,15 @@ afr_sh_entry_impunge_perform_xattrop (call_frame_t *impunge_frame,
         afr_local_t     *impunge_local    = NULL;
         afr_self_heal_t *impunge_sh       = NULL;
         int32_t         op_errno          = 0;
+        int32_t         call_count        = 0;
+        int32_t         i                 = 0;
+
 
         priv = this->private;
         impunge_local = impunge_frame->local;
         impunge_sh = &impunge_local->self_heal;
         active_src = impunge_sh->active_source;
+        impunge_local->op_ret = 0;
 
         afr_prepare_new_entry_pending_matrix (impunge_local->pending,
                                               afr_is_errno_unset,
@@ -1055,11 +1070,32 @@ afr_sh_entry_impunge_perform_xattrop (call_frame_t *impunge_frame,
         afr_set_pending_dict (priv, xattr, impunge_local->pending, active_src,
                               LOCAL_LAST);
 
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_xattrop_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->xattrop,
-                           &impunge_local->loc, GF_XATTROP_ADD_ARRAY, xattr, NULL);
+        for (i = 0; i < priv->child_count; i++) {
+                if ((impunge_sh->child_errno[i] == EEXIST) &&
+                    (impunge_local->child_up[i] == 1))
+
+                        call_count++;
+        }
+
+        impunge_local->call_count  = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+
+                if ((impunge_sh->child_errno[i] == EEXIST)
+                    && (impunge_local->child_up[i] == 1)) {
+
+
+                        STACK_WIND_COOKIE (impunge_frame,
+                                           afr_sh_entry_impunge_xattrop_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->xattrop,
+                                           &impunge_local->loc,
+                                           GF_XATTROP_ADD_ARRAY, xattr, NULL);
+                        if (!--call_count)
+                                break;
+                }
+        }
 
         if (xattr)
                 dict_unref (xattr);