cluster/distribute: Ignore non-participating subvols for layout checks

When subvols-per-directory is < available subvols, then there are layouts
which are not populated. This leads to incorrect identification of holes or
overlaps. We need to ignore layouts, which have err == 0, and start == stop.
In the current scenario (start == stop == 0).

Additionally, in layout-merge, treat missing xattrs as err = 0. In case of
missing layouts, anomalies will reset them.

For any other valid subvoles, err != 0 in case of layouts being zeroed out.

Also reverted back dht_selfheal_dir_xattr, which does layout calculation only
on subvols which have errors.

Change-Id: I9f57062722c9e8a26285e10675c31a78921115a1
BUG: 921408
Signed-off-by: shishir gowda <sgowda@redhat.com>
Reviewed-on: http://review.gluster.org/4668
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>

4 files changed, 178 insertions, 27 deletions

diff --git a/tests/bugs/bug-921408.t b/tests/bugs/bug-921408.t
new file mode 100755
index 00000000000..ef2b4fb21cd
--- /dev/null
+++ b/tests/bugs/bug-921408.t
@@ -0,0 +1,89 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../fileio.rc
+. $(dirname $0)/../dht.rc
+
+cleanup;
+wait_check_status ()
+{
+        n=0
+        while [ $n -lt $1 ]
+        do
+                ret=$(rebalance_completed)
+                if [ $ret == "0" ]
+                then
+                        return 0;
+                else
+                        sleep 1
+                        n=`expr $n + 1`;
+                fi
+       done
+       return 1;
+}
+
+addbr_rebal_till_layout_change()
+{
+        val=1
+        l=$1
+        i=1
+        while [ $i -lt 5 ]
+        do
+                $CLI volume add-brick $V0 $H0:$B0/${V0}$l &>/dev/null
+                $CLI volume rebalance $V0 fix-layout start &>/dev/null
+                wait_check_status 15
+                if [ $? -eq 1 ]
+                then
+                        break
+                fi
+                NEW_LAYOUT=`get_layout $B0/${V0}0`
+                if [ $OLD_LAYOUT == $NEW_LAYOUT ]
+                then
+                        i=`expr $i + 1`;
+                        l=`expr $l + 1`;
+                else
+                        val=0
+                        break
+                fi
+        done
+        return $val
+}
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}0
+TEST $CLI volume set $V0 subvols-per-directory 1
+TEST $CLI volume start $V0
+
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+
+TEST mkdir $M0/test
+TEST touch $M0/test/test
+
+fd=`fd_available`
+TEST fd_open $fd "rw" $M0/test/test
+
+OLD_LAYOUT=`get_layout $B0/${V0}0`
+
+addbr_rebal_till_layout_change 1
+
+TEST [ $? -eq 0 ]
+
+for i in $(seq 1 1000)
+do
+	ls -l $M0/ >/dev/null
+	ret=$?
+	if [ $ret != 0 ]
+	then
+		break
+	fi
+done
+
+TEST [ $ret == 0 ];
+TEST fd_close $fd;
+
+TEST umount $M0
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0
+
+cleanup
diff --git a/tests/dht.rc b/tests/dht.rc
index ee92a47bd62..663ea543173 100644
--- a/tests/dht.rc
+++ b/tests/dht.rc
@@ -2,13 +2,7 @@
 
 function get_layout()
 {
-        layout=`getfattr -n trusted.glusterfs.dht -e hex $1 2>&1|grep dht |cut -d = -f2`
-        if [ $? -eq 1]
-        then
-                return -1
-        else
-                return $layout
-        fi
+        getfattr -n trusted.glusterfs.dht -e hex $1 2>&1|grep dht |cut -d = -f2
 
 }
 
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
index 7d53e3d04e0..0572298699c 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -366,7 +366,7 @@ dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
         }
 
         if (ret != 0) {
-                layout->list[i].err = -1;
+                layout->list[i].err = 0;
                 gf_log (this->name, GF_LOG_TRACE,
                         "missing disk layout on %s. err = %d",
                         subvol->name, err);
@@ -530,23 +530,29 @@ dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
         prev_stop = last_stop;
 
         for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err) {
-                        switch (layout->list[i].err) {
-                        case -1:
-                        case ENOENT:
-                                missing++;
-                                break;
-                        case ENOTCONN:
-                                down++;
-                                break;
-                        case ENOSPC:
-                                no_space++;
-                                break;
-                        default:
-                                misc++;
+                switch (layout->list[i].err) {
+                case -1:
+                case ENOENT:
+                        missing++;
+                        break;
+                case ENOTCONN:
+                        down++;
+                        break;
+                case ENOSPC:
+                        no_space++;
+                        break;
+                case 0:
+                        /* if err == 0 and start == stop, then it is a non misc++;
+                         * participating subvolume(spread-cnt). Then, do not
+                         * check for anomalies. If start != stop, then treat it
+                         * as misc err */
+                        if (layout->list[i].start == layout->list[i].stop) {
+                                continue;
                         }
-                        continue;
-                }
+                        break;
+                default:
+                        misc++;
+                 }
 
                 is_virgin = 0;
 
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index e1e962ab92e..037b26fedb4 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -35,6 +35,7 @@
                         layout->list[cnt].stop  = 0;                    \
                 }                                                       \
         } while (0)
+
 static uint32_t
 dht_overlap_calc (dht_layout_t *old, int o, dht_layout_t *new, int n)
 {
@@ -239,6 +240,67 @@ out:
 }
 
 int
+dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+{
+        dht_local_t *local = NULL;
+        int          missing_xattr = 0;
+        int          i = 0;
+        xlator_t    *this = NULL;
+        dht_conf_t   *conf = NULL;
+        dht_layout_t *dummy = NULL;
+
+        local = frame->local;
+        this = frame->this;
+        conf = this->private;
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err != -1 || !layout->list[i].stop) {
+                        /* err != -1 would mean xattr present on the directory
+                         * or the directory is non existent.
+                         * !layout->list[i].stop would mean layout absent
+                         */
+
+                        continue;
+                }
+                missing_xattr++;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "%d subvolumes missing xattr for %s",
+                missing_xattr, loc->path);
+
+        if (missing_xattr == 0) {
+                dht_selfheal_dir_finish (frame, this, 0);
+                return 0;
+        }
+
+        local->call_cnt = missing_xattr;
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err != -1 || !layout->list[i].stop)
+                        continue;
+
+                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
+
+                if (--missing_xattr == 0)
+                        break;
+        }
+        dummy = dht_layout_new (this, 1);
+        if (!dummy)
+                goto out;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
+                                                          conf->subvolumes[i]);
+                }
+        }
+        dht_layout_unref (this, dummy);
+out:
+        return 0;
+}
+
+int
 dht_selfheal_dir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                               int op_ret, int op_errno, struct iatt *statpre,
                               struct iatt *statpost, dict_t *xdata)
@@ -253,7 +315,7 @@ dht_selfheal_dir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         this_call_cnt = dht_frame_return (frame);
 
         if (is_last_call (this_call_cnt)) {
-                dht_fix_dir_xattr (frame, &local->loc, layout);
+                dht_selfheal_dir_xattr (frame, &local->loc, layout);
         }
 
         return 0;
@@ -278,7 +340,7 @@ dht_selfheal_dir_setattr (call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
         }
 
         if (missing_attr == 0) {
-                dht_fix_dir_xattr (frame, loc, layout);
+                dht_selfheal_dir_xattr (frame, loc, layout);
                 return 0;
         }
 
@@ -782,7 +844,7 @@ dht_selfheal_new_directory (call_frame_t *frame,
 
         dht_layout_sort_volname (layout);
         dht_selfheal_layout_new_directory (frame, &local->loc, layout);
-        dht_fix_dir_xattr (frame, &local->loc, layout);
+        dht_selfheal_dir_xattr (frame, &local->loc, layout);
         return 0;
 }