tests: ignore results from some spuriously-failing tests

These tests were selected based on a survey of the last 50 apparently
spurious regression failures involving only one test, as of April 21, 2015.
They were responsible for the following number of failures out of those 50.

  volume-snapshot.t: 24
  uss.t:              8
  glupy.t:            5

In other words, they were responsible for 74% of those failures.  Until
they're fixed (or we find some generic problem to which they're particularly
vulnerable for some reason), ignoring their results should significantly
improve our regression-test success rate.

(As part of the rebase from master, it made more sense to bring along the
early-termination code in run_all than to attempt surgery on it.)

BUG: 1163543
Change-Id: I2727735cd57174618c56a9cdcb4bd4bf58301f05
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-on: http://review.gluster.org/10351
Reviewed-by: Vijay Bellur <vbellur@redhat.com>
Tested-by: NetBSD Build System
Tested-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 46 insertions, 2 deletions

diff --git a/run-tests.sh b/run-tests.sh
index d6de6c6c31a..32b6e8e1d0b 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -186,11 +186,55 @@ function run_tests()
     return ${RES}
 }
 
+# If you're submitting a fix related to one of these tests and want its result
+# to be considered, you'll need to remove it from the list as part of your
+# patch.
+function is_bad_test ()
+{
+    local name=$1
+    for bt in ./tests/basic/volume-snapshot-clone.t \
+              ./tests/basic/uss.t                   \
+              ./tests/features/glupy.t; do
+        [ x"$name" = x"$bt" ] && return 0 # bash: zero means true/success
+    done
+    return 1                              # bash: non-zero means false/failure
+}
+
+function run_all ()
+{
+    find ${regression_testsdir}/tests -name '*.t' \
+    | LC_COLLATE=C sort \
+    | while read t; do
+        old_cores=$(ls /core.* 2> /dev/null | wc -l)
+        retval=0
+        prove -f --timer $t
+        TMP_RES=$?
+        if [ ${TMP_RES} -ne 0 ] ; then
+            echo "$t: bad status $TMP_RES"
+            retval=$((retval+1))
+        fi
+        new_cores=$(ls /core.* 2> /dev/null | wc -l)
+        if [ x"$new_cores" != x"$old_cores" ]; then
+            core_diff=$((new_cores-old_cores))
+            echo "$t: $core_diff new core files"
+            retval=$((retval+2))
+        fi
+        if [ $retval -ne 0 ]; then
+            if is_bad_test $t; then
+                echo  "Ignoring failure from known-bad test $t"
+            else
+                return $retval
+            fi
+        fi
+    done
+}
+
 function main()
 {
     if [ $# -lt 1 ]; then
-        echo "Running all the regression test cases"
-        prove -rf --timer ${regression_testsdir}/tests;
+        echo "Running all the regression test cases (new way)"
+        #prove -rf --timer ${regression_testsdir}/tests;
+        run_all
     else
         run_tests "$@"
     fi