tests: Add mechanism for disabled tests

Requirements:
Should be able to skip tests from run-tests.sh run.
Should be granular enough to disable on subset of OSes.

Solution:
Tests can have special comment lines with some comma separated values
within them.

Key names used to determine test status are
G_TESTDEF_TEST_STATUS_CENTOS6
G_TESTDEF_TEST_STATUS_NETBSD7
Some examples:
G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456
G_TESTDEF_TEST_STATUS_NETBSD7=KNOWN_ISSUE,BUG=4444444
G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456;555555
You can change status of test to enabled or delete the line only if all the
bugs are closed or modified or if the patch fixes it.

Change-Id: Idee21fecaa5837fd4bd06e613f5c07a024f7b0c2
BUG: 1295704
Signed-off-by: Raghavendra Talur <rtalur@redhat.com>
Reviewed-on: http://review.gluster.org/13393
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
Reviewed-by: Kaushal M <kaushal@redhat.com>

1 files changed, 104 insertions, 51 deletions

diff --git a/run-tests.sh b/run-tests.sh
index 18dc3bc35e5..c681f7d727e 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -7,6 +7,9 @@ force="no"
 retry="no"
 tests=""
 exit_on_failure="yes"
+skip_bad_tests="yes"
+skip_known_bugs="yes"
+section_separator="========================================"
 
 OSTYPE=$(uname -s)
 
@@ -155,46 +158,68 @@ function match()
         return $match
 }
 
-# If you're submitting a fix related to one of these tests and want its result
-# to be considered, you'll need to remove it from the list as part of your
-# patch.
-function is_bad_test ()
+# Tests can have comment lines with some comma separated values within them.
+# Key names used to determine test status are
+# G_TESTDEF_TEST_STATUS_CENTOS6
+# G_TESTDEF_TEST_STATUS_NETBSD7
+# Some examples:
+# G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456
+# G_TESTDEF_TEST_STATUS_NETBSD7=KNOWN_ISSUE,BUG=4444444
+# G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456;555555
+# You can change status of test to enabled or delete the line only if all the
+# bugs are closed or modified or if the patch fixes it.
+function get_test_status ()
 {
-    local name=$1
-    for bt in ./tests/basic/quota-anon-fd-nfs.t \
-              ./tests/bugs/quota/bug-1235182.t \
-              ./tests/basic/quota-nfs.t \
-              ./tests/basic/tier/tier_lookup_heal.t \
-              ./tests/basic/tier/bug-1214222-directories_missing_after_attach_tier.t \
-              ./tests/basic/tier/fops-during-migration.t \
-              ./tests/basic/tier/record-metadata-heat.t \
-              ./tests/basic/tier/tier-snapshot.t \
-              ./tests/bugs/snapshot/bug-1109889.t \
-              ./tests/bugs/distribute/bug-1066798.t \
-              ./tests/bugs/glusterd/bug-1238706-daemons-stop-on-peer-cleanup.t \
-              ./tests/geo-rep/georep-basic-dr-rsync.t \
-              ./tests/geo-rep/georep-basic-dr-tarssh.t \
-              ./tests/bugs/fuse/bug-924726.t \
-              ./tests/basic/afr/split-brain-healing.t \
-              ./tests/bugs/snapshot/bug-1140162-file-snapshot-features-encrypt-opts-validation.t \
-              ./tests/bugs/tier/bug-1286974.t \
-              ./tests/features/weighted-rebalance.t \
-              ./tests/performance/open-behind.t \
-              ./tests/basic/afr/self-heald.t \
-              ./tests/basic/tier/tier-file-create.t \
-              ; do
-        [ x"$name" = x"$bt" ] && return 0 # bash: zero means true/success
-    done
-    return 1                              # bash: non-zero means false/failure
+    local test_name=$1
+    local host_os=""
+    local result=""
+
+    host_os=$(uname -s)
+
+    case "$host_os" in
+    # Leaving out the logic to determine the particular distro and version
+    # for later. Why does the key have the distro and version then?
+    # Because changing the key in all test files would be very big process
+    # updating just this function with a better logic much simpler.
+    Linux)
+        result=$(grep -e "^#G_TESTDEF_TEST_STATUS_CENTOS6" $test_name | \
+                 awk -F"," {'print $1'} | awk -F"=" {'print $2'}) ;;
+    NetBSD)
+        result=$(grep -e "^#G_TESTDEF_TEST_STATUS_NETBSD7" $test_name | \
+                 awk -F"," {'print $1'} | awk -F"=" {'print $2'}) ;;
+    *)
+        result="ENABLED" ;;
+    esac
+
+    echo "$result"
+
 }
 
-function is_unsupported_test()
+function get_bug_list_for_disabled_test ()
 {
-        if [ x"$OSTYPE" != x"NetBSD" ]; then
-                return 1
-        fi
+    local test_name=$1
+    local host_os=""
+    local result=""
+
+    host_os=$(uname -s)
+
+    case "$host_os" in
+    # Leaving out the logic to determine the particular distro and version
+    # for later. Why does the key have the distro and version then?
+    # Because changing the key in all test files would be very big process
+    # updating just this function with a better logic much simpler.
+    Linux)
+        result=$(grep -e "^#G_TESTDEF_TEST_STATUS_CENTOS6" $test_name | \
+                 awk -F"," {'print $2'} | awk -F"=" {'print $2'}) ;;
+    NetBSD)
+        result=$(grep -e "^#G_TESTDEF_TEST_STATUS_NETBSD7" $test_name | \
+                 awk -F"," {'print $2'} | awk -F"=" {'print $2'}) ;;
+    *)
+        result="0000000" ;;
+    esac
+
+    echo "$result"
 
-        grep -iqs tier $1
 }
 
 function run_tests()
@@ -202,6 +227,11 @@ function run_tests()
     RES=0
     FAILED=''
     GENERATED_CORE=''
+    total_tests=0
+    selected_tests=0
+    skipped_bad_tests=0
+    skipped_known_issue_tests=0
+    total_run_tests=0
 
     # key = path of .t file; value = time taken to run the .t file
     declare -A ELAPSEDTIMEMAP
@@ -209,24 +239,35 @@ function run_tests()
     for t in $(find ${regression_testsdir}/tests -name '*.t' \
                | LC_COLLATE=C sort) ; do
         old_cores=$(ls /core.* 2> /dev/null | wc -l)
+        total_tests=$((total_tests+1))
         if match $t "$@" ; then
+            selected_tests=$((selected_tests+1))
             echo
-            echo "=================================================="
-            if is_bad_test $t; then
+            echo $section_separator$section_separator
+            if [[ $(get_test_status $t) == "BAD_TEST" ]] && \
+               [[ $skip_bad_tests == "yes" ]]
+            then
+                skipped_bad_tests=$((skipped_bad_tests+1))
                 echo "Skipping bad test file $t"
-                echo "=================================================="
+                echo "Reason: bug(s):" $(get_bug_list_for_disabled_test $t)
+                echo $section_separator$section_separator
                 echo
                 continue
             fi
-            if is_unsupported_test $t; then
-                echo "Skipping test file $t (feature unsupported on platform)"
-                echo "=================================================="
+            if [[ $(get_test_status $t) == "KNOWN_ISSUE" ]] && \
+               [[ $skip_known_bugs == "yes" ]]
+            then
+                skipped_known_issue_tests=$((skipped_known_issue_tests+1))
+                echo "Skipping test file $t due to known issue"
+                echo "Reason: bug(s):" $(get_bug_list_for_disabled_test $t)
+                echo $section_separator$section_separator
                 echo
                 continue
             fi
-            echo "Running tests in file $t"
+            total_run_tests=$((total_run_tests+1))
+            echo "[$(date +%H:%M:%S)] Running tests in file $t"
             starttime="$(date +%s)"
-            prove -mf --timer $t
+            prove -vf $t
             TMP_RES=$?
             ELAPSEDTIMEMAP[$t]=`expr $(date +%s) - $starttime`
             if [ ${TMP_RES} -ne 0 ]  && [ "x${retry}" = "xyes" ] ; then
@@ -238,7 +279,7 @@ function run_tests()
                 echo "       * we got some spurous failures  *"
                 echo "       *********************************"
                 echo ""
-                prove -mf --timer $t
+                prove -vf $t
                 TMP_RES=$?
             fi
             if [ ${TMP_RES} -ne 0 ] ; then
@@ -253,7 +294,7 @@ function run_tests()
                 GENERATED_CORE="${GENERATED_CORE}${t} "
             fi
             echo "End of test $t"
-            echo "=================================================="
+            echo $section_separator$section_separator
             echo
             if [ $RES -ne 0 ] && [ x"$exit_on_failure" = "xyes" ] ; then
                 break;
@@ -262,33 +303,45 @@ function run_tests()
     done
     echo
     echo "Run complete"
+    echo $section_separator$section_separator
+    echo "Number of tests found:                             $total_tests"
+    echo "Number of tests selected for run based on pattern: $selected_tests"
+    echo "Number of tests skipped as they were marked bad:   $skipped_bad_tests"
+    echo "Number of tests skipped because of known_issues:   $skipped_known_issue_tests"
+    echo "Number of tests that were run:                     $total_run_tests"
     if [ ${RES} -ne 0 ] ; then
         FAILED=$( echo ${FAILED} | tr ' ' '\n' | sort -u )
         FAILED_COUNT=$( echo -n "${FAILED}" | grep -c '^' )
-        echo -e "$FAILED_COUNT test(s) failed \n${FAILED}"
+        echo -e "\n$FAILED_COUNT test(s) failed \n${FAILED}"
         GENERATED_CORE=$( echo  ${GENERATED_CORE} | tr ' ' '\n' | sort -u )
         GENERATED_CORE_COUNT=$( echo -n "${GENERATED_CORE}" | grep -c '^' )
-        echo -e "$GENERATED_CORE_COUNT test(s) generated core \n${GENERATED_CORE}"
+        echo -e "\n$GENERATED_CORE_COUNT test(s) generated core \n${GENERATED_CORE}"
     fi
 
-    echo "Slowest 10 tests: "
+    echo
+    echo "Tests ordered by time taken, slowest to fastest: "
+    echo $section_separator$section_separator
     for key in "${!ELAPSEDTIMEMAP[@]}"
     do
-        echo $key ' - ' ${ELAPSEDTIMEMAP["$key"]}
-    done | sort -rn -k3 | head
+        echo "$key  -  ${ELAPSEDTIMEMAP["$key"]} second"
+    done | sort -rn -k3
 
+    echo
     echo "Result is $RES"
+    echo
     return ${RES}
 }
 
 function parse_args () {
-    args=`getopt frc "$@"`
+    args=`getopt frcbk "$@"`
     set -- $args
     while [ $# -gt 0 ]; do
         case "$1" in
         -f)    force="yes" ;;
         -r)    retry="yes" ;;
         -c)    exit_on_failure="no" ;;
+        -b)    skip_bad_tests="no" ;;
+        -k)    skip_known_bugs="no" ;;
         --)    shift; break;;
         esac
         shift