tests: provide an option to mark tests as 'flaky'

* also add some time gap in other tests to see if we get things properly
* create a directory 'tests/000/', which can host any tests, which are flaky.
* move all the tests mentioned in the issue to above directory.
* as the above dir gets tested first, all flaky tests would be reported quickly.
* change `run-tests.sh` to continue tests even if flaky tests fail.

Reference: gluster/project-infrastructure#72
Updates: #1000
Change-Id: Ifdafa38d083ebd80f7ae3cbbc9aa3b68b6d21d0e
Signed-off-by: Amar Tumballi <amar@kadalu.io>

1 files changed, 22 insertions, 5 deletions

diff --git a/run-tests.sh b/run-tests.sh
index 6d48ecfcb03..e2a1655d8e0 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -328,6 +328,7 @@ function get_bug_list_for_disabled_test ()
 function run_tests()
 {
     RES=0
+    FLAKY=''
     FAILED=''
     TESTS_NEEDED_RETRY=''
     GENERATED_CORE=''
@@ -349,7 +350,7 @@ function run_tests()
         timeout_cmd_exists="no"
     fi
 
-    all_tests=($(find ${regression_testsdir}/tests -name '*.t' | LC_COLLATE=C sort))
+    all_tests=($(find ${regression_testsdir}/tests -name '*.t' | sort))
     all_tests_cnt=${#all_tests[@]}
     for t in "${all_tests[@]}" ; do
         old_cores=$(ls /*-*.core 2> /dev/null | wc -l)
@@ -433,9 +434,17 @@ function run_tests()
 
                 TESTS_NEEDED_RETRY="${TESTS_NEEDED_RETRY}${t} "
             fi
+
+
             if [ ${TMP_RES} -ne 0 ] ; then
-                RES=${TMP_RES}
-                FAILED="${FAILED}${t} "
+		if [[ "$t" == *"tests/000-flaky/"* ]]; then
+                    FLAKY="${FLAKY}${t} "
+		    echo "FAILURE -> SUCCESS: Flaky test"
+		    TMP_RES=0
+		else
+                    RES=${TMP_RES}
+                    FAILED="${FAILED}${t} "
+		fi
             fi
 
             new_cores=$(ls /*-*.core 2> /dev/null | wc -l)
@@ -470,8 +479,10 @@ function run_tests()
         echo "$key  -  ${ELAPSEDTIMEMAP["$key"]} second"
     done | sort -rn -k3
 
-    # Output the errors into a file
+    # initialize the output file
     echo > "${result_output}"
+
+    # Output the errors into a file
     if [ ${RES} -ne 0 ] ; then
         FAILED=$( echo ${FAILED} | tr ' ' '\n' | sort -u )
         FAILED_COUNT=$( echo -n "${FAILED}" | grep -c '^' )
@@ -484,7 +495,13 @@ function run_tests()
     TESTS_NEEDED_RETRY=$( echo ${TESTS_NEEDED_RETRY} | tr ' ' '\n' | sort -u )
     RETRY_COUNT=$( echo -n "${TESTS_NEEDED_RETRY}" | grep -c '^' )
     if [ ${RETRY_COUNT} -ne 0 ] ; then
-        echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}"
+        echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}" >> "${result_output}"
+    fi
+
+    FLAKY_TESTS_FAILED=$( echo ${FLAKY} | tr ' ' '\n' | sort -u )
+    RETRY_COUNT=$( echo -n "${FLAKY_TESTS_FAILED}" | grep -c '^' )
+    if [ ${RETRY_COUNT} -ne 0 ] ; then
+        echo -e "\n${RETRY_COUNT} flaky test(s) marked as success even though they failed \n${FLAKY_TESTS_FAILED}" >> "${result_output}"
     fi
 
     echo