tests: provide an option to mark tests as 'flaky'

* also add some time gap in other tests to see if we get things properly
* create a directory 'tests/000/', which can host any tests, which are flaky.
* move all the tests mentioned in the issue to above directory.
* as the above dir gets tested first, all flaky tests would be reported quickly.
* change `run-tests.sh` to continue tests even if flaky tests fail.

Reference: gluster/project-infrastructure#72
Updates: #1000
Change-Id: Ifdafa38d083ebd80f7ae3cbbc9aa3b68b6d21d0e
Signed-off-by: Amar Tumballi <amar@kadalu.io>
(cherry picked from 097db13c11390174c5b9f11aa0fd87eca1735871)

1 files changed, 21 insertions, 4 deletions

diff --git a/run-tests.sh b/run-tests.sh
index 2768adb5afd..e763dfb8a31 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -328,6 +328,7 @@ function get_bug_list_for_disabled_test ()
 function run_tests()
 {
     RES=0
+    FLAKY=''
     FAILED=''
     TESTS_NEEDED_RETRY=''
     GENERATED_CORE=''
@@ -432,9 +433,17 @@ function run_tests()
 
                 TESTS_NEEDED_RETRY="${TESTS_NEEDED_RETRY}${t} "
             fi
+
+
             if [ ${TMP_RES} -ne 0 ] ; then
-                RES=${TMP_RES}
-                FAILED="${FAILED}${t} "
+		if [[ "$t" == *"tests/000-flaky/"* ]]; then
+                    FLAKY="${FLAKY}${t} "
+		    echo "FAILURE -> SUCCESS: Flaky test"
+		    TMP_RES=0
+		else
+                    RES=${TMP_RES}
+                    FAILED="${FAILED}${t} "
+		fi
             fi
 
             new_cores=$(ls /*-*.core 2> /dev/null | wc -l)
@@ -469,8 +478,10 @@ function run_tests()
         echo "$key  -  ${ELAPSEDTIMEMAP["$key"]} second"
     done | sort -rn -k3
 
-    # Output the errors into a file
+    # initialize the output file
     echo > "${result_output}"
+
+    # Output the errors into a file
     if [ ${RES} -ne 0 ] ; then
         FAILED=$( echo ${FAILED} | tr ' ' '\n' | sort -u )
         FAILED_COUNT=$( echo -n "${FAILED}" | grep -c '^' )
@@ -483,7 +494,13 @@ function run_tests()
     TESTS_NEEDED_RETRY=$( echo ${TESTS_NEEDED_RETRY} | tr ' ' '\n' | sort -u )
     RETRY_COUNT=$( echo -n "${TESTS_NEEDED_RETRY}" | grep -c '^' )
     if [ ${RETRY_COUNT} -ne 0 ] ; then
-        echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}"
+        echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}" >> "${result_output}"
+    fi
+
+    FLAKY_TESTS_FAILED=$( echo ${FLAKY} | tr ' ' '\n' | sort -u )
+    RETRY_COUNT=$( echo -n "${FLAKY_TESTS_FAILED}" | grep -c '^' )
+    if [ ${RETRY_COUNT} -ne 0 ] ; then
+        echo -e "\n${RETRY_COUNT} flaky test(s) marked as success even though they failed \n${FLAKY_TESTS_FAILED}" >> "${result_output}"
     fi
 
     echo