Auto-retry failed tests to workaround spurious failures

Add a -r flag to run-tests.sh that will cause failed tests to be
retried once. The idea is to reduce the impact of spurious failures:
no need to retrigger in jenkins, and no need to replay all the tests.

Backport of:
  I176bb32678835a669992fbac7de4dd14ee5c62bc
  If4e3c96d51037fbcf6d9d0e10a3dd3e12da77447

Resubmit since regression did not catch that one.

BUG: 1212676
Change-Id: I31a1d71118bc9295d5c858dbf1e664d0f2aed6ff
Signed-off-by: Emmanuel Dreyfus <manu@netbsd.org>
Reviewed-on: http://review.gluster.org/10302
Tested-by: NetBSD Build System
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 41 insertions, 3 deletions

diff --git a/run-tests.sh b/run-tests.sh
index 32b6e8e1d0b..527d4abeedb 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -181,6 +181,7 @@ function run_tests()
         fi
     done
     if [ ${RES} -ne 0 ] ; then
+        FAILED=$( echo ${FAILED} | tr ' ' '\n' | sort -u )
         echo "Failed tests ${FAILED}"
     fi
     return ${RES}
@@ -240,12 +241,45 @@ function main()
     fi
 }
 
+function main_and_retry()
+{
+    RESFILE=`mktemp /tmp/${0##*/}.XXXXXX` || exit 1
+    main "$@" | tee ${RESFILE}
+    RET=$?
+
+    FAILED=$( awk '/Failed: /{print $1}' ${RESFILE} )
+    if [ "x${FAILED}" != "x" ] ; then
+       echo ""
+       echo "       *********************************"
+       echo "       *       REGRESSION FAILED       *"
+       echo "       * Retrying failed tests in case *"
+       echo "       * we got some spurous failures  *"
+       echo "       *********************************"
+       echo ""
+       main ${FAILED}
+       RET=$?
+    fi
+
+    rm -f ${RESFILE}
+    return ${RET}
+}
+
 echo
 echo ... GlusterFS Test Framework ...
 echo
 
-force=no
-test "x$1" = "x-f" && { force="yes"; shift; }
+force="no"
+retry="no"
+args=`getopt fr $*`
+set -- $args
+while [ $# -gt 0 ]; do
+    case "$1" in
+    -f)    force="yes" ;;
+    -r)    retry="yes" ;;
+    --)    shift; break;;
+    esac
+    shift
+done
 
 # Make sure we're running as the root user
 check_user
@@ -257,4 +291,8 @@ check_dependencies
 check_location
 
 # Run the tests
-main "$@"
+if [ "x${retry}" = "xyes" ] ; then
+    main_and_retry $@
+else
+    main "$@"
+fi