summaryrefslogtreecommitdiffstats
path: root/run-tests.sh
diff options
context:
space:
mode:
Diffstat (limited to 'run-tests.sh')
-rwxr-xr-xrun-tests.sh257
1 files changed, 237 insertions, 20 deletions
diff --git a/run-tests.sh b/run-tests.sh
index 1487f30d832..e2a1655d8e0 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -2,18 +2,106 @@
# Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
#
+# As many tests are designed to take values of variables from 'env.rc',
+# it is good to source the file. While it is also required to source the
+# file individually in each tests (as it should be possible to run the
+# tests separately), exporting variables from env.rc is not harmful if
+# done here
+
+source ./tests/env.rc
+
export TZ=UTC
force="no"
head="yes"
-retry="no"
+retry="yes"
tests=""
exit_on_failure="yes"
skip_bad_tests="yes"
skip_known_bugs="yes"
+result_output="/tmp/gluster_regression.txt"
section_separator="========================================"
+run_timeout=200
+kill_after_time=5
+nfs_tests=$RUN_NFS_TESTS
+
+# Option below preserves log tarballs for each run of a test separately
+# named: <test>-iteration-<n>.tar
+# If set to any other value, then log tarball is just named after the test and
+# overwritten in each iteration (saves space)
+# named: <test>.tar
+# Use option -p to override default behavior
+skip_preserve_logs="yes"
OSTYPE=$(uname -s)
+# Function for use in generating filenames with increasing "-<n>" index
+# In:
+# $1 basepath: Directory where file needs to be created
+# $2 filename: Name of the file sans extension
+# $3 extension: Extension string that would be appended to the generated
+# filename
+# Out:
+# string of next available filename with appended "-<n>"
+# Example:
+# Interested routines that want to create a file name, say foo-<n>.txt at
+# location /var/log/gluster would pass in "/var/log/gluster" "foo" "txt"
+# and be returned next available foo-<n> filename to create.
+# Notes:
+# Function will not accept empty extension, and will return the same name
+# over and over (which can be fixed when there is a need for it)
+function get_next_filename()
+{
+ local basepath=$1
+ local filename=$2
+ local extension=$3
+ local next=1
+ local tfilename="${filename}-${next}"
+ while [ -e "${basepath}/${tfilename}.${extension}" ]; do
+ next=$((next+1))
+ tfilename="${filename}-${next}"
+ done
+
+ echo "$tfilename"
+}
+
+# Tar the gluster logs and generate a tarball named after the first parameter
+# passed in to the function. Ideally the test name is passed to this function
+# to generate the required tarball.
+# Tarball name is further controlled by the variable skip_preserve_logs
+function tar_logs()
+{
+ t=$1
+
+ logdir=$(gluster --print-logdir)
+ basetarname=$(basename "$t" .t)
+
+ if [ -n "$logdir" ]
+ then
+ if [[ $skip_preserve_logs == "yes" ]]; then
+ savetarname=$(get_next_filename "${logdir}" \
+ "${basetarname}-iteration" "tar" \
+ | tail -1)
+ else
+ savetarname="$basetarname"
+ fi
+
+ # Can't use --exclude here because NetBSD doesn't have it.
+ # However, both it and Linux have -X to take patterns from
+ # a file, so use that.
+ (echo '*.tar'; echo .notar) > "${logdir}"/.notar \
+ && \
+ tar -cf "${logdir}"/"${savetarname}".tar -X "${logdir}"/.notar \
+ "${logdir}"/* 2> /dev/null \
+ && \
+ find "$logdir"/* -maxdepth 0 -name '*.tar' -prune \
+ -o -exec rm -rf '{}' ';'
+
+ echo "Logs preserved in tarball $savetarname.tar"
+ else
+ echo "Logs not preserved, as logdir is not set"
+ fi
+}
+
function check_dependencies()
{
## Check all dependencies are present
@@ -39,6 +127,12 @@ function check_dependencies()
fi
fi
+ # Check for netstat
+ env netstat --version > /dev/null 2>&1
+ if [ $? -ne 0 ]; then
+ MISSING="$MISSING netstat"
+ fi
+
# Check for the Perl Test Harness
env prove --version > /dev/null 2>&1
if [ $? -ne 0 ]; then
@@ -70,6 +164,12 @@ function check_dependencies()
MISSING="$MISSING pidof"
fi
+ # Check for netstat
+ env netstat --version > /dev/null 2>&1
+ if [ $? -ne 0 ]; then
+ MISSING="$MISSING netstat"
+ fi
+
# check for psutil python package
test `uname -s` == "Darwin" || test `uname -s` == "FreeBSD" && {
pip show psutil | grep -q psutil >/dev/null 2>&1
@@ -165,8 +265,10 @@ function match()
# G_TESTDEF_TEST_STATUS_NETBSD7
# Some examples:
# G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456
+# G_TESTDEF_TEST_STATUS_CENTOS6=BRICK_MUX_BAD_TEST,BUG=123456
# G_TESTDEF_TEST_STATUS_NETBSD7=KNOWN_ISSUE,BUG=4444444
# G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456;555555
+# G_TESTDEF_TEST_STATUS_CENTOS6=NFS_TESTS,BUG=1385758
# You can change status of test to enabled or delete the line only if all the
# bugs are closed or modified or if the patch fixes it.
function get_test_status ()
@@ -226,7 +328,9 @@ function get_bug_list_for_disabled_test ()
function run_tests()
{
RES=0
+ FLAKY=''
FAILED=''
+ TESTS_NEEDED_RETRY=''
GENERATED_CORE=''
total_tests=0
selected_tests=0
@@ -237,15 +341,25 @@ function run_tests()
# key = path of .t file; value = time taken to run the .t file
declare -A ELAPSEDTIMEMAP
- for t in $(find ${regression_testsdir}/tests -name '*.t' \
- | LC_COLLATE=C sort) ; do
+ # Test if -k is supported for timeout command
+ # This is not supported on centos6, but spuported on centos7
+ # The flags is required for running the command in both flavors
+ timeout_cmd_exists="yes"
+ timeout -k 1 10 echo "testing 'timeout' command"
+ if [ $? -ne 0 ]; then
+ timeout_cmd_exists="no"
+ fi
+
+ all_tests=($(find ${regression_testsdir}/tests -name '*.t' | sort))
+ all_tests_cnt=${#all_tests[@]}
+ for t in "${all_tests[@]}" ; do
old_cores=$(ls /*-*.core 2> /dev/null | wc -l)
total_tests=$((total_tests+1))
if match $t "$@" ; then
selected_tests=$((selected_tests+1))
echo
- echo $section_separator$section_separator
- if [[ $(get_test_status $t) == "BAD_TEST" ]] && \
+ echo $section_separator "(${total_tests} / ${all_tests_cnt})" $section_separator
+ if [[ $(get_test_status $t) =~ "BAD_TEST" ]] && \
[[ $skip_bad_tests == "yes" ]]
then
skipped_bad_tests=$((skipped_bad_tests+1))
@@ -265,28 +379,74 @@ function run_tests()
echo
continue
fi
+ if [[ $(get_test_status $t) == "NFS_TEST" ]] && \
+ [[ $nfs_tests == "no" ]]
+ then
+ echo "Skipping nfs test file $t"
+ echo $section_separator$section_separator
+ echo
+ continue
+ fi
total_run_tests=$((total_run_tests+1))
echo "[$(date +%H:%M:%S)] Running tests in file $t"
starttime="$(date +%s)"
- prove -vf $t
+
+ local cmd_timeout=$run_timeout;
+ if [ ${timeout_cmd_exists} == "yes" ]; then
+ if [ $(grep -c "SCRIPT_TIMEOUT=" ${t}) == 1 ] ; then
+ cmd_timeout=$(grep "SCRIPT_TIMEOUT=" ${t} | cut -f2 -d'=');
+ echo "Timeout set is ${cmd_timeout}, default ${run_timeout}"
+ fi
+ timeout --foreground -k ${kill_after_time} ${cmd_timeout} prove -vmfe '/bin/bash' ${t}
+ else
+ prove -vmfe '/bin/bash' ${t}
+ fi
TMP_RES=$?
ELAPSEDTIMEMAP[$t]=`expr $(date +%s) - $starttime`
+ tar_logs "$t"
+
+ # timeout always return 124 if it is actually a timeout.
+ if ((${TMP_RES} == 124)); then
+ echo "${t} timed out after ${cmd_timeout} seconds"
+ fi
+
if [ ${TMP_RES} -ne 0 ] && [ "x${retry}" = "xyes" ] ; then
echo "$t: bad status $TMP_RES"
echo ""
echo " *********************************"
echo " * REGRESSION FAILED *"
echo " * Retrying failed tests in case *"
- echo " * we got some spurous failures *"
+ echo " * we got some spurious failures *"
echo " *********************************"
echo ""
- prove -vf $t
+
+ if [ ${timeout_cmd_exists} == "yes" ]; then
+ timeout --foreground -k ${kill_after_time} ${cmd_timeout} prove -vmfe '/bin/bash' ${t}
+ else
+ prove -vmfe '/bin/bash' ${t}
+ fi
TMP_RES=$?
+ tar_logs "$t"
+
+ if ((${TMP_RES} == 124)); then
+ echo "${t} timed out after ${cmd_timeout} seconds"
+ fi
+
+ TESTS_NEEDED_RETRY="${TESTS_NEEDED_RETRY}${t} "
fi
+
+
if [ ${TMP_RES} -ne 0 ] ; then
- RES=${TMP_RES}
- FAILED="${FAILED}${t} "
+ if [[ "$t" == *"tests/000-flaky/"* ]]; then
+ FLAKY="${FLAKY}${t} "
+ echo "FAILURE -> SUCCESS: Flaky test"
+ TMP_RES=0
+ else
+ RES=${TMP_RES}
+ FAILED="${FAILED}${t} "
+ fi
fi
+
new_cores=$(ls /*-*.core 2> /dev/null | wc -l)
if [ x"$new_cores" != x"$old_cores" ]; then
core_diff=$((new_cores-old_cores))
@@ -310,14 +470,6 @@ function run_tests()
echo "Number of tests skipped as they were marked bad: $skipped_bad_tests"
echo "Number of tests skipped because of known_issues: $skipped_known_issue_tests"
echo "Number of tests that were run: $total_run_tests"
- if [ ${RES} -ne 0 ] ; then
- FAILED=$( echo ${FAILED} | tr ' ' '\n' | sort -u )
- FAILED_COUNT=$( echo -n "${FAILED}" | grep -c '^' )
- echo -e "\n$FAILED_COUNT test(s) failed \n${FAILED}"
- GENERATED_CORE=$( echo ${GENERATED_CORE} | tr ' ' '\n' | sort -u )
- GENERATED_CORE_COUNT=$( echo -n "${GENERATED_CORE}" | grep -c '^' )
- echo -e "\n$GENERATED_CORE_COUNT test(s) generated core \n${GENERATED_CORE}"
- fi
echo
echo "Tests ordered by time taken, slowest to fastest: "
@@ -327,6 +479,31 @@ function run_tests()
echo "$key - ${ELAPSEDTIMEMAP["$key"]} second"
done | sort -rn -k3
+ # initialize the output file
+ echo > "${result_output}"
+
+ # Output the errors into a file
+ if [ ${RES} -ne 0 ] ; then
+ FAILED=$( echo ${FAILED} | tr ' ' '\n' | sort -u )
+ FAILED_COUNT=$( echo -n "${FAILED}" | grep -c '^' )
+ echo -e "\n$FAILED_COUNT test(s) failed \n${FAILED}" >> "${result_output}"
+ GENERATED_CORE=$( echo ${GENERATED_CORE} | tr ' ' '\n' | sort -u )
+ GENERATED_CORE_COUNT=$( echo -n "${GENERATED_CORE}" | grep -c '^' )
+ echo -e "\n$GENERATED_CORE_COUNT test(s) generated core \n${GENERATED_CORE}" >> "${result_output}"
+ cat "${result_output}"
+ fi
+ TESTS_NEEDED_RETRY=$( echo ${TESTS_NEEDED_RETRY} | tr ' ' '\n' | sort -u )
+ RETRY_COUNT=$( echo -n "${TESTS_NEEDED_RETRY}" | grep -c '^' )
+ if [ ${RETRY_COUNT} -ne 0 ] ; then
+ echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}" >> "${result_output}"
+ fi
+
+ FLAKY_TESTS_FAILED=$( echo ${FLAKY} | tr ' ' '\n' | sort -u )
+ RETRY_COUNT=$( echo -n "${FLAKY_TESTS_FAILED}" | grep -c '^' )
+ if [ ${RETRY_COUNT} -ne 0 ] ; then
+ echo -e "\n${RETRY_COUNT} flaky test(s) marked as success even though they failed \n${FLAKY_TESTS_FAILED}" >> "${result_output}"
+ fi
+
echo
echo "Result is $RES"
echo
@@ -353,8 +530,38 @@ function run_head_tests()
run_tests "$htests"
}
-function parse_args () {
- args=`getopt frcbkhH "$@"`
+function show_usage ()
+{
+ cat <<EOF
+Usage: $0 <opts> [<glob>|<bzid>]...
+
+Options:
+
+-f force
+-h skip tests altering from HEAD
+-H run only tests altering from HEAD
+-r retry failed tests
+-R do not retry failed tests
+-c dont't exit on failure
+-b don't skip bad tests
+-k don't skip known bugs
+-p don't keep logs from preceding runs
+-o OUTPUT
+-t TIMEOUT
+-n skip NFS tests
+--help
+EOF
+}
+
+usage="no"
+
+function parse_args ()
+{
+ args=`getopt -u -l help frRcbkphHno:t: "$@"`
+ if ! [ $? -eq 0 ]; then
+ show_usage
+ exit 1
+ fi
set -- $args
while [ $# -gt 0 ]; do
case "$1" in
@@ -362,9 +569,15 @@ function parse_args () {
-h) head="no" ;;
-H) head="only" ;;
-r) retry="yes" ;;
+ -R) retry="no" ;;
-c) exit_on_failure="no" ;;
-b) skip_bad_tests="no" ;;
-k) skip_known_bugs="no" ;;
+ -p) skip_preserve_logs="no" ;;
+ -o) result_output="$2"; shift;;
+ -t) run_timeout="$2"; shift;;
+ -n) nfs_tests="no";;
+ --help) usage="yes" ;;
--) shift; break;;
esac
shift
@@ -379,6 +592,10 @@ echo
# Get user options
parse_args "$@"
+if [ x"$usage" == x"yes" ]; then
+ show_usage
+ exit 0
+fi
# Make sure we're running as the root user
check_user
generated by cgit (git 2.34.1) at 2025-12-28 18:47:33 +0000