From dace4bbd0fdce567703a1a8d2d74a0a3f166f334 Mon Sep 17 00:00:00 2001 From: Kotresh HR Date: Thu, 28 May 2015 19:26:00 +0530 Subject: tests: New simple geo-rep regression test suite This is a new simple regression test suite for geo-replication. This is written keeping in mind the run time for regression test. The existing regression test suite is rigorous one and could be run nightly. Hence the existing geo-rep tests are being removed as part of this. Also re-enable geo-rep regression with this patch. Thanks Aravinda for initial template and plan. Change-Id: If544ac295eaf67ac66e0b071903cc1096e71d437 BUG: 1227624 Signed-off-by: Kotresh HR Reviewed-on: http://review.gluster.org/11058 Tested-by: Gluster Build System Tested-by: NetBSD Build System Reviewed-by: Aravinda VK --- run-tests.sh | 4 +- tests/geo-rep.rc | 183 ++++++++++++++++++++ tests/geo-rep/compare-arequal.py | 244 -------------------------- tests/geo-rep/compare-gfid.py | 95 ---------- tests/geo-rep/geo-rep-config.rc | 14 -- tests/geo-rep/geo-rep-helper.rc | 296 -------------------------------- tests/geo-rep/georep-basic-dr-rsync.t | 89 ++++++++++ tests/geo-rep/georep-basic-dr-tarssh.t | 94 ++++++++++ tests/geo-rep/georep-rsync-changelog.t | 73 -------- tests/geo-rep/georep-rsync-hybrid.t | 65 ------- tests/geo-rep/georep-setup.t | 33 ---- tests/geo-rep/georep-tarssh-changelog.t | 73 -------- tests/geo-rep/georep-tarssh-hybrid.t | 65 ------- tests/include.rc | 4 + 14 files changed, 371 insertions(+), 961 deletions(-) create mode 100644 tests/geo-rep.rc delete mode 100644 tests/geo-rep/compare-arequal.py delete mode 100644 tests/geo-rep/compare-gfid.py delete mode 100644 tests/geo-rep/geo-rep-config.rc delete mode 100644 tests/geo-rep/geo-rep-helper.rc create mode 100644 tests/geo-rep/georep-basic-dr-rsync.t create mode 100644 tests/geo-rep/georep-basic-dr-tarssh.t delete mode 100644 tests/geo-rep/georep-rsync-changelog.t delete mode 100644 tests/geo-rep/georep-rsync-hybrid.t delete mode 100644 tests/geo-rep/georep-setup.t delete mode 100644 tests/geo-rep/georep-tarssh-changelog.t delete mode 100644 tests/geo-rep/georep-tarssh-hybrid.t diff --git a/run-tests.sh b/run-tests.sh index 6763fa2f028..50fe080c2ac 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -164,8 +164,7 @@ function run_tests() return $match } RES=0 - for t in $(find ${regression_testsdir}/tests | grep -v geo-rep \ - | LC_COLLATE=C sort) ; do + for t in $(find ${regression_testsdir}/tests | LC_COLLATE=C sort) ; do if match $t "$@" ; then if [ -d $t ] ; then echo "Running tests in directory $t" @@ -208,7 +207,6 @@ function is_bad_test () function run_all () { find ${regression_testsdir}/tests -name '*.t' \ - | grep -v geo-rep \ | LC_COLLATE=C sort \ | while read t; do old_cores=$(ls /core.* 2> /dev/null | wc -l) diff --git a/tests/geo-rep.rc b/tests/geo-rep.rc new file mode 100644 index 00000000000..4ef0796184b --- /dev/null +++ b/tests/geo-rep.rc @@ -0,0 +1,183 @@ +function check_status() +{ + local search_key=$1 + $GEOREP_CLI $master $slave status detail | egrep -i "$search_key" +} + +function check_status_num_rows() +{ + local search_key=$1 + $GEOREP_CLI $master $slave status detail | egrep -i "$search_key" | wc -l +} + +function create_data() +{ + prefix=$1 + + # GF_FOP_MKNOD + # GF_FOP_MKDIR + # GF_FOP_UNLINK + # GF_FOP_RMDIR + # GF_FOP_SYMLINK + # GF_FOP_RENAME + # GF_FOP_LINK + # GF_FOP_SETXATTR + # GF_FOP_REMOVEXATTR + # GF_FOP_CREATE + # GF_FOP_SETATTR + + # Regular file + touch ${master_mnt}/${prefix}_f1 + touch ${master_mnt}/${prefix}_f2 + touch ${master_mnt}/${prefix}_f3 + + # dir + mkdir ${master_mnt}/${prefix}_d1 + mkdir ${master_mnt}/${prefix}_d2 + touch ${master_mnt}/${prefix}_d3 + + # Hardlink + ln ${master_mnt}/${prefix}_f1 ${master_mnt}/${prefix}_hl1 + + # Symlink + ln -s ${master_mnt}/${prefix}_f1 ${master_mnt}/${prefix}_sl1 + + # data + echo "Hello World!" >> ${master_mnt}/${prefix}_f1 + + # UNLINK + rm ${master_mnt}/${prefix}_f2 + + # RMDIR + rmdir ${master_mnt}/${prefix}_d2 + + # Rename - File + mv ${master_mnt}/${prefix}_f3 ${master_mnt}/${prefix}_f4 + + # Rename - Dir + mv ${master_mnt}/${prefix}_d3 ${master_mnt}/${prefix}_d4 +} + +function regular_file_ok() +{ + local file_type=$(stat --format "%F" "$1") + if test "X$file_type" != "Xregular file"; then return 1; fi +} + +function directory_ok() +{ + file_type=$(stat --format "%F" "$1") + if test "X$file_type" != "Xdirectory"; then return 1; fi +} + +function unlink_ok() +{ + stat "$1" stat ./case > /dev/null 2>&1 + rc=$? + if test $rc != 0; then return 0; fi + return 1; +} + +function hardlink_file_ok() +{ + orig_file=$1 + link_file=$2 + + orig_inode=$(stat --format "%i" "$orig_file") + rc=$? + if test $rc != 0; then return $rc; fi + + link_inode=$(stat --format "%i" "$link_file") + rc=$? + if test $rc != 0; then return $rc; fi + + if test $orig_inode != $link_inode + then + return 1 + fi +} + +function data_ok() +{ + path=$1 + data1=$2 + data2=$(cat $path) + if test "X$data1" != "X$data2" + then + return 1 + fi +} + +function symlink_ok() +{ + local orig_file_name=$(basename $1) + local symlink_file=$2 + + local file_type=$(stat --format "%F" "$symlink_file") + if test "X$file_type" != "Xsymbolic link"; then return 1; fi + + local fname=$(stat $symlink_file --format "%N") + if test "X$fname" != "X$orig_file_name"; then return 1; fi +} + +function rename_ok() +{ + old_name=$1 + new_name=$2 + + if [ -f $old_name ] + then + return 1 + fi + + if [ ! -f $new_name ] + then + return 1 + fi +} + +function data_tests() +{ + local prefix=$1 + + sleep 10 + #Regular file checking + regular_file_ok ${slave_mnt}/${prefix}_f1 + if test $? != 0; then return $?; fi + + #Directory checking + directory_ok ${slave_mnt}/${prefix}_d1 + if test $? != 0; then return $?; fi + + #Rename file/dir checking + rename_ok ${slave_mnt}/${prefix}_f3 ${slave_mnt}/${prefix}_f4 + if test $? != 0; then return $?; fi + rename_ok ${slave_mnt}/${prefix}_d3 ${slave_mnt}/${prefix}_d4 + if test $? != 0; then return $?; fi + + #Symlink + symlink_ok ${slave_mnt}/${prefix}_f1 ${slave_mnt}/${prefix}_sl1 + if test $? != 0; then return $?; fi + + #Hardlink + hardlink_file_ok ${slave_mnt}/${prefix}_f1 ${slave_mnt}/${prefix}_hl1 + if test $? != 0; then return $?; fi + + #Unlink/rmdir checking + unlink_ok ${slave_mnt}/${prefix}_f2 + if test $? != 0; then return $?; fi + unlink_ok ${slave_mnt}/${prefix}_d2 + if test $? != 0; then return $?; fi + + #Data + data_ok ${master_mnt}/${prefix}_f1 "Hello World!" + if test $? != 0; then return $?; fi +} + +function create_georep_session() +{ + $CLI system:: execute gsec_create + if test $? != 0; then return $?; fi + $CLI volume geo-rep $master $slave create push-pem + if test $? != 0; then return $?; fi +} diff --git a/tests/geo-rep/compare-arequal.py b/tests/geo-rep/compare-arequal.py deleted file mode 100644 index 6d9850337b8..00000000000 --- a/tests/geo-rep/compare-arequal.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/python - -import sys -import os -import re -import tempfile -import subprocess -from multiprocessing import Pool -import time -from optparse import OptionParser - -slave_dict = {} -master_res = '' - - -def get_arequal_checksum(me, mnt): - global slave_dict - master_cmd = ['./tests/utils/arequal-checksum', '-p', mnt] - print "Calculating "+me+" checksum ..." - print "" - p = subprocess.Popen(master_cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - ret = p.wait() - stdout, stderr = p.communicate() - if ret: - print "Failed to get the checksum of " + me + " with following error" - print stderr - return 1 - else: - return stdout - - -def get_file_count(me, mnt): - global slave_dict - master_cmd = ['find ' + mnt + ' |wc -l'] - print "Calculating " + me + " files ..." - print "" - p = subprocess.Popen(master_cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=True) - ret = p.wait() - stdout, stderr = p.communicate() - if ret: - print "Failed to get the count of files in " + me - + " with following error" - print stderr - return 1 - else: - return stdout.strip() - - -def compare_checksum(master_mnt, slave_dict): - proc = len(slave_dict)+1 - pool = Pool(processes=proc) - master_res = pool.apply_async(get_arequal_checksum, args=("master", - master_mnt)) - results = [(slave, pool.apply_async(get_arequal_checksum, - args=(slave_dict[slave]["vol"], - slave_dict[slave]["mnt"]))) - for slave in slave_dict] - - pool.close() - pool.join() - for slave, result in results: - slave_dict[slave]["res"] = result.get() - # exception: OSError - - master_res = master_res.get() - - print "arequal-checksum of master is : \n %s" % master_res - for slave in slave_dict: - print "arequal-checksum of geo_rep_slave %s: \n %s" % ( - slave_dict[slave]["vol"], slave_dict[slave]["res"]) - - master_files, master_total = re.findall('Total[\s]+:\s(\w+)', master_res) - master_reg_meta, master_reg = re.findall('Regular files[\s]+:\s(\w+)', - master_res)[1:] - master_dir_meta, master_dir = re.findall('Directories[\s]+:\s(\w+)', - master_res)[1:] - - ret = 0 - for slave in slave_dict: - slave_dict[slave]["files"], slave_dict[slave]["total"] = re.findall( - 'Total[\s]+:\s(\w+)', slave_dict[slave]["res"]) - slave_dict[slave]["reg_meta"], slave_dict[slave]["reg"] = re.findall( - 'Regular files[\s]+:\s(\w+)', slave_dict[slave]["res"])[1:] - slave_dict[slave]["dir_meta"], slave_dict[slave]["dir"] = re.findall( - 'Directories[\s]+:\s(\w+)', slave_dict[slave]["res"])[1:] - - if master_reg_meta != slave_dict[slave]["reg_meta"]: - print ("Meta data checksum for regular files doesn't match " + - "between master and "+slave_dict[slave]["vol"]) - ret = 67 - - if master_dir_meta != slave_dict[slave]["dir_meta"]: - print ("Meta data checksum for directories doesn't match " + - "between master and "+slave_dict[slave]["vol"]) - ret = 68 - - if master_files != slave_dict[slave]["files"]: - print ("Failed to sync all the files from master to " + - slave_dict[slave]["vol"]) - ret = 1 - - if master_total != slave_dict[slave]["total"]: - if master_reg != slave_dict[slave]["reg"]: - print ("Checksum for regular files doesn't match " + - "between master and "+slave_dict[slave]["vol"]) - ret = 1 - elif master_dir != slave_dict[slave]["dir"]: - print ("Checksum for directories doesn't match between " + - "master and "+slave_dict[slave]["vol"]) - ret = 1 - else: - print ("Checksum for symlinks or others doesn't match " + - "between master and "+slave_dict[slave]["vol"]) - ret = 1 - - if ret is 0: - print ("Successfully synced all the files from master " + - "to the "+slave_dict[slave]["vol"]) - - return ret - - -def compare_filecount(master_mnt, slave_dict): - proc = len(slave_dict)+1 - pool = Pool(processes=proc) - - master_res = pool.apply_async(get_file_count, args=("master", master_mnt)) - results = [(slave, pool.apply_async(get_file_count, - args=(slave_dict[slave]["vol"], - slave_dict[slave]["mnt"]))) - for slave in slave_dict] - - pool.close() - pool.join() - for slave, result in results: - slave_dict[slave]["res"] = result.get() - - master_res = master_res.get() - ret = 0 - for slave in slave_dict: - if not master_res == slave_dict[slave]["res"]: - print ("files count between master and " + - slave_dict[slave]["vol"]+" doesn't match yet") - ret = 1 - - return ret - - -def parse_url(url): - match = re.search(r'([\w - _ @ \.]+)::([\w - _ @ \.]+)', url) - if match: - node = match.group(1) - vol = match.group(2) - else: - print 'given url is not a valid.' - sys.exit(1) - return node, vol - - -def cleanup(master_mnt, slave_dict): - try: - os.system("umount %s" % (master_mnt)) - except: - print("Failed to unmount the master volume") - - for slave in slave_dict: - - try: - os.system("umount %s" % (slave_dict[slave]["mnt"])) - os.removedirs(slave_dict[slave]["mnt"]) - except: - print("Failed to unmount the "+slave+" volume") - - os.removedirs(master_mnt) - - -def main(): - - slaves = args[1:] - - masterurl = args[0] - master_node, mastervol = parse_url(masterurl) - master_mnt = tempfile.mkdtemp() - - i = 1 - for slave in slaves: - slave_dict["slave"+str(i)] = {} - slave_dict["slave"+str(i)]["node"], slave_dict[ - "slave"+str(i)]["vol"] = parse_url(slave) - slave_dict["slave"+str(i)]["mnt"] = tempfile.mkdtemp() - i += 1 - - try: - print ("mounting the master volume on "+master_mnt) - os.system("glusterfs -s %s --volfile-id %s %s" % (master_node, - mastervol, - master_mnt)) - time.sleep(3) - except: - print("Failed to mount the master volume") - - for slave in slave_dict: - print slave - print slave_dict[slave] - try: - print ("mounting the slave volume on "+slave_dict[slave]['mnt']) - os.system("glusterfs -s %s --volfile-id %s %s" % ( - slave_dict[slave]["node"], slave_dict[slave]["vol"], - slave_dict[slave]["mnt"])) - time.sleep(3) - except: - print("Failed to mount the "+slave+" volume") - - res = 0 - if option.check == "arequal": - res = compare_checksum(master_mnt, slave_dict) - elif option.check == "find": - res = compare_filecount(master_mnt, slave_dict) - else: - print "wrong options given" - - cleanup(master_mnt, slave_dict) - - sys.exit(res) - - -if __name__ == '__main__': - - usage = "usage: %prog [option] :: \ - :: . . ." - parser = OptionParser(usage=usage) - parser.add_option("-c", dest="check", action="store", type="string", - default="arequal", - help="size of the files to be used [default: %default]") - (option, args) = parser.parse_args() - if not args: - print "usage: