From 439c9725888fcd63dfbcb0ea234a9bbe1ec019a4 Mon Sep 17 00:00:00 2001 From: Shwetha-H-Panduranga Date: Wed, 4 Jan 2012 18:25:02 +0530 Subject: Adding libraries for validating the testcases. 1)calculating md5sum of mountpoints and bricks using arequal-checksum. 2)validate whether the md5sum of mountpoints and bricks are same. 3)validate gfids of the files on the bricks are same. Change-Id: Id5392721887a348f9c06fedf572098d711956497 Signed-off-by: Shwetha-H-Panduranga --- libs/utils/clientutils.py | 203 +++++++++++++++++----------------------------- libs/utils/hostutils.py | 73 +++++++++++++---- libs/utils/serverutils.py | 112 ++++++++++++++++++++++--- libs/utils/validate.py | 138 +++++++++++++++++++++++++++++++ 4 files changed, 372 insertions(+), 154 deletions(-) create mode 100644 libs/utils/validate.py diff --git a/libs/utils/clientutils.py b/libs/utils/clientutils.py index 4de2998..347e159 100644 --- a/libs/utils/clientutils.py +++ b/libs/utils/clientutils.py @@ -12,7 +12,6 @@ Supported Wrappers :- import atfutils import hostutils from atfglobals import GlobalObj -import pdb def umount(mountkey): """unmounts a mountpoint @@ -20,22 +19,22 @@ def umount(mountkey): Parameters: mountkey : name given to a mount as specified in testenv.cfg file. Ex:-"mount1" - + Returns: Success : 0 - Failure : 1` + Failure : 1` """ logger = GlobalObj.getLoggerObj() base_command = "umount" env = GlobalObj.getTestenvObj() cm = GlobalObj.getConnectionsManagerObj() - + mount_obj = env.getMount(mountkey) if not mount_obj: logger.error("InValid Mount. '%s' not defined in TestEnvironment" % mountkey) return 1 - + clientkey = mount_obj.client client_connection = cm.getConnection(clientkey) if not client_connection: @@ -56,7 +55,7 @@ def umount(mountkey): else: logger.error("Unable to umount %s" % mountkey) - + return return_status def umountall(): @@ -65,10 +64,10 @@ def umountall(): Parameters: None - + Returns: Success : 0 - Failure : 1` + Failure : 1` """ env = GlobalObj.getTestenvObj() failure_flag = False @@ -90,10 +89,10 @@ def mount(mountkey): Parameters: mountkey : name given to a mount as specified in testenv.cfg file. Ex:-"mount1" - + Returns: Success : 0 - Failure : 1` + Failure : 1` """ logger = GlobalObj.getLoggerObj() base_command = "mount" @@ -101,13 +100,13 @@ def mount(mountkey): cm = GlobalObj.getConnectionsManagerObj() command = [base_command] options = [] - + mount_obj = env.getMount(mountkey) if not mount_obj: logger.error("InValid Mount. %s not defined in TestEnvironment" % mountkey) return 1 - + clientkey = mount_obj.client client_connection = cm.getConnection(clientkey) if not client_connection: @@ -128,10 +127,10 @@ def mount(mountkey): if mount_obj.options: options.extend([mount_obj.option]) options.extend([device, mount_obj.dir]) - + command.extend(options) command = ' '.join(command) - + logger.debug('%s: Executing Command: %s' % (clientkey, command)) output = client_connection.executecommand(command) return_status = atfutils.assert_success(output['exitstatus']) @@ -140,14 +139,14 @@ def mount(mountkey): return return_status def mountall(): - """mounts a filesystem for all mounts specified in testenv.cfg file. + """mounts a filesystem for all mounts specified in testenv.cfg file. Parameters: None - + Returns: Success : 0 - Failure : 1` + Failure : 1` """ env = GlobalObj.getTestenvObj() @@ -159,134 +158,82 @@ def mountall(): return 0 +def md5sum_of_mount(mountkey): + """ + Parameter: mount (tye: string) + Returns: output of arequal-checksum command execution(type:dict) + Key : Value of the Output + exitstatus: exit status of the arequal-checksum command on mount + stdoutdata: stdout data of arequal-checksum command execution + stderrdata: stderr data of arequal-checksum command execution + """ + output = {} + output["exitstatus"] = None + output["stdoutdata"] = None + output["stderrdata"] = None + + logger = GlobalObj.getLoggerObj() + env = GlobalObj.getTestenvObj() + mount_obj = env.getMount(mountkey) + if not mount_obj: + logger.error("InValid Mount. %s not defined in TestEnvironment" + % mountkey) + output["exitstatus"] = 1 + return output + + clientkey = mount_obj.client + path = mount_obj.dir + output = hostutils.md5sum(clientkey, path) + return output + +def md5sum_of_mounts(mounts): + """ + Description: + Calculate md5sum of mounts using arequal-checksum + + Parameters: mounts (type: List) + + Returns: md5sums of all the mounts (type: dict) + Keys: mounts + Value: ouput (type:dict) + exitstatus: exit status of the arequal-checksum command on mount + stdoutdata: stdout data of arequal-checksum command execution + stderrdata: stderr data of arequal-checksum command execution + """ + md5sums = {} + for mountkey in mounts: + output = md5sum_of_mount(mountkey) + md5sums[mountkey] = output + + return md5sums def execute_on_mount(mountkey, command, commandInput=None): """ """ + output = {} + output["exitstatus"] = None + output["stdoutdata"] = None + output["stderrdata"] = None + logger = GlobalObj.getLoggerObj() env = GlobalObj.getTestenvObj() mount_obj = env.getMount(mountkey) if not mount_obj: logger.error("InValid Mount. %s not defined in TestEnvironment" % mountkey) - return 1 - - + output["exitstatus"] = 1 + return output + + clientkey = mount_obj.client mountdir = mount_obj.dir command = "cd " + mountdir + " ;" + command output = hostutils.execute_command(clientkey, command, commandInput) return output - + __all__ = ['execute_on_mount', + 'md5sum_of_mounts', 'umount', 'umountall', 'mount', 'mountall'] - -##def umountall(clientkey): -## """ -## """ -## base_command = "umount " -## env = GlobalObj.get_testenv_obj() -## cm = GlobalObj.get_connectionsmanager_obj() -## client_obj = env.getclient(clientkey) -## mountdir = client_obj.mountdir -## volume = client_obj.device -## client = cm.getconnection(clientkey) -## -## mountpoints = [] -## success_flag = False -## failure_flag = False -## command = "mount | grep " + mountdir -## output = client.executecommand(command) -## if not output["exitstatus"]: -## for data in output["stdoutdata"]: -## mountpoints.append(data.split(" ")[2]) -## -## for mountpoint in mountpoints: -## command = base_command + mountpoint -## output = client.executecommand(command) -## return_code = utils.assert_success(output['exitstatus']) -## if return_code: -## failure_flag = True -## else: -## success_flag = True -## continue -## -## if failure_flag: -## return 1 -## -## mountpoints = [] -## success_flag = False -## failure_flag = False -## command = "mount | grep " + volume -## output = client.executecommand(command) -## if not output["exitstatus"]: -## for data in output["stdoutdata"]: -## mountpoints.append(data.split(" ")[2]) -## -## for mountpoint in mountpoints: -## command = base_command + mountpoint -## output = client.executecommand(command) -## return_code = utils.assert_success(output['exitstatus']) -## if return_code: -## failure_flag = True -## else: -## success_flag = True -## continue -## -## if failure_flag: -## return 1 -## -## return 0 - -##def cd_mount(mountkey): -## """ -## """ -## env = GlobalObj.getTestenvObj() -## mount_obj = env.getMount(mountkey) -## if not mount_obj: -## print "InValid Mount. %s not defined in TestEnvironment" % mountkey -## return 1 -## -## clientkey = mount_obj.client -## dirpath = mount_obj.dir -## return_status = hostutils.cd(clientkey, dirpath) -## return return_status -## -##def cd_allmounts(): -## """ -## """ -## env = GlobalObj.getTestenvObj() -## mounts_keys = env.getMountsKeys() -## for mountkey in mounts_keys: -## return_status = cd_mount(mountkey) -## if return_status: -## return return_status -## -## return 0 - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/libs/utils/hostutils.py b/libs/utils/hostutils.py index e6f7189..bb4ca29 100644 --- a/libs/utils/hostutils.py +++ b/libs/utils/hostutils.py @@ -13,7 +13,7 @@ import re from collections import namedtuple import atfutils from atfglobals import GlobalObj -import pdb +import os system_dirs = re.compile('(/bin|/boot|/dev|/etc|/lib|/mnt|/net|/opt|/root|/sbin|/usr|/var|/sys)\/?$') @@ -27,15 +27,15 @@ def cd(hostkey, dirpath): if not host_connection: logger.error("SSH Connection Not established to host '%s' " % hostkey) return 1 - + command = ' '.join([base_command, dirpath]) logger.debug('%s: Executing Command: %s' % (hostkey, command)) output = host_connection.executecommand(command) return_status = atfutils.assert_success(output['exitstatus']) atfutils.print_stdout(output['stdoutdata']) atfutils.print_stderr(output['stderrdata']) - return return_status - + return return_status + def rmdir(hostkey, dirpath): """ """ @@ -45,7 +45,7 @@ def rmdir(hostkey, dirpath): if system_dirs.match(dirpath): logger.error("System Directiories cannot be deleted") return 1 - + else: host_connection = cm.getConnection(hostkey) if not host_connection: @@ -60,7 +60,7 @@ def rmdir(hostkey, dirpath): atfutils.print_stdout(output['stdoutdata']) atfutils.print_stderr(output['stderrdata']) return return_status - + def mkdir(hostkey, dirpath): """ """ @@ -92,7 +92,7 @@ def mkfs(hostkey, device, fstype=None): """ logger = GlobalObj.getLoggerObj() base_command = "mkfs" - cm = GlobalObj.getConnectionsManagerObj() + cm = GlobalObj.getConnectionsManagerObj() host_connection = cm.getConnection(hostkey) command = [base_command] options = [] @@ -103,9 +103,9 @@ def mkfs(hostkey, device, fstype=None): if fstype is None: fstype = "xfs" - + options.extend(["-t", fstype, "-f", device]) - + command.extend(options) command = ' '.join(command) logger.debug('%s: Executing Command: %s' % (hostkey, command)) @@ -115,13 +115,48 @@ def mkfs(hostkey, device, fstype=None): atfutils.print_stderr(output['stderrdata']) return return_status +def md5sum(hostkey, path): + """ + """ + output = {} + output["exitstatus"] = None + output["stdoutdata"] = None + output["stderrdata"] = None + + base_command1 = "rm -rf" + base_command2 = "arequal-checksum" + landfill_dir = os.path.join(path, ".landfill") + + cm = GlobalObj.getConnectionsManagerObj() + host_connection = cm.getConnection(hostkey) + if not host_connection: + logger.error("SSH connection to host '%s' has not been established" + % hostkey) + output["exitstatus"] = 1 + return output + + command1 = ' '.join([base_command1, landfill_dir]) + command2 = ' '.join([base_command2, path]) + command = ';'.join([command1, command2]) + output = host_connection.executecommand(command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + if output['stdoutdata'] is None or (not output['stdoutdata']): + output['stdoutdata'] = "" + + else: + output['stdoutdata'] = str(output['stdoutdata']) + + return output + def find_mountpoints(hostkey, device): """ """ logger = GlobalObj.getLoggerObj() - base_command = "mount | grep " + base_command = "mount | grep " cm = GlobalObj.getConnectionsManagerObj() - + host_connection = cm.getConnection(hostkey) if not host_connection: logger.error("SSH connection to host '%s' has not been established" @@ -141,13 +176,18 @@ def find_mountpoints(hostkey, device): def execute_command(hostkey, command, commandInput=None): """ """ + output = {} + output["exitstatus"] = None + output["stdoutdata"] = None + output["stderrdata"] = None logger = GlobalObj.getLoggerObj() cm = GlobalObj.getConnectionsManagerObj() host_connection = cm.getConnection(hostkey) if not host_connection: logger.error("SSH Connection Not established to host '%s' " % hostkey) - return 1 + output["exitstatus"] = 1 + return output new_command = _substitute_value_for_variables(command) logger.debug('%s: Executing Command: %s' % (hostkey, command)) @@ -178,14 +218,14 @@ def _substitute_value_for_variables(command): "name", "option"]) variables_to_replace = [] - + variables = pattern_for_variables.findall(command) - if not variables: + if not variables: return new_command - + else: active_volume = env.getActiveVolume() - + for variable in variables: if variable not in variables_to_replace: name, option = (variable.strip("<>")).split(".") @@ -230,5 +270,6 @@ __all__ = ['cd', 'rmdir', 'mkdir', 'mkfs', + 'md5sum', 'find_mountpoints', 'execute_command'] diff --git a/libs/utils/serverutils.py b/libs/utils/serverutils.py index 068dd17..10fe830 100644 --- a/libs/utils/serverutils.py +++ b/libs/utils/serverutils.py @@ -3,6 +3,98 @@ import re import hostutils from atfglobals import GlobalObj +import atfutils + +def md5sum_of_brick(brickkey): + """ + Parameter: brick (tye: string) + Returns: output of arequal-checksum command execution(type:dict) + Key : Value of the Output + exitstatus: exit status of the arequal-checksum command on brick + stdoutdata: stdout data of arequal-checksum command execution + stderrdata: stderr data of arequal-checksum command execution + """ + output = {} + output["exitstatus"] = None + output["stdoutdata"] = None + output["stderrdata"] = None + logger = GlobalObj.getLoggerObj() + env = GlobalObj.getTestenvObj() + + raw_brick_obj = env.getRawBrick(brickkey) + if not raw_brick_obj: + logger.error("InValid Brick. %s not defined in TestEnvironment" % + brickkey) + output["exitstatus"] = 1 + return output + + else: + serverkey = re.split("\.", raw_brick_obj.hostname, maxsplit=1)[0] + + brick_obj = env.getBrick(brickkey) + if not brick_obj: + logger.error("InValid Brick. %s not defined in TestEnvironment" + % brickkey) + output["exitstatus"] = 1 + return output + else: + brick_path = brick_obj.path + + output = hostutils.md5sum(serverkey, brick_path) + return output + +def md5sum_of_bricks(bricks): + """ + Description: + Calculate md5sum of bricks using arequal-checksum + + Parameters: bricks (type: List) + + Returns: md5sums of all the bricks (type: dict) + Keys: bricks + Value: ouput (type:dict) + exitstatus: exit status of the arequal-checksum command on brick + stdoutdata: stdout data of arequal-checksum command execution + stderrdata: stderr data of arequal-checksum command execution + """ + md5sums = {} + for brickkey in bricks: + output = md5sum_of_brick(brickkey) + md5sums[brickkey] = output + + return md5sums + +def get_gfid_on_brick(brickkey, filename="."): + """ + """ + output = {} + output["exitstatus"] = None + output["stdoutdata"] = None + output["stderrdata"] = None + base_command = "getfattr -n 'trusted.gfid' -e hex" + command = ' '.join([base_command, filename]) + output = execute_on_brick(brickkey, command) + if atfutils.assert_success(output['exitstatus']): + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + elif output['stdoutdata'] is None or (not output['stdoutdata']): + output['stdoutdata'] = "" + + else: + output['stdoutdata'] = str(output['stdoutdata']) + + return output + +def get_gfid_on_bricks(bricks, filename="."): + """ + """ + gfid_on_bricks = {} + for brickkey in bricks: + output = get_gfid_on_brick(brickkey, filename) + gfid_on_bricks[brickkey] = output + + return gfid_on_bricks def execute_on_brick(brickkey, command, commandInput=None): """ @@ -11,7 +103,7 @@ def execute_on_brick(brickkey, command, commandInput=None): output["exitstatus"] = None output["stdoutdata"] = None output["stderrdata"] = None - + logger = GlobalObj.getLoggerObj() env = GlobalObj.getTestenvObj() @@ -19,23 +111,23 @@ def execute_on_brick(brickkey, command, commandInput=None): if not raw_brick_obj: logger.error("InValid Brick. %s not defined in TestEnvironment" % brickkey) + output["exitstatus"] = 1 return output - serverkey = re.split("\.", raw_brick_obj.hostname, maxsplit=1)[0] - + else: + serverkey = re.split("\.", raw_brick_obj.hostname, maxsplit=1)[0] + brick_obj = env.getBrick(brickkey) if not brick_obj: logger.error("InValid Brick. %s not defined in TestEnvironment" % brickkey) + output["exitstatus"] = 1 return output - exportdirpath = brick_obj.path + else: + exportdirpath = brick_obj.path command = "cd " + exportdirpath + ";" + command output = hostutils.execute_command(serverkey, command, commandInput) return output -__all__ = ['execute_on_brick'] - - - - - +__all__ = ['execute_on_brick', + 'md5sum_of_bricks'] diff --git a/libs/utils/validate.py b/libs/utils/validate.py new file mode 100644 index 0000000..2cdfa45 --- /dev/null +++ b/libs/utils/validate.py @@ -0,0 +1,138 @@ +import operator +import atfutils +import serverutils +import clientutils +from atfglobals import GlobalObj +import re + +def assert_success_of_outputs(outputs): + """ + """ + for key in outputs.keys(): + output = outputs[key] + if atfutils.assert_success(output["exitstatus"]): + return 1 + return 0 + +def match_md5sum(sum1, sum2): + """ + Parameters: + sum1: md5sum1 (type: string) + sum2: md5sum2 (type: string) + + Description: + Compares md5sum1 with md5sum2. + If they are equal, return True. else False + """ + md5sum_match_status = False + md5sum_match_status = operator.eq(sum1, sum2) + return md5sum_match_status + +def match_md5sums(md5sums): + """ + Parameters: + md5sums: MD5Sums of files + (type: dict where key: mountkey, value: md5sum (type:string)) + + Description: + Compares md5sums of files. + If the md5sum of all mounts matches returns True else return False. + """ + logger = GlobalObj.getLoggerObj() + md5sum_match_status = False + compare_key, output = md5sums.popitem() + compare_value = output["stdoutdata"] + + if not md5sums: + md5sum_match_status = True + else: + for key in md5sums.keys(): + output = md5sums[key] + value = output["stdoutdata"] + md5sum_match_status = match_md5sum(compare_value, value) + if md5sum_match_status is False: + logger.error("Md5sum of %s didn't match with Md5sum of %s" % + (key, compare_key)) + return md5sum_match_status + + return md5sum_match_status + +def match_gfid(gfid1, gfid2): + """ + """ + gfid_match_status = False + gfid_match_status = operator.eq(gfid1, gfid2) + return gfid_match_status + +def match_gfids(gfid_on_bricks): + """ + """ + gfid_match_status = False + compare_key, output = gfid_on_bricks.popitem() + compare_value = output["stdoutdata"] + + if not gfid_on_bricks: + gfid_match_status = True + else: + for key in gfid_on_bricks.keys(): + output = gfid_on_bricks[key] + value = output["stdoutdata"] + gfid_match_status = match_gfid(compare_value, value) + if gfid_match_status is False: + logger.error("gfid of %s didn't match gfid of %s" % + compare_key, key) + return gfid_match_status + + return gfid_match_status + +def validate_on_bricks(bricks, command, expected_status, + expected_output, stream="stdout"): + """ + Parameters: + bricks : list of bricks + command : command to execute on each brick + expected_status : return status to expect + expected_output : output to expect + """ + for brick in bricks: + output = serverutils.execute_on_brick(brick, command) + validate_status = atfutils.validate_output(output, expected_status, + expected_output, stream) + if validate_status is not 0: + return validate_status + + return 0 + +def validate_md5sums(mounts, bricks): + """ + """ + mounts_md5sums = clientutils.md5sum_of_mounts(mounts) + bricks_md5sums = serverutils.md5sum_of_bricks(bricks) + + md5sums = mounts_md5sums + md5sums.update(bricks_md5sums) + + assert_success_status = assert_success_of_outputs(md5sums) + if assert_success_status is not 0: + return assert_success_status + + md5sum_match_status = match_md5sums(md5sums) + if md5sum_match_status is False: + return 1 + else: + return 0 + +def validate_gfids(bricks, filename="."): + """ + """ + gfid_on_bricks = serverutils.get_gfid_on_bricks(bricks, filename) + + assert_success_status = assert_success_of_outputs(gfid_on_bricks) + if assert_success_status is not 0: + return assert_success_status + + gfid_match_status = match_gfids(gfid_on_bricks) + if gfid_match_status is False: + return 1 + else: + return 0 -- cgit