diff options
Diffstat (limited to 'tests/distaf/distaf_libs/distaflibs-gluster/distaflibs/gluster/lib_utils.py')
| -rw-r--r-- | tests/distaf/distaf_libs/distaflibs-gluster/distaflibs/gluster/lib_utils.py | 467 |
1 files changed, 0 insertions, 467 deletions
diff --git a/tests/distaf/distaf_libs/distaflibs-gluster/distaflibs/gluster/lib_utils.py b/tests/distaf/distaf_libs/distaflibs-gluster/distaflibs/gluster/lib_utils.py deleted file mode 100644 index 24267f8bc1d..00000000000 --- a/tests/distaf/distaf_libs/distaflibs-gluster/distaflibs/gluster/lib_utils.py +++ /dev/null @@ -1,467 +0,0 @@ -#!/usr/bin/env python -# This file is part of DiSTAF -# Copyright (C) 2015-2016 Red Hat, Inc. <http://www.redhat.com> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -""" - Description: Helper library for gluster modules. -""" - -from distaf.util import tc -from distaflibs.gluster.volume_ops import get_volume_info -from distaflibs.gluster.mount_ops import mount_volume, umount_volume -import re -import time -from collections import OrderedDict -import tempfile - -try: - import xml.etree.cElementTree as etree -except ImportError: - import xml.etree.ElementTree as etree - - -def append_string_to_file(mnode, filename, str_to_add_in_file): - """Appends the given string in the file. - - Example: - append_string_to_file("abc.def.com", "/var/log/messages", - "test_1_string") - - Args: - mnode (str): Node on which cmd has to be executed. - filename (str): absolute file path to append the string - str_to_add_in_file (str): string to be added in the file, - which is used as a start and stop string for parsing - the file in search_pattern_in_file(). - - Returns: - True, on success, False otherwise - """ - try: - conn = tc.get_connection(mnode, 'root') - if conn == -1: - tc.logger.error("Unable to get connection to 'root' of node %s" - " in append_string_to_file()" % mnode) - return False - - with conn.builtin.open(filename, 'a') as _filehandle: - _filehandle.write(str_to_add_in_file) - - return True - except: - tc.logger.error("Exception occured while adding string to " - "file %s in append_string_to_file()" % filename) - return False - finally: - conn.close() - - -def search_pattern_in_file(mnode, search_pattern, filename, start_str_to_parse, - end_str_to_parse): - """checks if the given search pattern exists in the file - in between 'start_str_to_parse' and 'end_str_to_parse' string. - - Example: - search_pattern = r'.*scrub.*' - search_log("abc.def.com", search_pattern, "/var/log/messages", - "start_pattern", "end_pattern") - - Args: - mnode (str): Node on which cmd has to be executed. - search_pattern (str): regex string to be matched in the file - filename (str): absolute file path to search given string - start_str_to_parse (str): this will be as start string in the - file from which this method will check - if the given search string is present. - end_str_to_parse (str): this will be as end string in the - file whithin which this method will check - if the given search string is present. - - Returns: - True, if search_pattern is present in the file - False, otherwise - """ - - cmd = ("awk '{a[NR]=$0}/" + start_str_to_parse + "/{s=NR}/" + - end_str_to_parse + "/{e=NR}END{for(i=s;i<=e;++i)print " - "a[i]}' " + filename) - - ret, out, err = tc.run(mnode, cmd) - if ret != 0: - tc.logger.error("Failed to match start and end pattern in file" - % filename) - return False - - if not re.search(search_pattern, str(out), re.S): - tc.logger.error("file %s did not have the expected message" - % filename) - return False - - return True - - -def calculate_checksum(file_list, chksum_type='sha256sum', mnode=None): - """This module calculates given checksum for the given file list - - Example: - calculate_checksum([file1, file2]) - - Args: - file_list (list): absolute file names for which checksum - to be calculated - - Kwargs: - mnode (str): Node on which cmd has to be executed. - chksum_type (str): type of the checksum algorithm. - Defaults to sha256sum - - Returns: - NoneType: None if command execution fails, parse errors. - dict: checksum value for each file in the given file list - """ - - if mnode is None: - mnode = tc.servers[0] - - cmd = chksum_type + " %s" % ' '.join(file_list) - ret = tc.run(mnode, cmd) - if ret[0] != 0: - tc.logger.error("Failed to execute checksum command in server %s" - % mnode) - return None - - checksum_dict = {} - for line in ret[1].split('\n')[:-1]: - match = re.search(r'^(\S+)\s+(\S+)', line.strip()) - if match is None: - tc.logger.error("checksum output is not in" - "expected format") - return None - - checksum_dict[match.group(2)] = match.group(1) - - return checksum_dict - - -def get_extended_attributes_info(file_list, encoding='hex', attr_name='', - mnode=None): - """This module gets extended attribute info for the given file list - - Example: - get_extended_attributes_info([file1, file2]) - - Args: - file_list (list): absolute file names for which extended - attributes to be fetched - - Kwargs: - encoding (str): encoding format - attr_name (str): extended attribute name - mnode (str): Node on which cmd has to be executed. - - Returns: - NoneType: None if command execution fails, parse errors. - dict: extended attribute for each file in the given file list - """ - - if mnode is None: - mnode = tc.servers[0] - - if attr_name == '': - cmd = "getfattr -d -m . -e %s %s" % (encoding, ' '.join(file_list)) - else: - cmd = "getfattr -d -m . -n %s %s" % (attr_name, ' '.join(file_list)) - - ret = tc.run(mnode, cmd) - if ret[0] != 0: - tc.logger.error("Failed to execute getfattr command in server %s" - % mnode) - return None - - attr_dict = {} - for each_attr in ret[1].split('\n\n')[:-1]: - for line in each_attr.split('\n'): - if line.startswith('#'): - match = re.search(r'.*file:\s(\S+).*', line) - if match is None: - tc.logger.error("getfattr output is not in " - "expected format") - return None - key = "/" + match.group(1) - attr_dict[key] = {} - else: - output = line.split('=') - attr_dict[key][output[0]] = output[1] - return attr_dict - - -def get_pathinfo(filename, volname, mnode=None): - """This module gets filepath of the given file in gluster server. - - Example: - get_pathinfo("file1", "testvol") - - Args: - filename (str): relative path of file - volname (str): volume name - - Kwargs: - mnode (str): Node on which cmd has to be executed. Defaults - to tc.servers[0]. - - Returns: - NoneType: None if command execution fails, parse errors. - list: file path for the given file in gluster server - """ - - if mnode is None: - mnode = tc.servers[0] - - mount_point = tempfile.mkdtemp() - - # Performing glusterfs mount because only with glusterfs mount - # the file location in gluster server can be identified - ret, _, _ = mount_volume(volname, mtype='glusterfs', - mpoint=mount_point, - mserver=mnode, - mclient=mnode) - if ret != 0: - tc.logger.error("Failed to do gluster mount on volume %s to fetch" - "pathinfo from server %s" - % (volname, mnode)) - return None - - filename = mount_point + '/' + filename - attr_name = 'trusted.glusterfs.pathinfo' - output = get_extended_attributes_info([filename], - attr_name=attr_name, - mnode=mnode) - if output is None: - tc.logger.error("Failed to get path info for %s" % filename) - return None - - pathinfo = output[filename][attr_name] - - umount_volume(mnode, mount_point) - tc.run(mnode, "rm -rf " + mount_point) - - return re.findall(".*?POSIX.*?:(\S+)\>", pathinfo) - - -def list_files(dir_path, parse_str="", mnode=None): - """This module list files from the given file path - - Example: - list_files("/root/dir1/") - - Args: - dir_path (str): directory path name - - Kwargs: - parse_str (str): sub string of the filename to be fetched - mnode (str): Node on which cmd has to be executed. - - Returns: - NoneType: None if command execution fails, parse errors. - list: files with absolute name - """ - if mnode is None: - mnode = tc.clients[0] - - try: - conn = tc.get_connection(mnode, 'root') - if conn == -1: - tc.logger.error("Unable to get connection to 'root' of node %s" - % mnode) - return None - - filepaths = [] - for root, directories, files in conn.modules.os.walk(dir_path): - for filename in files: - if parse_str != "": - if parse_str in filename: - filepath = conn.modules.os.path.join(root, filename) - filepaths.append(filepath) - else: - filepath = conn.modules.os.path.join(root, filename) - filepaths.append(filepath) - return filepaths - except: - tc.logger.error("Exception occured in list_files()") - return None - - finally: - conn.close() - - -def get_servers_bricks_dict(servers): - """This module returns servers_bricks dictionary. - Args: - servers (list): List of servers for which we need the - list of bricks available on it. - Returns: - OrderedDict: key - server - value - list of bricks - Example: - get_servers_bricks_dict(tc.servers) - """ - servers_bricks_dict = OrderedDict() - if not isinstance(servers, list): - servers = [servers] - for server in servers: - for server_list in tc.global_config["servers"]: - if server_list["host"] == server: - brick_root = server_list["brick_root"] - ret, out, err = tc.run(server, "cat /proc/mounts | grep %s" - " | awk '{ print $2}'" % brick_root) - if ret != 0: - tc.logger.error("bricks not available on %s" % server) - else: - servers_bricks_dict[server] = out.strip().split("\n") - - for key, value in servers_bricks_dict.items(): - value.sort() - - return servers_bricks_dict - - -def get_servers_used_bricks_dict(servers, mnode): - """This module returns servers_used_bricks dictionary. - This information is fetched from gluster volume info command. - Args: - servers (list): List of servers for which we need the - list of unused bricks on it. - mnode (str): The node on which gluster volume info command has - to be executed. - Returns: - OrderedDict: key - server - value - list of used bricks - or empty list(if all bricks are free) - Example: - get_servers_used_bricks_dict(tc.servers[:], tc.servers[0]) - """ - if not isinstance(servers, list): - servers = [servers] - - servers_used_bricks_dict = OrderedDict() - for server in servers: - servers_used_bricks_dict[server] = [] - - ret, out, err = tc.run(mnode, "gluster volume info | egrep " - "\"^Brick[0-9]+\" | grep -v \"ss_brick\"", - verbose=False) - if ret != 0: - tc.logger.error("error in getting bricklist using gluster v info") - else: - list1 = list2 = [] - list1 = out.strip().split('\n') - for item in list1: - x = re.search(':(.*)/(.*)', item) - list2 = x.group(1).strip().split(':') - if servers_used_bricks_dict.has_key(list2[0]): - value = servers_used_bricks_dict[list2[0]] - value.append(list2[1]) - else: - servers_used_bricks_dict[list2[0]] = [list2[1]] - - for key, value in servers_used_bricks_dict.items(): - value.sort() - - return servers_used_bricks_dict - - -def get_servers_unused_bricks_dict(servers, mnode): - """This module returns servers_unused_bricks dictionary. - Gets a list of unused bricks for each server by using functions, - get_servers_bricks_dict() and get_servers_used_bricks_dict() - Args: - servers (list): List of servers for which we need the - list of unused bricks available on it. - mnode (str): The node on which gluster volume info command has - to be executed. - Returns: - OrderedDict: key - server - value - list of unused bricks - Example: - get_servers_unused_bricks_dict(tc.servers, tc.servers[0]) - """ - if not isinstance(servers, list): - servers = [servers] - dict1 = get_servers_bricks_dict(servers) - dict2 = get_servers_used_bricks_dict(servers, mnode) - servers_unused_bricks_dict = OrderedDict() - for key, value in dict1.items(): - if dict2.has_key(key): - unused_bricks = list(set(value) - set(dict2[key])) - servers_unused_bricks_dict[key] = unused_bricks - else: - servers_unused_bricks_dict[key] = value - - for key, value in servers_unused_bricks_dict.items(): - value.sort() - - return servers_unused_bricks_dict - - -def form_bricks_path(number_of_bricks, servers, mnode, volname): - """Forms complete bricks path for create-volume/add-brick - given the num_of_bricks - Args: - number_of_bricks (int): The number of bricks for which brick list - has to be created. - servers (list): The list of servers from which the bricks - needs to be selected for creating the brick list. - mnode (str): The node on which the command has to be run. - volname (str): Volume name for which we require brick-list. - Returns: - str - complete brick path. - None - if number_of_bricks is greater than unused bricks. - Example: - form_bricks_path(6, tc.servers, tc.servers(0), "testvol") - """ - if not isinstance(servers, list): - servers = [servers] - dict_index = 0 - bricks_path = '' - - server_bricks_dict = get_servers_unused_bricks_dict(servers, servers[0]) - num_of_unused_bricks = 0 - for server_brick in server_bricks_dict.values(): - num_of_unused_bricks = num_of_unused_bricks + len(server_brick) - - if num_of_unused_bricks < number_of_bricks: - tc.logger.error("Not enough bricks available for creating the bricks") - return None - - brick_index = 0 - vol_info_dict = get_volume_info(volname, mnode) - if vol_info_dict: - brick_index = int(vol_info_dict[volname]['brickCount']) - - for num in range(brick_index, brick_index + number_of_bricks): - if server_bricks_dict.values()[dict_index]: - bricks_path = ("%s %s:%s/%s_brick%s" % (bricks_path, - server_bricks_dict.keys()[dict_index], - server_bricks_dict.values()[dict_index][0], - volname, num)) - server_bricks_dict.values()[dict_index].pop(0) - if dict_index < len(server_bricks_dict) - 1: - dict_index = dict_index + 1 - else: - dict_index = 0 - - return bricks_path |