diff options
10 files changed, 173 insertions, 382 deletions
diff --git a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py index f8d41752..7266445a 100644 --- a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py +++ b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py @@ -6,10 +6,12 @@ except ImportError: import json import re +import time from glusto.core import Glusto as g +import six -from openshiftstoragelibs import exceptions +from openshiftstoragelibs import command from openshiftstoragelibs import heketi_version from openshiftstoragelibs.utils import parse_prometheus_data @@ -18,6 +20,53 @@ HEKETI_COMMAND_TIMEOUT = g.config.get("common", {}).get( "heketi_command_timeout", 120) TIMEOUT_PREFIX = "timeout %s " % HEKETI_COMMAND_TIMEOUT +MASTER_NODE = list(g.config["ocp_servers"]["master"].keys())[0] +HEKETI_DC = g.config.get("cns", g.config.get("openshift"))[ + "heketi_config"]["heketi_dc_name"] +GET_HEKETI_PODNAME_CMD = ( + "oc get pods -l deploymentconfig=%s -o=custom-columns=:.metadata.name " + "--no-headers" % HEKETI_DC +) + + +def cmd_run_on_heketi_pod(cmd, raise_on_error=True): + """Autodetect Heketi podname and run specified command on it.""" + heketi_podname = command.cmd_run( + cmd=GET_HEKETI_PODNAME_CMD, hostname=MASTER_NODE).strip() + # NOTE(vponomar): we redefine '--server' option which is provided + # as part of the 'cmd' var. + assert heketi_podname.strip(), ( + "Heketi POD not found on '%s' node using following command: \n%s" % ( + MASTER_NODE, GET_HEKETI_PODNAME_CMD)) + if '--server=' in cmd and 'heketi-cli' in cmd: + cmd_with_podname_prefix = ( + "oc exec %s -- %s --server=http://localhost:8080" % ( + heketi_podname, cmd)) + else: + cmd_with_podname_prefix = "oc exec %s -- %s" % (heketi_podname, cmd) + result = command.cmd_run( + cmd=cmd_with_podname_prefix, hostname=MASTER_NODE, + raise_on_error=raise_on_error) + return result + + +def heketi_cmd_run(hostname, cmd, raise_on_error=True): + """Run Heketi client command from a node backing up with Heketi pod CLI.""" + try: + out = command.cmd_run( + cmd=cmd, hostname=hostname, raise_on_error=raise_on_error) + except Exception as e: + g.log.error( + 'Failed to run "%s" command on the "%s" host. ' + 'Got following error:\n%s' % (cmd, hostname, e)) + if ('connection refused' in six.text_type(e).lower() or + 'operation timed out' in six.text_type(e).lower()): + time.sleep(1) + out = cmd_run_on_heketi_pod(cmd, raise_on_error=raise_on_error) + else: + raise + return out + def _set_heketi_global_flags(heketi_server_url, **kwargs): """Helper function to set heketi-cli global flags.""" @@ -44,7 +93,7 @@ def _set_heketi_global_flags(heketi_server_url, **kwargs): def heketi_volume_create(heketi_client_node, heketi_server_url, size, - raw_cli_output=False, **kwargs): + **kwargs): """Creates heketi volume with the given user options. Args: @@ -75,9 +124,8 @@ def heketi_volume_create(heketi_client_node, heketi_server_url, size, Returns: dict: volume create info on success, only cli option is specified without --json option, then it returns raw string output. - Tuple (ret, out, err): if raw_cli_output is True. Raises: - exceptions.ExecutionError when error occurs and raw_cli_output is False + exceptions.ExecutionError when error occurs Example: heketi_volume_create(heketi_client_node, heketi_server_url, size) @@ -134,8 +182,6 @@ def heketi_volume_create(heketi_client_node, heketi_server_url, size, "--secret %s" % kwargs.get("secret") if kwargs.get("secret") else "") user_arg = "--user %s" % kwargs.get("user") if kwargs.get("user") else "" - err_msg = "Failed to create volume. " - cmd = ("heketi-cli -s %s volume create --size=%s %s %s %s %s %s %s " "%s %s %s %s %s %s %s %s %s %s" % ( heketi_server_url, str(size), block_arg, clusters_arg, @@ -145,21 +191,14 @@ def heketi_volume_create(heketi_client_node, heketi_server_url, size, persistent_volume_file_arg, redundancy_arg, replica_arg, snapshot_factor_arg, json_arg, secret_arg, user_arg)) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if raw_cli_output: - return ret, out, err - if ret != 0: - err_msg += "Out: %s \n Err: %s" % (out, err) - g.log.error(err_msg) - raise exceptions.ExecutionError(err_msg) + out = heketi_cmd_run(heketi_client_node, cmd) if json_arg: return json.loads(out) return out def heketi_volume_info(heketi_client_node, heketi_server_url, volume_id, - raw_cli_output=False, **kwargs): + **kwargs): """Executes heketi volume info command. Args: @@ -175,8 +214,6 @@ def heketi_volume_info(heketi_client_node, heketi_server_url, volume_id, Returns: dict: volume info on success - False: in case of failure - Tuple (ret, out, err): if raw_cli_output is True Raises: exceptions.ExecutionError: if command fails. @@ -191,24 +228,14 @@ def heketi_volume_info(heketi_client_node, heketi_server_url, volume_id, cmd = "heketi-cli -s %s volume info %s %s %s %s" % ( heketi_server_url, volume_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if raw_cli_output: - return ret, out, err - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) - + out = heketi_cmd_run(heketi_client_node, cmd) if json_arg: return json.loads(out) return out def heketi_volume_expand(heketi_client_node, heketi_server_url, volume_id, - expand_size, raw_cli_output=False, **kwargs): + expand_size, **kwargs): """Executes heketi volume expand command. Args: @@ -226,7 +253,6 @@ def heketi_volume_expand(heketi_client_node, heketi_server_url, volume_id, Returns: dict: volume expand info on success, only cli option is specified without --json option, then it returns raw string output. - Tuple (ret, out, err): if raw_cli_output is True Raises: exceptions.ExecutionError: if command fails. @@ -244,24 +270,14 @@ def heketi_volume_expand(heketi_client_node, heketi_server_url, volume_id, heketi_server_url, volume_id, expand_size, json_arg, admin_key, user)) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if raw_cli_output: - return ret, out, err - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) if json_arg: return json.loads(out) return out def heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id, - raw_cli_output=False, raise_on_error=True, **kwargs): + raise_on_error=True, **kwargs): """Executes heketi volume delete command. Args: @@ -279,10 +295,9 @@ def heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id, Returns: str: volume delete command output on success - Tuple (ret, out, err): if raw_cli_output is True Raises: - exceptions.ExecutionError when error occurs and raw_cli_output is False + exceptions.ExecutionError when error occurs Example: heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id) @@ -290,25 +305,16 @@ def heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id, heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags( heketi_server_url, **kwargs) - err_msg = "Failed to delete '%s' volume. " % volume_id cmd = "heketi-cli -s %s volume delete %s %s %s %s" % ( heketi_server_url, volume_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if raw_cli_output: - return ret, out, err - if ret != 0: - err_msg += "Out: %s, \nErr: %s" % (out, err) - g.log.error(err_msg) - if raise_on_error: - raise exceptions.ExecutionError(err_msg) + out = heketi_cmd_run( + heketi_client_node, cmd, raise_on_error=raise_on_error) return out -def heketi_volume_list(heketi_client_node, heketi_server_url, - raw_cli_output=False, **kwargs): +def heketi_volume_list(heketi_client_node, heketi_server_url, **kwargs): """Executes heketi volume list command. Args: @@ -324,7 +330,6 @@ def heketi_volume_list(heketi_client_node, heketi_server_url, Returns: dict: volume list with --json on success, if cli option is specified without --json option or with url, it returns raw string output. - Tuple (ret, out, err): if raw_cli_output is True Raises: exceptions.ExecutionError: if command fails. @@ -339,23 +344,13 @@ def heketi_volume_list(heketi_client_node, heketi_server_url, cmd = "heketi-cli -s %s volume list %s %s %s" % ( heketi_server_url, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if raw_cli_output: - return ret, out, err - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) if json_arg: return json.loads(out) return out -def heketi_topology_info(heketi_client_node, heketi_server_url, - raw_cli_output=False, **kwargs): +def heketi_topology_info(heketi_client_node, heketi_server_url, **kwargs): """Executes heketi topology info command. Args: @@ -371,7 +366,6 @@ def heketi_topology_info(heketi_client_node, heketi_server_url, Returns: dict: topology info if --json option is specified. If only cli option is specified, raw command output is returned on success. - Tuple (ret, out, err): if raw_cli_output is True Raises: exceptions.ExecutionError: if command fails. @@ -386,16 +380,7 @@ def heketi_topology_info(heketi_client_node, heketi_server_url, cmd = "heketi-cli -s %s topology info %s %s %s" % ( heketi_server_url, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if raw_cli_output: - return ret, out, err - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) if json_arg: return json.loads(out) return out @@ -427,14 +412,20 @@ def hello_heketi(heketi_client_node, heketi_server_url, **kwargs): heketi_server_url, **kwargs) cmd = "curl --max-time 10 %s/hello" % heketi_server_url - ret, out, err = g.run(heketi_client_node, cmd) - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + try: + command.cmd_run(cmd=cmd, hostname=heketi_client_node) + except Exception as e: + g.log.error( + 'Failed to run "%s" command on the "%s" host. ' + 'Got following error:\n%s' % (cmd, heketi_client_node, e)) + if ('connection refused' in six.text_type(e).lower() or + 'operation timed out' in six.text_type(e).lower()): + time.sleep(1) + cmd_run_on_heketi_pod( + "curl --max-time 10 http://localhost:8080/hello") + else: + raise return True @@ -470,14 +461,7 @@ def heketi_cluster_delete(heketi_client_node, heketi_server_url, cluster_id, cmd = "heketi-cli -s %s cluster delete %s %s %s %s" % ( heketi_server_url, cluster_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) return out @@ -513,14 +497,7 @@ def heketi_cluster_info(heketi_client_node, heketi_server_url, cluster_id, cmd = "heketi-cli -s %s cluster info %s %s %s %s" % ( heketi_server_url, cluster_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) if json_arg: return json.loads(out) return out @@ -556,21 +533,14 @@ def heketi_cluster_list(heketi_client_node, heketi_server_url, **kwargs): cmd = "heketi-cli -s %s cluster list %s %s %s" % ( heketi_server_url, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) if json_arg: return json.loads(out) return out def heketi_device_add(heketi_client_node, heketi_server_url, device_name, - node_id, raw_cli_output=False, **kwargs): + node_id, **kwargs): """Executes heketi device add command. Args: @@ -587,7 +557,6 @@ def heketi_device_add(heketi_client_node, heketi_server_url, device_name, Returns: str: heketi device add command output on success. - Tuple (ret, out, err): if raw_cli_output is True Raises: exceptions.ExecutionError: if command fails. @@ -603,21 +572,12 @@ def heketi_device_add(heketi_client_node, heketi_server_url, device_name, cmd = "heketi-cli -s %s device add --name=%s --node=%s %s %s %s" % ( heketi_server_url, device_name, node_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if raw_cli_output: - return ret, out, err - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) return out def heketi_device_delete(heketi_client_node, heketi_server_url, device_id, - raw_cli_output=False, **kwargs): + **kwargs): """Executes heketi device delete command. Args: @@ -633,7 +593,6 @@ def heketi_device_delete(heketi_client_node, heketi_server_url, device_id, Returns: str: heketi device delete command output on success. - Tuple (ret, out, err): if raw_cli_output is True Raises: exceptions.ExecutionError: if command fails. @@ -648,21 +607,12 @@ def heketi_device_delete(heketi_client_node, heketi_server_url, device_id, cmd = "heketi-cli -s %s device delete %s %s %s %s" % ( heketi_server_url, device_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if raw_cli_output: - return ret, out, err - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) return out def heketi_device_disable(heketi_client_node, heketi_server_url, device_id, - raw_cli_output=False, **kwargs): + **kwargs): """Executes heketi device disable command. Args: @@ -678,7 +628,6 @@ def heketi_device_disable(heketi_client_node, heketi_server_url, device_id, Returns: str: heketi device disable command output on success. - Tuple (ret, out, err): if raw_cli_output is True Raises: exceptions.ExecutionError: if command fails. @@ -692,22 +641,12 @@ def heketi_device_disable(heketi_client_node, heketi_server_url, device_id, cmd = "heketi-cli -s %s device disable %s %s %s %s" % ( heketi_server_url, device_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - - ret, out, err = g.run(heketi_client_node, cmd) - - if raw_cli_output: - return ret, out, err - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) return out def heketi_device_enable(heketi_client_node, heketi_server_url, device_id, - raw_cli_output=False, **kwargs): + **kwargs): """Executes heketi device enable command. Args: @@ -723,7 +662,6 @@ def heketi_device_enable(heketi_client_node, heketi_server_url, device_id, Returns: str: heketi device enable command output on success. - Tuple (ret, out, err): if raw_cli_output is True Raises: exceptions.ExecutionError: if command fails. @@ -737,22 +675,12 @@ def heketi_device_enable(heketi_client_node, heketi_server_url, device_id, cmd = "heketi-cli -s %s device enable %s %s %s %s" % ( heketi_server_url, device_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - - ret, out, err = g.run(heketi_client_node, cmd) - - if raw_cli_output: - return ret, out, err - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) return out def heketi_device_info(heketi_client_node, heketi_server_url, device_id, - raw_cli_output=False, **kwargs): + **kwargs): """Executes heketi device info command. Args: @@ -769,7 +697,6 @@ def heketi_device_info(heketi_client_node, heketi_server_url, device_id, Returns: Str: device info as raw CLI output if "json" arg is not provided. Dict: device info parsed to dict if "json" arg is provided. - Tuple (ret, out, err): if raw_cli_output is True Raises: exceptions.ExecutionError: if command fails. @@ -784,17 +711,7 @@ def heketi_device_info(heketi_client_node, heketi_server_url, device_id, cmd = "heketi-cli -s %s device info %s %s %s %s" % ( heketi_server_url, device_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if raw_cli_output: - return ret, out, err - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) - + out = heketi_cmd_run(heketi_client_node, cmd) if json_arg: device_info = json.loads(out) return device_info @@ -803,7 +720,7 @@ def heketi_device_info(heketi_client_node, heketi_server_url, device_id, def heketi_device_remove(heketi_client_node, heketi_server_url, device_id, - raw_cli_output=False, **kwargs): + **kwargs): """Executes heketi device remove command. Args: @@ -819,7 +736,6 @@ def heketi_device_remove(heketi_client_node, heketi_server_url, device_id, Returns: str: heketi device remove command output on success. - Tuple (ret, out, err): if raw_cli_output is True Raises: exceptions.ExecutionError: if command fails. @@ -834,17 +750,7 @@ def heketi_device_remove(heketi_client_node, heketi_server_url, device_id, cmd = "heketi-cli -s %s device remove %s %s %s %s" % ( heketi_server_url, device_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if raw_cli_output: - return ret, out, err - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) - + out = heketi_cmd_run(heketi_client_node, cmd) return out @@ -879,14 +785,7 @@ def heketi_node_delete(heketi_client_node, heketi_server_url, node_id, cmd = "heketi-cli -s %s node delete %s %s %s %s" % ( heketi_server_url, node_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) return out @@ -921,14 +820,7 @@ def heketi_node_disable(heketi_client_node, heketi_server_url, node_id, cmd = "heketi-cli -s %s node disable %s %s %s %s" % ( heketi_server_url, node_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) return out @@ -963,14 +855,7 @@ def heketi_node_enable(heketi_client_node, heketi_server_url, node_id, cmd = "heketi-cli -s %s node enable %s %s %s %s" % ( heketi_server_url, node_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) return out @@ -1005,14 +890,7 @@ def heketi_node_info(heketi_client_node, heketi_server_url, node_id, **kwargs): cmd = "heketi-cli -s %s node info %s %s %s %s" % ( heketi_server_url, node_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) if json_arg: return json.loads(out) return out @@ -1038,14 +916,7 @@ def heketi_node_list(heketi_client_node, heketi_server_url, cmd = "heketi-cli -s %s node list %s %s %s" % ( heketi_server_url, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) heketi_node_id_list = [] for line in out.strip().split("\n"): @@ -1087,14 +958,7 @@ def heketi_blockvolume_info(heketi_client_node, heketi_server_url, cmd = "heketi-cli -s %s blockvolume info %s %s %s %s" % ( heketi_server_url, block_volume_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) if json_arg: return json.loads(out) return out @@ -1163,14 +1027,7 @@ def heketi_blockvolume_create(heketi_client_node, heketi_server_url, size, clusters_arg, ha_arg, name_arg, name_arg, admin_key, user, json_arg)) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) if json_arg: return json.loads(out) return out @@ -1206,18 +1063,12 @@ def heketi_blockvolume_delete(heketi_client_node, heketi_server_url, heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags( heketi_server_url, **kwargs) - err_msg = "Failed to delete '%s' volume. " % block_volume_id cmd = "heketi-cli -s %s blockvolume delete %s %s %s %s" % ( heketi_server_url, block_volume_id, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - err_msg += "Out: %s, \nErr: %s" % (out, err) - g.log.error(err_msg) - if raise_on_error: - raise exceptions.ExecutionError(err_msg) + out = heketi_cmd_run( + heketi_client_node, cmd, raise_on_error=raise_on_error) return out @@ -1252,14 +1103,7 @@ def heketi_blockvolume_list(heketi_client_node, heketi_server_url, **kwargs): cmd = "heketi-cli -s %s blockvolume list %s %s %s" % ( heketi_server_url, json_arg, admin_key, user) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, heketi_client_node, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(heketi_client_node, cmd) if json_arg: return json.loads(out) return out @@ -1296,14 +1140,7 @@ def verify_volume_name_prefix(hostname, prefix, namespace, pvc_name, cmd = "heketi-cli -s %s volume list %s %s %s | grep %s" % ( heketi_server_url, json_arg, admin_key, user, heketi_vol_name_prefix) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(hostname, cmd, "root") - - if ret != 0: - msg = ( - "Failed to execute '%s' command on '%s' node with following " - "error: %s" % (cmd, hostname, err)) - g.log.error(msg) - raise exceptions.ExecutionError(msg) + out = heketi_cmd_run(hostname, cmd) output = out.strip() g.log.info("heketi volume with volnameprefix present %s" % output) return True @@ -1347,14 +1184,9 @@ def set_tags(heketi_client_node, heketi_server_url, source, source_id, tag, cmd = ("heketi-cli -s %s %s settags %s %s %s %s" % (heketi_server_url, source, source_id, tag, user, secret)) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if not ret: - g.log.info("Tagging of %s to %s is successful" % (source, tag)) - return True - - g.log.error(err) - raise exceptions.ExecutionError(err) + heketi_cmd_run(heketi_client_node, cmd) + g.log.info("Tagging of %s to %s is successful" % (source, tag)) + return True def set_arbiter_tag(heketi_client_node, heketi_server_url, source, @@ -1437,14 +1269,9 @@ def rm_tags(heketi_client_node, heketi_server_url, source, source_id, tag, cmd = ("heketi-cli -s %s %s rmtags %s %s %s %s" % (heketi_server_url, source, source_id, tag, user, secret)) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if not ret: - g.log.info("Removal of %s tag from %s is successful." % (tag, source)) - return True - - g.log.error(err) - raise exceptions.ExecutionError(err) + heketi_cmd_run(heketi_client_node, cmd) + g.log.info("Removal of %s tag from %s is successful." % (tag, source)) + return True def rm_arbiter_tag(heketi_client_node, heketi_server_url, source, source_id, @@ -1508,11 +1335,21 @@ def get_heketi_metrics(heketi_client_node, heketi_server_url, raise NotImplementedError(msg) cmd = "curl --max-time 10 %s/metrics" % heketi_server_url - ret, out, err = g.run(heketi_client_node, cmd) - if ret != 0: - msg = "failed to get Heketi metrics with following error: %s" % err - g.log.error(msg) - raise exceptions.ExecutionError(msg) + + try: + out = command.cmd_run(cmd=cmd, hostname=heketi_client_node) + except Exception as e: + g.log.error( + 'Failed to run "%s" command on the "%s" host. ' + 'Got following error:\n%s' % (cmd, heketi_client_node, e)) + if ('connection refused' in six.text_type(e).lower() or + 'operation timed out' in six.text_type(e).lower()): + time.sleep(1) + out = cmd_run_on_heketi_pod( + "curl --max-time 10 http://localhost:8080/metrics") + else: + raise + if prometheus_format: return out.strip() return parse_prometheus_data(out) @@ -1546,13 +1383,7 @@ def heketi_examine_gluster(heketi_client_node, heketi_server_url): cmd = ("heketi-cli server state examine gluster -s %s %s %s" % (heketi_server_url, user, secret)) cmd = TIMEOUT_PREFIX + cmd - ret, out, err = g.run(heketi_client_node, cmd) - - if ret != 0: - msg = "failed to examine gluster with following error: %s" % err - g.log.error(msg) - raise exceptions.ExecutionError(msg) - + out = heketi_cmd_run(heketi_client_node, cmd) return json.loads(out) diff --git a/tests/functional/heketi/test_create_distributed_replica_heketi_volume.py b/tests/functional/heketi/test_create_distributed_replica_heketi_volume.py index 8a3beb29..f5c41d8d 100644 --- a/tests/functional/heketi/test_create_distributed_replica_heketi_volume.py +++ b/tests/functional/heketi/test_create_distributed_replica_heketi_volume.py @@ -5,7 +5,6 @@ from glusto.core import Glusto as g from glustolibs.gluster.volume_ops import get_volume_list, get_volume_info from openshiftstoragelibs.baseclass import BaseClass -from openshiftstoragelibs import exceptions from openshiftstoragelibs.heketi_ops import ( heketi_device_disable, heketi_device_enable, @@ -102,7 +101,7 @@ class TestHeketiVolume(BaseClass): "Trying to create distributed '%s'Gb volume." % vol_size_gb) heketi_vol = heketi_volume_create( self.heketi_client_node, heketi_url, vol_size_gb, json=True) - except exceptions.ExecutionError as e: + except AssertionError as e: # NOTE: rare situation when we need to decrease size of a volume. # and we expect this vol to be distributed. g.log.info("Failed to create distributed '%s'Gb volume. " diff --git a/tests/functional/heketi/test_disabling_device.py b/tests/functional/heketi/test_disabling_device.py index c8ec026b..27e50190 100644 --- a/tests/functional/heketi/test_disabling_device.py +++ b/tests/functional/heketi/test_disabling_device.py @@ -2,7 +2,6 @@ from glusto.core import Glusto as g from glustolibs.gluster.volume_ops import get_volume_info from openshiftstoragelibs import baseclass -from openshiftstoragelibs import exceptions from openshiftstoragelibs import heketi_ops from openshiftstoragelibs import podcmd @@ -77,9 +76,10 @@ class TestDisableHeketiDevice(baseclass.BaseClass): self.assertTrue(out, "Failed to get device info %s" % device_id) g.log.info("Successfully retrieved device info %s" % device_id) name = out["name"] - if out["state"].lower().strip() != "offline": - raise exceptions.ExecutionError( - "Device %s is not in offline state." % name) + self.assertEqual( + out["state"].lower().strip(), "offline", + "Device %s is not in offline state." % name + ) g.log.info("Device %s is now offine" % name) # Try to create heketi volume @@ -88,7 +88,7 @@ class TestDisableHeketiDevice(baseclass.BaseClass): out = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 1, json=True) - except exceptions.ExecutionError: + except AssertionError: g.log.info("Volume was not created as expected.") else: self.addCleanup( @@ -111,9 +111,10 @@ class TestDisableHeketiDevice(baseclass.BaseClass): self.assertTrue(out, ("Failed to get device info %s" % device_id)) g.log.info("Successfully retrieved device info %s" % device_id) name = out["name"] - if out["state"] != "online": - raise exceptions.ExecutionError( - "Device %s is not in online state." % name) + self.assertEqual( + out["state"], "online", + "Device %s is not in online state." % name + ) # Create heketi volume of size out = heketi_ops.heketi_volume_create( diff --git a/tests/functional/heketi/test_heketi_create_volume.py b/tests/functional/heketi/test_heketi_create_volume.py index e9679317..3e242667 100644 --- a/tests/functional/heketi/test_heketi_create_volume.py +++ b/tests/functional/heketi/test_heketi_create_volume.py @@ -3,7 +3,6 @@ from glustolibs.gluster.volume_ops import get_volume_list, get_volume_info import six from openshiftstoragelibs.baseclass import BaseClass -from openshiftstoragelibs.exceptions import ExecutionError from openshiftstoragelibs.heketi_ops import ( heketi_blockvolume_create, heketi_blockvolume_delete, @@ -137,7 +136,7 @@ class TestHeketiVolume(BaseClass): " which contains volumes and/or nodes:" " Expected to fail") self.assertRaises( - ExecutionError, + AssertionError, heketi_cluster_delete, self.heketi_client_node, self.heketi_server_url, cluster_id, ) @@ -192,7 +191,7 @@ class TestHeketiVolume(BaseClass): g.log.info("Trying to delete the node which contains devices in it. " "Expecting failure.") self.assertRaises( - ExecutionError, + AssertionError, heketi_node_delete, self.heketi_client_node, heketi_url, node_id) @@ -245,7 +244,7 @@ class TestHeketiVolume(BaseClass): blockvol2 = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, too_big_vol_size, json=True) - except ExecutionError: + except AssertionError: return if blockvol2 and blockvol2.get('id'): diff --git a/tests/functional/heketi/test_heketi_device_operations.py b/tests/functional/heketi/test_heketi_device_operations.py index 76736ecf..eb016941 100644 --- a/tests/functional/heketi/test_heketi_device_operations.py +++ b/tests/functional/heketi/test_heketi_device_operations.py @@ -1,15 +1,7 @@ -try: - # py2/3 - import simplejson as json -except ImportError: - # py2 - import json - import ddt from glusto.core import Glusto as g from openshiftstoragelibs.baseclass import BaseClass -from openshiftstoragelibs.exceptions import ExecutionError from openshiftstoragelibs.heketi_ops import ( heketi_device_add, heketi_device_delete, @@ -150,17 +142,15 @@ class TestHeketiDeviceOperations(BaseClass): self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, online_device_id) - ret, out, err = heketi_volume_create( - self.heketi_client_node, self.heketi_server_url, - vol_size, json=True, raw_cli_output=True) - if ret == 0: + with self.assertRaises(AssertionError): + out = heketi_volume_create( + self.heketi_client_node, self.heketi_server_url, + vol_size, json=True) self.addCleanup( heketi_volume_delete, self.heketi_client_node, - self.heketi_server_url, json.loads(out)["id"]) - self.assertNotEqual(ret, 0, - ("Volume creation did not fail. ret- %s " - "out- %s err- %s" % (ret, out, err))) - g.log.info("Volume creation failed as expected, err- %s", err) + self.heketi_server_url, out["id"]) + self.assertFalse(True, "Volume creation didn't fail: %s" % out) + g.log.info("Volume creation failed as expected") # Enable back the device which was previously disabled g.log.info("Going to enable device id %s", online_device_id) @@ -267,18 +257,16 @@ class TestHeketiDeviceOperations(BaseClass): self.skipTest(skip_msg) g.log.info("Removing device id %s" % lowest_device_id) - ret, out, err = heketi_device_remove( - self.heketi_client_node, self.heketi_server_url, - lowest_device_id, raw_cli_output=True) - if ret == 0: + with self.assertRaises(AssertionError): + out = heketi_device_remove( + self.heketi_client_node, self.heketi_server_url, + lowest_device_id) self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) - self.assertNotEqual(ret, 0, ( - "Device removal did not fail. ret: %s, out: %s, err: %s." % ( - ret, out, err))) - g.log.info("Device removal failed as expected, err- %s", err) + self.assertFalse(True, "Device removal didn't fail: %s" % out) + g.log.info("Device removal failed as expected") # Need to disable device before removing heketi_device_disable( @@ -414,7 +402,7 @@ class TestHeketiDeviceOperations(BaseClass): heketi_device_enable, heketi_node, heketi_url, device_id) try: self.assertRaises( - ExecutionError, heketi_device_remove, + AssertionError, heketi_device_remove, heketi_node, heketi_url, device_id) except Exception: self.addCleanup( diff --git a/tests/functional/heketi/test_heketi_metrics.py b/tests/functional/heketi/test_heketi_metrics.py index f6601074..23351aea 100644 --- a/tests/functional/heketi/test_heketi_metrics.py +++ b/tests/functional/heketi/test_heketi_metrics.py @@ -1,5 +1,4 @@ from openshiftstoragelibs.baseclass import BaseClass -from openshiftstoragelibs import exceptions from openshiftstoragelibs.heketi_ops import ( get_heketi_metrics, heketi_cluster_info, @@ -183,7 +182,7 @@ class TestHeketiMetrics(BaseClass): self.heketi_dc_name, pod_amount=1) # verify that metrics is not accessable when heketi pod is down - with self.assertRaises(exceptions.ExecutionError): + with self.assertRaises(AssertionError): get_heketi_metrics( self.heketi_client_node, self.heketi_server_url, diff --git a/tests/functional/heketi/test_node_enable_disable.py b/tests/functional/heketi/test_node_enable_disable.py index e1e416fc..2d88ffb7 100644 --- a/tests/functional/heketi/test_node_enable_disable.py +++ b/tests/functional/heketi/test_node_enable_disable.py @@ -1,11 +1,4 @@ """Test cases to disable and enable node in heketi.""" -try: - # py2/3 - import simplejson as json -except ImportError: - # py2 - import json - from glusto.core import Glusto as g from openshiftstoragelibs.baseclass import BaseClass @@ -126,19 +119,16 @@ class TestHeketiNodeState(BaseClass): self.addCleanup(self.enable_node, node_id) # try to create a volume, volume creation should fail - ret, out, err = heketi_volume_create( - self.heketi_client_node, self.heketi_server_url, - vol_size, raw_cli_output=True) - if ret == 0: - out_json = json.loads(out) + with self.assertRaises(AssertionError): + out = heketi_volume_create( + self.heketi_client_node, self.heketi_server_url, vol_size) self.addCleanup( heketi_volume_delete, self.heketi_client_node, - self.heketi_server_url, out_json["id"]) - self.assertNotEqual(ret, 0, - ("Volume creation did not fail ret- %s " - "out- %s err- %s" % (ret, out, err))) + self.heketi_server_url, out["id"]) + self.assertFalse(True, "Volume creation didn't fail: %s" % out) + + g.log.info("Volume creation failed as expected.") - g.log.info("Volume creation failed as expected, err- %s", err) # enable node self.enable_node(node_id) diff --git a/tests/functional/heketi/test_volume_creation.py b/tests/functional/heketi/test_volume_creation.py index f8ca5d81..b2a3bdb4 100644 --- a/tests/functional/heketi/test_volume_creation.py +++ b/tests/functional/heketi/test_volume_creation.py @@ -2,7 +2,6 @@ from glusto.core import Glusto as g from glustolibs.gluster import volume_ops from openshiftstoragelibs.baseclass import BaseClass -from openshiftstoragelibs import exceptions from openshiftstoragelibs import heketi_ops from openshiftstoragelibs import podcmd @@ -137,7 +136,7 @@ class TestVolumeCreationTestCases(BaseClass): try: vol_fail = heketi_ops.heketi_volume_create( node, server_url, min_space_gb, json=True) - except exceptions.ExecutionError: + except AssertionError: g.log.info("Volume was not created as expected.") else: self.addCleanup( diff --git a/tests/functional/heketi/test_volume_deletion.py b/tests/functional/heketi/test_volume_deletion.py index 7b1f2ded..97c4924d 100644 --- a/tests/functional/heketi/test_volume_deletion.py +++ b/tests/functional/heketi/test_volume_deletion.py @@ -53,17 +53,11 @@ class TestVolumeDeleteTestCases(BaseClass): "Free space is not reclaimed after deletion of %s" % volume_id) def test_delete_heketidb_volume(self): - """ - Method to test heketidb volume deletion via heketi-cli - """ - heketidbexists = False - msg = "Error: Cannot delete volume containing the Heketi database" - + """Method to test heketidb volume deletion via heketi-cli.""" for i in range(0, 2): volume_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) - self.addCleanup( heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_info["id"]) @@ -72,8 +66,8 @@ class TestVolumeDeleteTestCases(BaseClass): self.heketi_client_node, self.heketi_server_url, json=True) - if volume_list_info["volumes"] == []: - raise ExecutionError("Heketi volume list empty") + self.assertTrue( + volume_list_info["volumes"], "Heketi volume list empty.") for volume_id in volume_list_info["volumes"]: volume_info = heketi_ops.heketi_volume_info( @@ -81,18 +75,10 @@ class TestVolumeDeleteTestCases(BaseClass): volume_id, json=True) if volume_info["name"] == "heketidbstorage": - heketidbexists = True - delete_ret, delete_output, delete_error = ( - heketi_ops.heketi_volume_delete( - self.heketi_client_node, - self.heketi_server_url, volume_id, - raw_cli_output=True)) - - self.assertNotEqual(delete_ret, 0, "Return code not 0") - self.assertEqual( - delete_error.strip(), msg, - "Invalid reason for heketidb deletion failure") - - if not heketidbexists: - raise ExecutionError( - "Warning: heketidbstorage doesn't exist in list of volumes") + self.assertRaises( + AssertionError, + heketi_ops.heketi_volume_delete, + self.heketi_client_node, self.heketi_server_url, volume_id) + return + raise ExecutionError( + "Warning: heketidbstorage doesn't exist in list of volumes") diff --git a/tests/functional/heketi/test_volume_expansion_and_devices.py b/tests/functional/heketi/test_volume_expansion_and_devices.py index d87c18e5..830f49f5 100644 --- a/tests/functional/heketi/test_volume_expansion_and_devices.py +++ b/tests/functional/heketi/test_volume_expansion_and_devices.py @@ -5,7 +5,6 @@ from glusto.core import Glusto as g from glustolibs.gluster import volume_ops, rebalance_ops from openshiftstoragelibs.baseclass import BaseClass -from openshiftstoragelibs.exceptions import ExecutionError from openshiftstoragelibs import ( heketi_ops, podcmd, @@ -404,7 +403,7 @@ class TestVolumeExpansionAndDevicesTestCases(BaseClass): # Try to expand volume with not enough device space self.assertRaises( - ExecutionError, heketi_ops.heketi_volume_expand, + AssertionError, heketi_ops.heketi_volume_expand, h_node, h_server_url, volume_id, expand_size) # Enable new devices to be able to expand our volume |