From 9a79735144ec8caad346feb6897d9e78dfea1a29 Mon Sep 17 00:00:00 2001 From: vamahaja Date: Wed, 10 Jul 2019 22:26:42 +0530 Subject: Fix test cases skipped due to closed bugs Add fix to test cases which are skipped due to bugs - BZ-1644685, BZ-1652913, BZ-1634745, BZ-1635736, BZ-1636477, BZ-1641668 Change-Id: I03a7f7dddaba5d3e41770969352b2263c84cb920 Signed-off-by: vamahaja --- .../openshiftstoragelibs/gluster_ops.py | 2 +- .../openshiftstoragelibs/heketi_ops.py | 97 ++++++++++++++++++++++ .../test_restart_gluster_services.py | 29 ++++++- .../provisioning/test_storage_class_cases.py | 15 ++-- 4 files changed, 136 insertions(+), 7 deletions(-) diff --git a/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py b/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py index 798e3c2a..950fd077 100644 --- a/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py +++ b/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py @@ -96,7 +96,7 @@ def restart_gluster_vol_brick_processes(ocp_client_node, file_vol, # Get Gluster vol brick PIDs gluster_volume_status = get_gluster_vol_status(file_vol) - pids = () + pids = [] for gluster_node in gluster_nodes: pid = None for g_node, g_node_data in gluster_volume_status.items(): diff --git a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py index 0021e79b..78c2e073 100644 --- a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py +++ b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py @@ -12,10 +12,14 @@ from glusto.core import Glusto as g import six from openshiftstoragelibs import command +from openshiftstoragelibs import exceptions from openshiftstoragelibs import heketi_version from openshiftstoragelibs.utils import parse_prometheus_data +from openshiftstoragelibs import waiter + HEKETI_BHV = re.compile(r"Id:(\S+)\s+Cluster:(\S+)\s+Name:(\S+)\s\[block\]") +HEKETI_OPERATIONS = re.compile(r"Id:(\S+)\s+Type:(\S+)\s+Status:(\S+)") HEKETI_COMMAND_TIMEOUT = g.config.get("common", {}).get( "heketi_command_timeout", 120) TIMEOUT_PREFIX = "timeout %s " % HEKETI_COMMAND_TIMEOUT @@ -1446,3 +1450,96 @@ def get_total_free_space(heketi_client_node, heketi_server_url): total_device_free_space += (device["storage"]["free"]) device_free_spaces.append(total_device_free_space / 1024 ** 2) return int(sum(device_free_spaces)), len(device_free_spaces) + + +def heketi_server_operations_list( + heketi_client_node, heketi_server_url, **kwargs): + """Executes heketi server operations list command. + + Args: + heketi_client_node (str): Node on which cmd has to be executed. + heketi_server_url (str): Heketi server url + + Returns: + list: list of server operations pending + + Raises: + exceptions.ExecutionError: if command fails. + """ + version = heketi_version.get_heketi_version(heketi_client_node) + if version < '8.0.0-10': + msg = ( + "heketi-client package %s does not support operations " + "list functionality" % version.v_str) + g.log.error(msg) + raise NotImplementedError(msg) + + heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags( + heketi_server_url, **kwargs) + + cmd = "heketi-cli -s %s %s %s server operations list" % ( + heketi_server_url, admin_key, user) + cmd = TIMEOUT_PREFIX + cmd + out = heketi_cmd_run(heketi_client_node, cmd) + if out: + operations = [] + for operation in HEKETI_OPERATIONS.findall(out.strip()): + operations.append({ + 'id': operation[0], + 'type': operation[1], + 'status': operation[2] + }) + return operations + else: + g.log.info("No any pendig heketi server operation") + return [] + + +def heketi_server_operation_cleanup( + heketi_client_node, heketi_server_url, operation_id=None, + timeout=120, wait_time=5, **kwargs): + """Executes heketi server operations cleanup command and wait until + cleanup operations get completed for given timeout. + + Args: + heketi_client_node (str): Node on which cmd has to be executed. + heketi_server_url (str): Heketi server url + operation_id (str): Operation Id needs to be cleaned. + + Raises: + exceptions.ExecutionError: If cleanup not completed in given timeout. + """ + version = heketi_version.get_heketi_version(heketi_client_node) + if version < '8.0.0-10': + msg = ( + "heketi-client package %s does not support operations " + "cleanup functionality" % version.v_str) + g.log.error(msg) + raise NotImplementedError(msg) + + heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags( + heketi_server_url, **kwargs) + cmd = "heketi-cli -s %s %s %s server operations cleanup" % ( + heketi_server_url, admin_key, user) + if operation_id: + cmd += " %s" % operation_id + + cmd = TIMEOUT_PREFIX + cmd + heketi_cmd_run(heketi_client_node, cmd) + for w in waiter.Waiter(timeout=timeout, interval=wait_time): + cleanup_operations = heketi_server_operations_list( + heketi_client_node, heketi_server_url, **kwargs) + + cleanup_operation = [ + operation["id"] + for operation in cleanup_operations + if operation["id"] == operation_id] + if not cleanup_operation: + break + + if w.expired: + err_msg = ( + "Heketi server cleanup operation still pending even " + "after %s second" % timeout) + g.log.error(err_msg) + raise exceptions.ExecutionError(err_msg) diff --git a/tests/functional/gluster_stability/test_restart_gluster_services.py b/tests/functional/gluster_stability/test_restart_gluster_services.py index b68da6b6..85069688 100644 --- a/tests/functional/gluster_stability/test_restart_gluster_services.py +++ b/tests/functional/gluster_stability/test_restart_gluster_services.py @@ -15,7 +15,11 @@ from openshiftstoragelibs.gluster_ops import ( restart_gluster_vol_brick_processes, wait_to_heal_complete, ) -from openshiftstoragelibs.heketi_ops import heketi_blockvolume_list +from openshiftstoragelibs.heketi_ops import ( + heketi_blockvolume_list, + heketi_server_operation_cleanup, + heketi_server_operations_list, +) from openshiftstoragelibs.openshift_ops import ( get_pv_name_from_pvc, match_pv_and_heketi_block_volumes, @@ -139,6 +143,17 @@ class GlusterStabilityTestSetup(GlusterBlockBaseClass): def validate_volumes_and_blocks(self): """Validates PVC and block volumes generated through heketi and OCS """ + heketi_operations = heketi_server_operations_list( + self.heketi_client_node, self.heketi_server_url, + secret=self.heketi_cli_key, user=self.heketi_cli_user) + + for heketi_operation in heketi_operations: + if heketi_operation["status"] == "failed": + heketi_server_operation_cleanup( + self.heketi_client_node, self.heketi_server_url, + heketi_operation["id"], secret=self.heketi_cli_key, + user=self.heketi_cli_user + ) # verify pvc status is in "Bound" for all the pvc for pvc in self.pvc_list: @@ -204,9 +219,21 @@ class GlusterStabilityTestSetup(GlusterBlockBaseClass): wait_to_heal_complete() + @skip("Blocked by BZ-1634745, BZ-1635736, BZ-1636477") @ddt.data(SERVICE_BLOCKD, SERVICE_TCMU, SERVICE_TARGET) def test_restart_services_provision_volume_and_run_io(self, service): """Restart gluster service then validate volumes""" + skip_msg = ( + "Skipping this test case due to bugs " + "BZ-1634745, BZ-1635736, BZ-1636477, BZ-1641668") + + # TODO(vamahaja): Add check for CRS version + if not self.is_containerized_gluster(): + self.skipTest(skip_msg + " and not implemented CRS version check") + + if get_openshift_storage_version() < "3.11.2": + self.skipTest(skip_msg) + self.deploy_and_verify_resouces() block_hosting_vol = self.get_block_hosting_volume_by_pvc_name( diff --git a/tests/functional/provisioning/test_storage_class_cases.py b/tests/functional/provisioning/test_storage_class_cases.py index 6cf6e96c..2e319f6c 100644 --- a/tests/functional/provisioning/test_storage_class_cases.py +++ b/tests/functional/provisioning/test_storage_class_cases.py @@ -1,5 +1,3 @@ -from unittest import skip - import ddt from glusto.core import Glusto as g @@ -261,7 +259,6 @@ class TestStorageClassCases(BaseClass): ) self.validate_multipath_info(hacount) - @skip("Blocked by BZ-1644685") def test_gluster_block_provisioning_with_invalid_ha_count(self): """Validate gluster-block provisioning with any invalid 'hacount' value @@ -278,12 +275,20 @@ class TestStorageClassCases(BaseClass): "is not supported in OCS 3.9") # get hacount as no of gluster pods + 1 to fail the pvc creation - hacount = get_amount_of_gluster_nodes(self.ocp_master_node[0]) + 1 + gluster_pod_count = get_amount_of_gluster_nodes( + self.ocp_master_node[0]) + hacount = gluster_pod_count + 1 # create storage class and pvc with given parameters self.create_sc_with_parameter( - 'glusterblock', parameter={'hacount': str(hacount)} + 'glusterblock', success=True, parameter={'hacount': str(hacount)} + ) + + # validate HA parameter with gluster block volume + self.validate_gluster_block_volume_info( + self.assertEqual, 'HA', gluster_pod_count ) + self.validate_multipath_info(gluster_pod_count) @ddt.data('true', 'false', '') def test_gluster_block_chapauthenabled_parameter(self, chapauthenabled): -- cgit