diff options
5 files changed, 83 insertions, 49 deletions
diff --git a/cns-libs/cnslibs/common/openshift_ops.py b/cns-libs/cnslibs/common/openshift_ops.py index 523cc375..6cfff3f8 100644 --- a/cns-libs/cnslibs/common/openshift_ops.py +++ b/cns-libs/cnslibs/common/openshift_ops.py @@ -32,6 +32,7 @@ PODS_WIDE_RE = re.compile( SERVICE_STATUS = "systemctl status %s" SERVICE_RESTART = "systemctl restart %s" SERVICE_STATUS_REGEX = "Active: active \((.*)\) since .*;.*" +OC_VERSION = None def oc_get_pods(ocp_node): @@ -470,7 +471,17 @@ def oc_delete(ocp_node, rtype, name, raise_on_absence=True): if not oc_get_yaml(ocp_node, rtype, name, raise_on_error=raise_on_absence): return - ret, out, err = g.run(ocp_node, ['oc', 'delete', rtype, name]) + cmd = ['oc', 'delete', rtype, name] + + global OC_VERSION + if not OC_VERSION: + OC_VERSION = oc_version(ocp_node) + + versions = ['v3.6', 'v3.7', 'v3.9', 'v3.10'] + if not OC_VERSION.rsplit('.', 1)[0] in versions: + cmd.append('--wait=false') + + ret, out, err = g.run(ocp_node, cmd) if ret != 0: g.log.error('Failed to delete resource: %s, %s: %r; %r', rtype, name, out, err) @@ -621,11 +632,12 @@ def create_namespace(hostname, namespace): def wait_for_resource_absence(ocp_node, rtype, name, - interval=10, timeout=120): + interval=5, timeout=300): _waiter = waiter.Waiter(timeout=timeout, interval=interval) + resource, pv_name = None, None for w in _waiter: try: - oc_get_yaml(ocp_node, rtype, name, raise_on_error=True) + resource = oc_get_yaml(ocp_node, rtype, name, raise_on_error=True) except AssertionError: break if rtype == 'pvc': @@ -633,11 +645,29 @@ def wait_for_resource_absence(ocp_node, rtype, name, name) for w in _waiter: ret, out, err = g.run(ocp_node, cmd, "root") + _pv_name = out.strip() + if _pv_name and not pv_name: + pv_name = _pv_name if ret != 0: break if w.expired: - error_msg = "%s '%s' still exists after waiting for it %d seconds" % ( - rtype, name, timeout) + # Gather more info for ease of debugging + try: + r_events = get_events(ocp_node, obj_name=name) + except Exception: + r_events = '?' + error_msg = ( + "%s '%s' still exists after waiting for it %d seconds.\n" + "Resource info: %s\n" + "Resource related events: %s" % ( + rtype, name, timeout, resource, r_events)) + if rtype == 'pvc' and pv_name: + try: + pv_events = get_events(ocp_node, obj_name=pv_name) + except Exception: + pv_events = '?' + error_msg += "\nPV events: %s" % pv_events + g.log.error(error_msg) raise exceptions.ExecutionError(error_msg) diff --git a/cns-libs/cnslibs/common/waiter.py b/cns-libs/cnslibs/common/waiter.py index 0a6e72b5..0d0c8c3a 100644 --- a/cns-libs/cnslibs/common/waiter.py +++ b/cns-libs/cnslibs/common/waiter.py @@ -33,3 +33,6 @@ class Waiter(object): time.sleep(self.interval) self._attempt += 1 return self + + # NOTE(vponomar): py3 uses "__next__" method instead of "next" one. + __next__ = next diff --git a/tests/functional/common/arbiter/test_arbiter.py b/tests/functional/common/arbiter/test_arbiter.py index 8b86c640..2567483c 100644 --- a/tests/functional/common/arbiter/test_arbiter.py +++ b/tests/functional/common/arbiter/test_arbiter.py @@ -196,7 +196,7 @@ class TestArbiterVolumeCreateExpandDelete(cns_baseclass.CnsBaseClass): total_size, max_size, "Volume has bigger size '%s' than expected - '%s'." % ( total_size, max_size)) - min_available_size = int(max_size * 0.95) + min_available_size = int(max_size * 0.93) available_size = int(data[4]) self.assertLessEqual( min_available_size, available_size, diff --git a/tests/functional/common/heketi/heketi_tests/test_heketi_create_volume.py b/tests/functional/common/heketi/heketi_tests/test_heketi_create_volume.py index c28f455b..dbb72e9b 100644 --- a/tests/functional/common/heketi/heketi_tests/test_heketi_create_volume.py +++ b/tests/functional/common/heketi/heketi_tests/test_heketi_create_volume.py @@ -1,4 +1,4 @@ -from glustolibs.gluster.exceptions import ExecutionError, ConfigError +from glustolibs.gluster.exceptions import ExecutionError from glusto.core import Glusto as g from glustolibs.gluster.volume_ops import get_volume_list, get_volume_info @@ -156,47 +156,52 @@ class TestHeketiVolume(HeketiClientSetupBaseClass): g.log.info("All heketi cluster successfully listed") def test_to_check_deletion_of_node(self): - """ - Deletion of a node which contains devices - """ - - # List of heketi node - g.log.info("List heketi nodes") - heketi_node_id_list = heketi_node_list( - self.heketi_client_node, self.heketi_server_url) - self.assertTrue(heketi_node_id_list, ("List of node IDs is empty.")) + """Deletion of a node which contains devices""" + + # Create Heketi volume to make sure we have devices with usages + heketi_url = self.heketi_server_url + vol = heketi_volume_create( + self.heketi_client_node, heketi_url, 1, json=True) + self.assertTrue(vol, "Failed to create heketi volume.") + g.log.info("Heketi volume successfully created") + volume_id = vol["bricks"][0]["volume"] + self.addCleanup(self.delete_volumes, volume_id) - g.log.info("Successfully got the list of nodes") - for node_id in heketi_node_id_list: - g.log.info("Retrieve the node info") - node_info_dict = heketi_node_info( - self.heketi_client_node, self.heketi_server_url, - node_id, json=True) - if not(node_info_dict["devices"][1]["storage"]["used"]): - raise ConfigError("No device in node %s" % node_id) - g.log.info("Used space in device %s" % node_info_dict[ - "devices"][1]["storage"]["used"]) - node_id = heketi_node_id_list[0] - - # Deleting a node - g.log.info("Trying to delete a node which" - " contains devices in it:" - " Expected to fail") - out = heketi_node_delete(self.heketi_client_node, - self.heketi_server_url, - node_id) - self.assertFalse(out, ("Successfully deletes a " - "node %s" % str(node_id))) - g.log.info("Expected result: Unable to delete " - "node %s because it contains devices") - - # To confrim deletion failed, check node list - # TODO: fix following, it doesn't verify absence of the deleted nodes + # Pick up suitable node + node_ids = heketi_node_list(self.heketi_client_node, heketi_url) + self.assertTrue(node_ids) + for node_id in node_ids: + node_info = heketi_node_info( + self.heketi_client_node, heketi_url, node_id, json=True) + if (node_info['state'].lower() != 'online' or + not node_info['devices']): + continue + for device in node_info['devices']: + if device['state'].lower() != 'online': + continue + if device['storage']['used']: + node_id = node_info['id'] + break + else: + self.assertTrue( + node_id, + "Failed to find online node with online device which " + "has some usages.") + + # Try to delete the node by its ID + g.log.info("Trying to delete the node which contains devices in it. " + "Expecting failure.") + out = heketi_node_delete(self.heketi_client_node, heketi_url, node_id) + self.assertFalse(out, "Node '%s' got unexpectedly deleted." % node_id) + + # Make sure our node hasn't been deleted g.log.info("Listing heketi node list") - node_list = heketi_node_list(self.heketi_client_node, - self.heketi_server_url) + node_list = heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_list, ("Failed to list heketi nodes")) - g.log.info("Successfully got the list of nodes") + self.assertIn(node_id, node_list) + node_info = heketi_node_info( + self.heketi_client_node, heketi_url, node_id, json=True) + self.assertEqual(node_info['state'].lower(), 'online') def test_blockvolume_create_no_free_space(self): """Test case CNS-550""" diff --git a/tests/functional/common/provisioning/test_dynamic_provisioning_p0_cases.py b/tests/functional/common/provisioning/test_dynamic_provisioning_p0_cases.py index 9875e6dd..2f43ff1e 100644 --- a/tests/functional/common/provisioning/test_dynamic_provisioning_p0_cases.py +++ b/tests/functional/common/provisioning/test_dynamic_provisioning_p0_cases.py @@ -19,7 +19,6 @@ from cnslibs.common.openshift_ops import ( oc_delete, oc_get_custom_resource, oc_rsh, - oc_version, scale_dc_pod_amount_and_wait, verify_pvc_status_is_bound, wait_for_pod_be_ready, @@ -422,9 +421,6 @@ class TestDynamicProvisioningP0(CnsBaseClass): def test_pvc_deletion_while_pod_is_running(self): # CNS-584 Verify PVC deletion while pod is running - if "v3.11" in oc_version(self.node): - self.skipTest("Blocked by BZ-1644696") - self._create_storage_class() self._create_and_wait_for_pvc() |