From 32724906d1fd5fd09759e1daf927594461c26d5e Mon Sep 17 00:00:00 2001 From: Valerii Ponomarov Date: Mon, 24 Jun 2019 01:29:41 +0530 Subject: Make Heketi commands run on a Heketi POD when main command fails It happens that heketi client located out of the Heketi POD may fail not reaching the server side. So, add back-up approach where we run Heketi commands on a Heketi POD when main commands fail. Change-Id: Ie6ae5be82082f34426f9288b02575e3abd4940f5 --- ...est_create_distributed_replica_heketi_volume.py | 3 +- tests/functional/heketi/test_disabling_device.py | 17 ++++----- .../functional/heketi/test_heketi_create_volume.py | 7 ++-- .../heketi/test_heketi_device_operations.py | 40 ++++++++-------------- tests/functional/heketi/test_heketi_metrics.py | 3 +- .../functional/heketi/test_node_enable_disable.py | 24 ++++--------- tests/functional/heketi/test_volume_creation.py | 3 +- tests/functional/heketi/test_volume_deletion.py | 34 ++++++------------ .../heketi/test_volume_expansion_and_devices.py | 3 +- 9 files changed, 47 insertions(+), 87 deletions(-) (limited to 'tests') diff --git a/tests/functional/heketi/test_create_distributed_replica_heketi_volume.py b/tests/functional/heketi/test_create_distributed_replica_heketi_volume.py index 8a3beb29..f5c41d8d 100644 --- a/tests/functional/heketi/test_create_distributed_replica_heketi_volume.py +++ b/tests/functional/heketi/test_create_distributed_replica_heketi_volume.py @@ -5,7 +5,6 @@ from glusto.core import Glusto as g from glustolibs.gluster.volume_ops import get_volume_list, get_volume_info from openshiftstoragelibs.baseclass import BaseClass -from openshiftstoragelibs import exceptions from openshiftstoragelibs.heketi_ops import ( heketi_device_disable, heketi_device_enable, @@ -102,7 +101,7 @@ class TestHeketiVolume(BaseClass): "Trying to create distributed '%s'Gb volume." % vol_size_gb) heketi_vol = heketi_volume_create( self.heketi_client_node, heketi_url, vol_size_gb, json=True) - except exceptions.ExecutionError as e: + except AssertionError as e: # NOTE: rare situation when we need to decrease size of a volume. # and we expect this vol to be distributed. g.log.info("Failed to create distributed '%s'Gb volume. " diff --git a/tests/functional/heketi/test_disabling_device.py b/tests/functional/heketi/test_disabling_device.py index c8ec026b..27e50190 100644 --- a/tests/functional/heketi/test_disabling_device.py +++ b/tests/functional/heketi/test_disabling_device.py @@ -2,7 +2,6 @@ from glusto.core import Glusto as g from glustolibs.gluster.volume_ops import get_volume_info from openshiftstoragelibs import baseclass -from openshiftstoragelibs import exceptions from openshiftstoragelibs import heketi_ops from openshiftstoragelibs import podcmd @@ -77,9 +76,10 @@ class TestDisableHeketiDevice(baseclass.BaseClass): self.assertTrue(out, "Failed to get device info %s" % device_id) g.log.info("Successfully retrieved device info %s" % device_id) name = out["name"] - if out["state"].lower().strip() != "offline": - raise exceptions.ExecutionError( - "Device %s is not in offline state." % name) + self.assertEqual( + out["state"].lower().strip(), "offline", + "Device %s is not in offline state." % name + ) g.log.info("Device %s is now offine" % name) # Try to create heketi volume @@ -88,7 +88,7 @@ class TestDisableHeketiDevice(baseclass.BaseClass): out = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 1, json=True) - except exceptions.ExecutionError: + except AssertionError: g.log.info("Volume was not created as expected.") else: self.addCleanup( @@ -111,9 +111,10 @@ class TestDisableHeketiDevice(baseclass.BaseClass): self.assertTrue(out, ("Failed to get device info %s" % device_id)) g.log.info("Successfully retrieved device info %s" % device_id) name = out["name"] - if out["state"] != "online": - raise exceptions.ExecutionError( - "Device %s is not in online state." % name) + self.assertEqual( + out["state"], "online", + "Device %s is not in online state." % name + ) # Create heketi volume of size out = heketi_ops.heketi_volume_create( diff --git a/tests/functional/heketi/test_heketi_create_volume.py b/tests/functional/heketi/test_heketi_create_volume.py index e9679317..3e242667 100644 --- a/tests/functional/heketi/test_heketi_create_volume.py +++ b/tests/functional/heketi/test_heketi_create_volume.py @@ -3,7 +3,6 @@ from glustolibs.gluster.volume_ops import get_volume_list, get_volume_info import six from openshiftstoragelibs.baseclass import BaseClass -from openshiftstoragelibs.exceptions import ExecutionError from openshiftstoragelibs.heketi_ops import ( heketi_blockvolume_create, heketi_blockvolume_delete, @@ -137,7 +136,7 @@ class TestHeketiVolume(BaseClass): " which contains volumes and/or nodes:" " Expected to fail") self.assertRaises( - ExecutionError, + AssertionError, heketi_cluster_delete, self.heketi_client_node, self.heketi_server_url, cluster_id, ) @@ -192,7 +191,7 @@ class TestHeketiVolume(BaseClass): g.log.info("Trying to delete the node which contains devices in it. " "Expecting failure.") self.assertRaises( - ExecutionError, + AssertionError, heketi_node_delete, self.heketi_client_node, heketi_url, node_id) @@ -245,7 +244,7 @@ class TestHeketiVolume(BaseClass): blockvol2 = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, too_big_vol_size, json=True) - except ExecutionError: + except AssertionError: return if blockvol2 and blockvol2.get('id'): diff --git a/tests/functional/heketi/test_heketi_device_operations.py b/tests/functional/heketi/test_heketi_device_operations.py index 76736ecf..eb016941 100644 --- a/tests/functional/heketi/test_heketi_device_operations.py +++ b/tests/functional/heketi/test_heketi_device_operations.py @@ -1,15 +1,7 @@ -try: - # py2/3 - import simplejson as json -except ImportError: - # py2 - import json - import ddt from glusto.core import Glusto as g from openshiftstoragelibs.baseclass import BaseClass -from openshiftstoragelibs.exceptions import ExecutionError from openshiftstoragelibs.heketi_ops import ( heketi_device_add, heketi_device_delete, @@ -150,17 +142,15 @@ class TestHeketiDeviceOperations(BaseClass): self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, online_device_id) - ret, out, err = heketi_volume_create( - self.heketi_client_node, self.heketi_server_url, - vol_size, json=True, raw_cli_output=True) - if ret == 0: + with self.assertRaises(AssertionError): + out = heketi_volume_create( + self.heketi_client_node, self.heketi_server_url, + vol_size, json=True) self.addCleanup( heketi_volume_delete, self.heketi_client_node, - self.heketi_server_url, json.loads(out)["id"]) - self.assertNotEqual(ret, 0, - ("Volume creation did not fail. ret- %s " - "out- %s err- %s" % (ret, out, err))) - g.log.info("Volume creation failed as expected, err- %s", err) + self.heketi_server_url, out["id"]) + self.assertFalse(True, "Volume creation didn't fail: %s" % out) + g.log.info("Volume creation failed as expected") # Enable back the device which was previously disabled g.log.info("Going to enable device id %s", online_device_id) @@ -267,18 +257,16 @@ class TestHeketiDeviceOperations(BaseClass): self.skipTest(skip_msg) g.log.info("Removing device id %s" % lowest_device_id) - ret, out, err = heketi_device_remove( - self.heketi_client_node, self.heketi_server_url, - lowest_device_id, raw_cli_output=True) - if ret == 0: + with self.assertRaises(AssertionError): + out = heketi_device_remove( + self.heketi_client_node, self.heketi_server_url, + lowest_device_id) self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) - self.assertNotEqual(ret, 0, ( - "Device removal did not fail. ret: %s, out: %s, err: %s." % ( - ret, out, err))) - g.log.info("Device removal failed as expected, err- %s", err) + self.assertFalse(True, "Device removal didn't fail: %s" % out) + g.log.info("Device removal failed as expected") # Need to disable device before removing heketi_device_disable( @@ -414,7 +402,7 @@ class TestHeketiDeviceOperations(BaseClass): heketi_device_enable, heketi_node, heketi_url, device_id) try: self.assertRaises( - ExecutionError, heketi_device_remove, + AssertionError, heketi_device_remove, heketi_node, heketi_url, device_id) except Exception: self.addCleanup( diff --git a/tests/functional/heketi/test_heketi_metrics.py b/tests/functional/heketi/test_heketi_metrics.py index f6601074..23351aea 100644 --- a/tests/functional/heketi/test_heketi_metrics.py +++ b/tests/functional/heketi/test_heketi_metrics.py @@ -1,5 +1,4 @@ from openshiftstoragelibs.baseclass import BaseClass -from openshiftstoragelibs import exceptions from openshiftstoragelibs.heketi_ops import ( get_heketi_metrics, heketi_cluster_info, @@ -183,7 +182,7 @@ class TestHeketiMetrics(BaseClass): self.heketi_dc_name, pod_amount=1) # verify that metrics is not accessable when heketi pod is down - with self.assertRaises(exceptions.ExecutionError): + with self.assertRaises(AssertionError): get_heketi_metrics( self.heketi_client_node, self.heketi_server_url, diff --git a/tests/functional/heketi/test_node_enable_disable.py b/tests/functional/heketi/test_node_enable_disable.py index e1e416fc..2d88ffb7 100644 --- a/tests/functional/heketi/test_node_enable_disable.py +++ b/tests/functional/heketi/test_node_enable_disable.py @@ -1,11 +1,4 @@ """Test cases to disable and enable node in heketi.""" -try: - # py2/3 - import simplejson as json -except ImportError: - # py2 - import json - from glusto.core import Glusto as g from openshiftstoragelibs.baseclass import BaseClass @@ -126,19 +119,16 @@ class TestHeketiNodeState(BaseClass): self.addCleanup(self.enable_node, node_id) # try to create a volume, volume creation should fail - ret, out, err = heketi_volume_create( - self.heketi_client_node, self.heketi_server_url, - vol_size, raw_cli_output=True) - if ret == 0: - out_json = json.loads(out) + with self.assertRaises(AssertionError): + out = heketi_volume_create( + self.heketi_client_node, self.heketi_server_url, vol_size) self.addCleanup( heketi_volume_delete, self.heketi_client_node, - self.heketi_server_url, out_json["id"]) - self.assertNotEqual(ret, 0, - ("Volume creation did not fail ret- %s " - "out- %s err- %s" % (ret, out, err))) + self.heketi_server_url, out["id"]) + self.assertFalse(True, "Volume creation didn't fail: %s" % out) + + g.log.info("Volume creation failed as expected.") - g.log.info("Volume creation failed as expected, err- %s", err) # enable node self.enable_node(node_id) diff --git a/tests/functional/heketi/test_volume_creation.py b/tests/functional/heketi/test_volume_creation.py index f8ca5d81..b2a3bdb4 100644 --- a/tests/functional/heketi/test_volume_creation.py +++ b/tests/functional/heketi/test_volume_creation.py @@ -2,7 +2,6 @@ from glusto.core import Glusto as g from glustolibs.gluster import volume_ops from openshiftstoragelibs.baseclass import BaseClass -from openshiftstoragelibs import exceptions from openshiftstoragelibs import heketi_ops from openshiftstoragelibs import podcmd @@ -137,7 +136,7 @@ class TestVolumeCreationTestCases(BaseClass): try: vol_fail = heketi_ops.heketi_volume_create( node, server_url, min_space_gb, json=True) - except exceptions.ExecutionError: + except AssertionError: g.log.info("Volume was not created as expected.") else: self.addCleanup( diff --git a/tests/functional/heketi/test_volume_deletion.py b/tests/functional/heketi/test_volume_deletion.py index 7b1f2ded..97c4924d 100644 --- a/tests/functional/heketi/test_volume_deletion.py +++ b/tests/functional/heketi/test_volume_deletion.py @@ -53,17 +53,11 @@ class TestVolumeDeleteTestCases(BaseClass): "Free space is not reclaimed after deletion of %s" % volume_id) def test_delete_heketidb_volume(self): - """ - Method to test heketidb volume deletion via heketi-cli - """ - heketidbexists = False - msg = "Error: Cannot delete volume containing the Heketi database" - + """Method to test heketidb volume deletion via heketi-cli.""" for i in range(0, 2): volume_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) - self.addCleanup( heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_info["id"]) @@ -72,8 +66,8 @@ class TestVolumeDeleteTestCases(BaseClass): self.heketi_client_node, self.heketi_server_url, json=True) - if volume_list_info["volumes"] == []: - raise ExecutionError("Heketi volume list empty") + self.assertTrue( + volume_list_info["volumes"], "Heketi volume list empty.") for volume_id in volume_list_info["volumes"]: volume_info = heketi_ops.heketi_volume_info( @@ -81,18 +75,10 @@ class TestVolumeDeleteTestCases(BaseClass): volume_id, json=True) if volume_info["name"] == "heketidbstorage": - heketidbexists = True - delete_ret, delete_output, delete_error = ( - heketi_ops.heketi_volume_delete( - self.heketi_client_node, - self.heketi_server_url, volume_id, - raw_cli_output=True)) - - self.assertNotEqual(delete_ret, 0, "Return code not 0") - self.assertEqual( - delete_error.strip(), msg, - "Invalid reason for heketidb deletion failure") - - if not heketidbexists: - raise ExecutionError( - "Warning: heketidbstorage doesn't exist in list of volumes") + self.assertRaises( + AssertionError, + heketi_ops.heketi_volume_delete, + self.heketi_client_node, self.heketi_server_url, volume_id) + return + raise ExecutionError( + "Warning: heketidbstorage doesn't exist in list of volumes") diff --git a/tests/functional/heketi/test_volume_expansion_and_devices.py b/tests/functional/heketi/test_volume_expansion_and_devices.py index d87c18e5..830f49f5 100644 --- a/tests/functional/heketi/test_volume_expansion_and_devices.py +++ b/tests/functional/heketi/test_volume_expansion_and_devices.py @@ -5,7 +5,6 @@ from glusto.core import Glusto as g from glustolibs.gluster import volume_ops, rebalance_ops from openshiftstoragelibs.baseclass import BaseClass -from openshiftstoragelibs.exceptions import ExecutionError from openshiftstoragelibs import ( heketi_ops, podcmd, @@ -404,7 +403,7 @@ class TestVolumeExpansionAndDevicesTestCases(BaseClass): # Try to expand volume with not enough device space self.assertRaises( - ExecutionError, heketi_ops.heketi_volume_expand, + AssertionError, heketi_ops.heketi_volume_expand, h_node, h_server_url, volume_id, expand_size) # Enable new devices to be able to expand our volume -- cgit