Completely rewrite 'heketi_with_device_removal_insuff_space' tc

Because it was very fragile and skipped due to bug.
So, after bug was fixed we need to enable it back.
But, also, considering it's stability, it required complete change of
logic. So, it is rewritten completely and moved to more appropriate
module where we have other test cases which perform device operations.

Change-Id: I1b8c18c8ea883f83861e81e97c534459d7588175

2 files changed, 90 insertions, 169 deletions

diff --git a/tests/functional/common/heketi/test_heketi_device_operations.py b/tests/functional/common/heketi/test_heketi_device_operations.py
index 718ed80b..3fa90bc3 100644
--- a/tests/functional/common/heketi/test_heketi_device_operations.py
+++ b/tests/functional/common/heketi/test_heketi_device_operations.py
@@ -3,6 +3,7 @@ import json
 import ddt
 from glusto.core import Glusto as g
 
+from cnslibs.common.exceptions import ExecutionError
 from cnslibs.common.heketi_libs import HeketiBaseClass
 from cnslibs.common.heketi_ops import (heketi_node_enable,
                                        heketi_node_info,
@@ -319,3 +320,85 @@ class TestHeketiDeviceOperations(HeketiBaseClass):
             present,
             "Some of the '%s' volume bricks is present of the removed "
             "'%s' device." % (vol_info['id'], lowest_device_id))
+
+    def test_heketi_with_device_removal_insuff_space(self):
+        """Test case CNS-624"""
+
+        # Disable 4+ nodes and 3+ devices on the first 3 nodes
+        min_free_space_gb = 5
+        min_free_space = min_free_space_gb * 1024**2
+        heketi_url = self.heketi_server_url
+        heketi_node = self.heketi_client_node
+        nodes = {}
+
+        node_ids = heketi_node_list(heketi_node, heketi_url)
+        self.assertTrue(node_ids)
+        for node_id in node_ids:
+            node_info = heketi_node_info(
+                heketi_node, heketi_url, node_id, json=True)
+            if (node_info["state"].lower() != "online" or
+                    not node_info["devices"]):
+                continue
+            if len(nodes) > 2:
+                heketi_node_disable(heketi_node, heketi_url, node_id)
+                self.addCleanup(
+                    heketi_node_enable, heketi_node, heketi_url, node_id)
+                continue
+            for device in node_info["devices"]:
+                if device["state"].lower() != "online":
+                    continue
+                free_space = device["storage"]["free"]
+                if node_id not in nodes:
+                    nodes[node_id] = []
+                if (free_space < min_free_space or len(nodes[node_id]) > 1):
+                    heketi_device_disable(
+                        heketi_node, heketi_url, device["id"])
+                    self.addCleanup(
+                        heketi_device_enable,
+                        heketi_node, heketi_url, device["id"])
+                    continue
+                nodes[node_id].append({
+                    "device_id": device["id"], "free": free_space})
+
+        # Skip test if nodes requirements are not met
+        if (len(nodes) < 3 or
+                not all(map((lambda _list: len(_list) > 1), nodes.values()))):
+            raise self.skipTest(
+                "Could not find 3 online nodes with 2 online devices "
+                "having free space bigger than %dGb." % min_free_space_gb)
+
+        # Calculate size of a potential distributed vol
+        if nodes[node_ids[0]][0]["free"] > nodes[node_ids[0]][1]["free"]:
+            index = 0
+        else:
+            index = 1
+        vol_size_gb = int(nodes[node_ids[0]][index]["free"] / (1024 ** 2)) + 1
+        device_id = nodes[node_ids[0]][index]["device_id"]
+
+        # Create volume with such size that we consume space more than
+        # size of smaller disks
+        try:
+            heketi_vol = heketi_volume_create(
+                heketi_node, heketi_url, vol_size_gb, json=True)
+        except Exception as e:
+            g.log.warning(
+                "Got following error trying to create '%s'Gb vol: %s" % (
+                    vol_size_gb, e))
+            vol_size_gb -= 1
+            heketi_vol = heketi_volume_create(
+                heketi_node, heketi_url, vol_size_gb, json=True)
+        self.addCleanup(self.delete_volumes, heketi_vol["bricks"][0]["volume"])
+
+        # Try to 'remove' bigger Heketi disk expecting error,
+        # because there is no space on smaller disk to relocate bricks to
+        heketi_device_disable(heketi_node, heketi_url, device_id)
+        self.addCleanup(
+            heketi_device_enable, heketi_node, heketi_url, device_id)
+        try:
+            self.assertRaises(
+                ExecutionError, heketi_device_remove,
+                heketi_node, heketi_url, device_id)
+        except Exception:
+            self.addCleanup(
+                heketi_device_disable, heketi_node, heketi_url, device_id)
+            raise
diff --git a/tests/functional/common/heketi/test_heketi_volume_operations.py b/tests/functional/common/heketi/test_heketi_volume_operations.py
index 404d11cb..f82521c5 100644
--- a/tests/functional/common/heketi/test_heketi_volume_operations.py
+++ b/tests/functional/common/heketi/test_heketi_volume_operations.py
@@ -1,19 +1,8 @@
-from unittest import skip
-
-from glusto.core import Glusto as g
 from cnslibs.common.heketi_ops import (heketi_volume_delete,
                                        heketi_volume_create,
                                        heketi_volume_expand,
-                                       heketi_volume_info,
-                                       heketi_device_add,
-                                       heketi_device_enable,
-                                       heketi_device_disable,
-                                       heketi_device_remove,
-                                       heketi_device_delete,
-                                       heketi_node_info,
-                                       heketi_node_list)
+                                       heketi_volume_info)
 from cnslibs.common.heketi_libs import HeketiBaseClass
-from cnslibs.common.exceptions import ExecutionError
 
 
 class TestHeketiVolumeOperations(HeketiBaseClass):
@@ -24,58 +13,8 @@ class TestHeketiVolumeOperations(HeketiBaseClass):
     @classmethod
     def setUpClass(cls):
         super(TestHeketiVolumeOperations, cls).setUpClass()
-        cls.volume_id = None
         cls.volume_size = 1
 
-    def volume_cleanup(self, volume_id):
-        """
-        Method to cleanup volume in self.addCleanup()
-        """
-        if volume_id is not None:
-            out = heketi_volume_delete(self.heketi_client_node,
-                                       self.heketi_server_url,
-                                       volume_id)
-            output_str = 'Volume %s deleted' % volume_id
-            if output_str not in out:
-                raise ExecutionError("Failed to delete heketi volume of"
-                                     "id %s" % volume_id)
-
-    def add_device(self, device_name, node_id):
-        """
-        Adds a device through heketi-cli
-        """
-        ret = heketi_device_add(self.heketi_client_node,
-                                self.heketi_server_url,
-                                device_name,
-                                node_id)
-
-        self.assertTrue(ret, ("Failed to add a device %s" % device_name))
-
-    def detach_devices_attached(self, device_id_list):
-        """
-        All the devices attached are gracefully
-        detached in this function
-        """
-        if not isinstance(device_id_list, (list, set, tuple)):
-            device_id_list = [device_id_list]
-
-        for device_id in device_id_list:
-            device_disable = heketi_device_disable(
-                self.heketi_client_node, self.heketi_server_url, device_id)
-            self.assertNotEqual(
-                device_disable, False,
-                "Device %s could not be disabled" % device_id)
-            device_remove = heketi_device_remove(
-                self.heketi_client_node, self.heketi_server_url, device_id)
-            self.assertNotEqual(
-                device_remove, False,
-                "Device %s could not be removed" % device_id)
-            device_delete = heketi_device_delete(
-                self.heketi_client_node, self.heketi_server_url, device_id)
-            self.assertNotEqual(
-                device_delete, False,
-                "Device %s could not be deleted" % device_id)
-
     def test_heketi_with_default_options(self):
         """
         Test to create volume with default options.
@@ -86,7 +25,9 @@ class TestHeketiVolumeOperations(HeketiBaseClass):
                                         self.volume_size, json=True)
         self.assertTrue(vol_info, ("Failed to create heketi volume of size %s"
                                    % self.volume_size))
-        self.addCleanup(self.volume_cleanup, vol_info['id'])
+        self.addCleanup(
+            heketi_volume_delete,
+            self.heketi_client_node, self.heketi_server_url, vol_info['id'])
 
         self.assertEqual(vol_info['size'], self.volume_size,
                          ("Failed to create volume with default options."
@@ -103,7 +44,9 @@ class TestHeketiVolumeOperations(HeketiBaseClass):
                                         self.volume_size, json=True)
         self.assertTrue(vol_info, ("Failed to create heketi volume of size %s"
                                    % self.volume_size))
-        self.addCleanup(self.volume_cleanup, vol_info['id'])
+        self.addCleanup(
+            heketi_volume_delete,
+            self.heketi_client_node, self.heketi_server_url, vol_info['id'])
         self.assertEqual(vol_info['size'], self.volume_size,
                          ("Failed to create volume."
                           "Expected Size: %s, Actual Size: %s"
@@ -123,108 +66,3 @@ class TestHeketiVolumeOperations(HeketiBaseClass):
                          ("Volume Expansion failed Expected Size: %s, Actual "
                           "Size: %s" % (str(expected_size),
                                         str(volume_info['size']))))
-
-    @skip("Blocked by BZ-1629889")
-    def test_heketi_with_device_removal_insuff_space(self):
-        """
-        Test to create volume consuming all space and then adding new device
-        and then trying to remove an existing device. We should get an error
-        saying insufficient space when removing device.
-        """
-        device_id_list = []
-
-        vol_info = heketi_volume_create(self.heketi_client_node,
-                                        self.heketi_server_url,
-                                        650,
-                                        json=True)
-
-        self.assertNotEqual(vol_info, False, "Failed to create heketi volume")
-        self.addCleanup(self.volume_cleanup, vol_info["id"])
-
-        node_id_list = heketi_node_list(
-            self.heketi_client_node, self.heketi_server_url)
-
-        for node_id in node_id_list[:2]:
-            device_present = False
-            node_info = heketi_node_info(
-                self.heketi_client_node, self.heketi_server_url,
-                node_id, json=True)
-
-            self.assertNotEqual(
-                node_info, False,
-                "Heketi node info on node %s failed" % node_id)
-
-            node_ip = node_info["hostnames"]["storage"][0]
-
-            for gluster_server in g.config["gluster_servers"].keys():
-                gluster_server_ip = (g.config["gluster_servers"]
-                                     [gluster_server]["storage"])
-                if gluster_server_ip == node_ip:
-                    device_name = (g.config["gluster_servers"][gluster_server]
-                                   ["additional_devices"][0])
-                    break
-            device_addition_info = heketi_device_add(
-                self.heketi_client_node, self.heketi_server_url,
-                device_name, node_id, json=True)
-
-            self.assertNotEqual(device_addition_info, False,
-                                "Device %s addition failed" % device_name)
-
-            node_info_after_addition = heketi_node_info(
-                self.heketi_client_node, self.heketi_server_url,
-                node_id, json=True)
-
-            self.assertNotEqual(node_info_after_addition, False,
-                                "Node info failed for node %s" % node_id)
-
-            self.assertNotEqual(
-                node_info_after_addition["devices"], [],
-                "No devices in node %s" % node_id)
-
-            for device in node_info_after_addition["devices"]:
-                if device["name"] == device_name:
-                    device_present = True
-                    device_id_list.append(device["id"])
-                    break
-
-            self.assertEqual(device_present, True,
-                             "device %s not present" % device["id"])
-
-        self.addCleanup(self.detach_devices_attached, device_id_list)
-
-        node_1_id = node_id_list[0]
-
-        node_1_info = heketi_node_info(
-                self.heketi_client_node, self.heketi_server_url,
-                node_1_id, json=True)
-
-        self.assertNotEqual(node_1_info, False,
-                            "Node info failed for node %s" % node_1_id)
-        self.assertNotEqual(
-            node_1_info["devices"], [],
-            "No devices in node %s" % node_1_id)
-        device = any([d for d in node_1_info["devices"]
-                      if device["id"] != device_id_list[0]])
-        device_disable = heketi_device_disable(
-            self.heketi_client_node, self.heketi_server_url,
-            device["id"])
-        self.assertNotEqual(
-            device_disable, False,
-            "Device %s could not be disabled" % device["id"])
-        ret, out, err = heketi_device_remove(
-            self.heketi_client_node, self.heketi_server_url,
-            device["id"],
-            raw_cli_output=True)
-        self.assertNotEqual(ret, 0, "Device %s removal successfull")
-        msg = "Error: Failed to remove device, error: No " +\
-              "Replacement was found for resource requested to be " +\
-              "removed"
-        self.assertEqual(
-            msg, err.strip(),
-            "Device %s removal failed due to invalid reason")
-        device_enable = heketi_device_enable(
-            self.heketi_client_node, self.heketi_server_url,
-            device["id"])
-        self.assertNotEqual(
-            device_enable, False,
-            "Device %s could not be enabled" % device["id"])