CNS-439: adding heketi_down_pvc_delete case

Change-Id: Id3b679cf3193b36bfd10fc498fb0ba930c5d2536
Signed-off-by: Apeksha D Khakharia <akhakhar@redhat.com>

2 files changed, 49 insertions, 5 deletions

diff --git a/cns-libs/cnslibs/common/openshift_ops.py b/cns-libs/cnslibs/common/openshift_ops.py
index d05529d8..830dc215 100644
--- a/cns-libs/cnslibs/common/openshift_ops.py
+++ b/cns-libs/cnslibs/common/openshift_ops.py
@@ -447,23 +447,27 @@ def oc_create_tiny_pod_with_volume(hostname, pvc_name, pod_name_prefix='',
     return pod_name
 
 
-def oc_delete(ocp_node, rtype, name):
+def oc_delete(ocp_node, rtype, name, raise_on_absence=True):
     """Delete an OCP resource by name.
 
     Args:
         ocp_node (str): Node on which the ocp command will run.
         rtype (str): Name of the resource type (pod, storageClass, etc).
         name (str): Name of the resource to delete.
-    Raises:
-        AssertionError: Raised when resource fails to create.
+        raise_on_absence (bool): if resource absent raise
+                                 exception if value is true,
+                                 else return
+                                 default value: True
     """
+    if not oc_get_yaml(ocp_node, rtype, name,
+                       raise_on_error=raise_on_absence):
+        return
     ret, out, err = g.run(ocp_node, ['oc', 'delete', rtype, name])
     if ret != 0:
         g.log.error('Failed to delete resource: %s, %s: %r; %r',
                     rtype, name, out, err)
         raise AssertionError('failed to delete resource: %r; %r' % (out, err))
     g.log.info('Deleted resource: %r %r', rtype, name)
-    return
 
 
 def oc_get_yaml(ocp_node, rtype, name=None, raise_on_error=True):
diff --git a/tests/functional/common/provisioning/test_dynamic_provisioning_block_p0_cases.py b/tests/functional/common/provisioning/test_dynamic_provisioning_block_p0_cases.py
index 2e0268cd..76e1d317 100644
--- a/tests/functional/common/provisioning/test_dynamic_provisioning_block_p0_cases.py
+++ b/tests/functional/common/provisioning/test_dynamic_provisioning_block_p0_cases.py
@@ -65,7 +65,8 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass):
             self.addCleanup(
                 wait_for_resource_absence, self.node, 'pvc', pvc_name)
         for pvc_name in pvc_names:
-            self.addCleanup(oc_delete, self.node, 'pvc', pvc_name)
+            self.addCleanup(oc_delete, self.node, 'pvc', pvc_name,
+                            raise_on_absence=False)
 
         # Wait for PVCs to be in bound state
         for pvc_name in pvc_names:
@@ -262,3 +263,42 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass):
                 self.assertFalse(err, "Error output is not empty: \n%s" % err)
                 self.assertEqual(ret, 0, "Failed to exec '%s' command." % cmd)
                 self.assertTrue(out, "Command '%s' output is empty." % cmd)
+
+    def test_dynamic_provisioning_glusterblock_heketidown_pvc_delete(self):
+        """ Delete PVC's when heketi is down CNS-439 """
+
+        # Create storage class and secret objects
+        self._create_storage_class()
+
+        self.pvc_name_list = self._create_and_wait_for_pvcs(
+            1, 'pvc-heketi-down', 3)
+
+        # remove heketi-pod
+        scale_dc_pod_amount_and_wait(self.ocp_client[0],
+                                     self.heketi_dc_name,
+                                     0,
+                                     self.cns_project_name)
+        try:
+            # delete pvc
+            for pvc in self.pvc_name_list:
+                oc_delete(self.ocp_client[0], 'pvc', pvc)
+            for pvc in self.pvc_name_list:
+                with self.assertRaises(ExecutionError):
+                    wait_for_resource_absence(
+                       self.ocp_client[0], 'pvc', pvc,
+                       interval=3, timeout=30)
+        finally:
+            # bring back heketi-pod
+            scale_dc_pod_amount_and_wait(self.ocp_client[0],
+                                         self.heketi_dc_name,
+                                         1,
+                                         self.cns_project_name)
+
+        # verify PVC's are deleted
+        for pvc in self.pvc_name_list:
+            wait_for_resource_absence(self.ocp_client[0], 'pvc',
+                                      pvc,
+                                      interval=1, timeout=120)
+
+        # create a new PVC
+        self._create_and_wait_for_pvc()