From e3d38114d334fa99f574a37ee8b5af7f60291be7 Mon Sep 17 00:00:00 2001 From: Arun Date: Fri, 15 Nov 2019 11:53:37 +0530 Subject: Add TC creating and deleting bunch of PVCs during network failure Create network side failure during creation and deletion of the PVC's Network side failure is introduced by opening and closing the ports related to gluster-blockd Change-Id: I0e7d97f0bf4a786f9ebb4cb5ccba5e5fd5812fc6 --- .../test_gluster_block_stability.py | 50 ++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/functional/gluster_stability/test_gluster_block_stability.py b/tests/functional/gluster_stability/test_gluster_block_stability.py index d62c8c21..8f31fd24 100644 --- a/tests/functional/gluster_stability/test_gluster_block_stability.py +++ b/tests/functional/gluster_stability/test_gluster_block_stability.py @@ -1463,3 +1463,53 @@ class TestGlusterBlockStability(GlusterBlockBaseClass): "Failed to get block list from bhv %s" % bhv_id) for blockvol in gluster_vol_info: self.assertNotIn("blockvol_", blockvol) + + def test_create_and_delete_block_pvcs_with_network_failure(self): + """Create and delete volumes after blocking the port 24010 on 51% of + the nodes""" + chain, pvc_amount = 'OS_FIREWALL_ALLOW', 5 + rules = '-p tcp -m state --state NEW -m tcp --dport 24010 -j ACCEPT' + + # Create 5 PVC's, get PV names and volume ids + sc_name = self.create_storage_class(hacount=len(self.gluster_servers)) + pvc_names = self.create_and_wait_for_pvcs( + sc_name=sc_name, pvc_amount=pvc_amount) + vol_details = self.get_vol_id_and_vol_names_from_pvc_names(pvc_names) + vol_names = [vol_name['gluster_vol'] for vol_name in vol_details] + vol_ids = [vol_id['heketi_vol'] for vol_id in vol_details] + + # Get the time to collect logs + since_time = cmd_run( + 'date -u --rfc-3339=ns| cut -d "+" -f 1', self.node).replace( + " ", "T") + "Z" + + # Close the port 24010 on 51% of the nodes + for i in range(len(self.gluster_servers) // 2 + 1): + node_delete_iptables_rules(self.gluster_servers[i], chain, rules) + self.addCleanup( + node_add_iptables_rules, self.gluster_servers[i], chain, rules) + + # Create and delete 5 PVC's + pvc_names_for_creations = self.create_pvcs_not_waiting( + pvc_amount=pvc_amount, sc_name=sc_name) + for pvc_name in pvc_names: + oc_delete(self.node, 'pvc', pvc_name) + self.addCleanup( + wait_for_resource_absence, self.node, 'pvc', pvc_name) + + # Check errors in heketi pod logs and get pending creations + self.check_errors_in_heketi_pod_network_failure_after_deletion( + since_time, vol_names) + + # Open the port 24010, wait for PVC's to get bound + for i in range(len(self.gluster_servers) // 2 + 1): + node_add_iptables_rules(self.gluster_servers[i], chain, rules) + wait_for_pvcs_be_bound(self.node, pvc_names_for_creations, timeout=300) + + # Verify volume deletion + blockvolume_list = heketi_blockvolume_list( + self.heketi_client_node, self.heketi_server_url) + msg = "Unexpectedly volume '%s' exists in the volume list %s" + for vol_id in vol_ids: + self.assertNotIn( + vol_id, blockvolume_list, msg % (vol_id, blockvolume_list)) -- cgit