1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
|
from __future__ import division
from glusto.core import Glusto as g
import mock
import pytest
from openshiftstoragelibs import baseclass
from openshiftstoragelibs import command
from openshiftstoragelibs import exceptions
from openshiftstoragelibs import heketi_ops
from openshiftstoragelibs import openshift_ops
from openshiftstoragelibs import waiter
HEKETI_COMMAND_TIMEOUT = g.config.get("common", {}).get(
"heketi_command_timeout", 120)
TIMEOUT_PREFIX = "timeout %s " % HEKETI_COMMAND_TIMEOUT
class TestVolumeDeleteTestCases(baseclass.BaseClass):
"""
Class for volume deletion related test cases
"""
def get_free_space_summary_devices(self):
"""
Calculates free space across all devices
"""
heketi_node_id_list = heketi_ops.heketi_node_list(
self.heketi_client_node, self.heketi_server_url)
total_free_space = 0
for node_id in heketi_node_id_list:
node_info_dict = heketi_ops.heketi_node_info(
self.heketi_client_node, self.heketi_server_url,
node_id, json=True)
for device in node_info_dict["devices"]:
total_free_space += (device["storage"]
["free"] / (1024 ** 2))
return total_free_space
def _heketi_pod_delete_cleanup(self, ocp_node):
"""Cleanup for deletion of heketi pod using force delete"""
try:
pod_name = openshift_ops.get_pod_name_from_dc(
ocp_node, self.heketi_dc_name)
# Check if heketi pod name is ready state
openshift_ops.wait_for_pod_be_ready(ocp_node, pod_name, timeout=1)
except exceptions.ExecutionError:
# Force delete and wait for new pod to come up
openshift_ops.oc_delete(ocp_node, 'pod', pod_name, is_force=True)
openshift_ops.wait_for_resource_absence(
self.ocp_master_node[0], 'pod', pod_name)
# Fetch heketi pod after force delete
pod_name = openshift_ops.get_pod_name_from_dc(
ocp_node, self.heketi_dc_name)
openshift_ops.wait_for_pod_be_ready(ocp_node, pod_name)
@pytest.mark.tier1
def test_delete_heketi_volume(self):
"""
Method to test heketi volume deletion and whether it
frees up used space after deletion
"""
volume_info = heketi_ops.heketi_volume_create(
self.heketi_client_node,
self.heketi_server_url, 10, json=True)
self.addCleanup(
heketi_ops.heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url,
volume_info["id"], raise_on_error=False)
free_space_after_creation = self.get_free_space_summary_devices()
heketi_ops.heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url,
volume_info["id"])
free_space_after_deletion = self.get_free_space_summary_devices()
self.assertTrue(
free_space_after_deletion > free_space_after_creation,
"Free space is not reclaimed after deletion "
"of %s" % volume_info["id"])
@pytest.mark.tier1
def test_delete_heketidb_volume(self):
"""Method to test heketidb volume deletion via heketi-cli."""
for i in range(0, 2):
volume_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url,
10, json=True)
self.addCleanup(
heketi_ops.heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, volume_info["id"])
volume_list_info = heketi_ops.heketi_volume_list(
self.heketi_client_node,
self.heketi_server_url, json=True)
self.assertTrue(
volume_list_info["volumes"], "Heketi volume list empty.")
for volume_id in volume_list_info["volumes"]:
volume_info = heketi_ops.heketi_volume_info(
self.heketi_client_node, self.heketi_server_url,
volume_id, json=True)
if volume_info["name"] == "heketidbstorage":
self.assertRaises(
AssertionError,
heketi_ops.heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url, volume_id)
return
raise exceptions.ExecutionError(
"Warning: heketidbstorage doesn't exist in list of volumes")
@pytest.mark.tier2
def test_heketi_server_stale_operations_during_heketi_pod_reboot(self):
"""
Validate failed/stale entries in db and performs a cleanup
of those entries
"""
volume_id_list, async_obj, ocp_node = [], [], self.ocp_master_node[0]
h_node, h_server = self.heketi_client_node, self.heketi_server_url
for i in range(0, 8):
volume_info = heketi_ops.heketi_volume_create(
h_node, h_server, 1, json=True)
volume_id_list.append(volume_info["id"])
self.addCleanup(
heketi_ops.heketi_volume_delete, h_node, h_server,
volume_info["id"], raise_on_error=False)
def run_async(cmd, hostname, raise_on_error=True):
async_op = g.run_async(host=hostname, command=cmd)
async_obj.append(async_op)
return async_op
# Temporary replace g.run with g.async_run in heketi_volume_delete
# to be able to run it in background.
for vol_id in volume_id_list:
with mock.patch.object(command, 'cmd_run', side_effect=run_async):
heketi_ops.heketi_volume_delete(h_node, h_server, vol_id)
# Restart heketi pod and check pod is running
heketi_pod_name = openshift_ops.get_pod_name_from_dc(
ocp_node, self.heketi_dc_name)
openshift_ops.oc_delete(
ocp_node, 'pod', heketi_pod_name,
collect_logs=self.heketi_logs_before_delete)
self.addCleanup(self._heketi_pod_delete_cleanup, ocp_node)
openshift_ops.wait_for_resource_absence(
ocp_node, 'pod', heketi_pod_name)
heketi_pod_name = openshift_ops.get_pod_name_from_dc(
ocp_node, self.heketi_dc_name)
openshift_ops.wait_for_pod_be_ready(ocp_node, heketi_pod_name)
self.assertTrue(
heketi_ops.hello_heketi(h_node, h_server),
"Heketi server {} is not alive".format(h_server))
# Wait for pending operations to get generate
for w in waiter.Waiter(timeout=30, interval=3):
h_db_check = heketi_ops.heketi_db_check(h_node, h_server)
h_db_check_vol = h_db_check.get("volumes")
h_db_check_bricks = h_db_check.get("bricks")
if ((h_db_check_vol.get("pending"))
and (h_db_check_bricks.get("pending"))):
break
if w.expired:
raise exceptions.ExecutionError(
"No any pending operations found during volumes deletion "
"volumes:{}, Bricks:{} ".format(
h_db_check_vol.get("pending"),
h_db_check_bricks.get("pending")))
# Verify pending bricks are multiples of 3
self.assertFalse(
h_db_check_bricks.get("pending") % 3,
"Expecting bricks pending count to be multiple of 3 but "
"found {}".format(h_db_check_bricks.get("pending")))
# Verify and Wait for pending operations to complete
for w in waiter.Waiter(timeout=120, interval=10):
h_db_check = heketi_ops.heketi_db_check(h_node, h_server)
h_db_check_vol = h_db_check.get("volumes")
h_db_check_bricks = h_db_check.get("bricks")
if ((not h_db_check_bricks.get("pending"))
and (not h_db_check_vol.get("pending"))):
break
if w.expired:
raise exceptions.AssertionError(
"Failed to delete volumes after 120 secs")
|
