summaryrefslogtreecommitdiffstats
path: root/tests/functional/logging/test_logging_validations.py
blob: 8c5d31fbac054807215faf01769f275f20832143 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
from pkg_resources import parse_version

import ddt
from glusto.core import Glusto as g
import pytest

from openshiftstoragelibs.baseclass import GlusterBlockBaseClass
from openshiftstoragelibs import command
from openshiftstoragelibs import exceptions
from openshiftstoragelibs import gluster_ops
from openshiftstoragelibs import openshift_ops


@ddt.ddt
class TestLoggingAndGlusterRegistryValidation(GlusterBlockBaseClass):

    def setUp(self):
        """Initialize all the variables necessary for test cases."""
        super(TestLoggingAndGlusterRegistryValidation, self).setUp()

        try:
            logging_config = g.config['openshift']['logging']
            self._logging_project_name = logging_config['logging_project_name']
            self._logging_fluentd_ds = logging_config['logging_fluentd_ds']
            self._logging_es_dc = logging_config['logging_es_dc']
            self._logging_kibana_dc = logging_config['logging_kibana_dc']
            self._registry_heketi_server_url = (
                g.config['openshift']['registry_heketi_config'][
                    'heketi_server_url'])
            self._registry_project_name = (
                g.config['openshift']['registry_project_name'])
            self._registry_servers_info = g.config['gluster_registry_servers']
        except KeyError as err:
            msg = "Config file doesn't have key {}".format(err)
            g.log.error(msg)
            self.skipTest(msg)

        # Skip the test if iscsi-initiator-utils version is not the expected
        cmd = ("rpm -q iscsi-initiator-utils "
               "--queryformat '%{version}-%{release}\n'"
               "| cut -d '.' -f 1,2,3,4")
        e_pkg_version = "6.2.0.874-17"
        for g_server in self.gluster_servers:
            out = self.cmd_run(cmd, g_server)
            if parse_version(out) < parse_version(e_pkg_version):
                msg = ("Skip test since isci initiator utils version actual: "
                       "{out} is less than expected: {ver} on node {server},"
                       " for more info refer to BZ-1624670"
                       .format(out=out, ver=e_pkg_version, server=g_server))
                g.log.error(msg)
                self.skipTest(msg)

        self._master = self.ocp_master_node[0]
        cmd = "oc project --short=true"
        current_project = command.cmd_run(cmd, self._master)
        openshift_ops.switch_oc_project(
            self._master, self._logging_project_name)
        self.addCleanup(
            openshift_ops.switch_oc_project, self._master, current_project)

    def _get_es_pod_and_verify_iscsi_sessions(self):
        """Fetch es pod and verify iscsi sessions"""
        pvc_custom = ":.spec.volumes[*].persistentVolumeClaim.claimName"

        # Get the elasticsearch pod name nad PVC name
        es_pod = openshift_ops.get_pod_name_from_dc(
            self._master, self._logging_es_dc)
        pvc_name = openshift_ops.oc_get_custom_resource(
            self._master, "pod", pvc_custom, es_pod)[0]

        # Validate iscsi and multipath
        self.verify_iscsi_sessions_and_multipath(
            pvc_name, self._logging_es_dc,
            heketi_server_url=self._registry_heketi_server_url,
            is_registry_gluster=True)
        return es_pod, pvc_name

    def _get_newly_deployed_gluster_pod(self, g_pod_list_before):
        # Fetch pod after delete
        g_pod_list_after = [
            pod["pod_name"]
            for pod in openshift_ops.get_ocp_gluster_pod_details(self._master)]

        # Fetch the new gluster pod
        g_new_pod = list(set(g_pod_list_after) - set(g_pod_list_before))
        self.assertTrue(g_new_pod, "No new gluster pod deployed after delete")
        return g_new_pod

    def _guster_pod_delete_cleanup(self, g_pod_list_before):
        """Cleanup for deletion of gluster pod using force delete"""
        # Switch to gluster project
        openshift_ops.switch_oc_project(
            self._master, self._registry_project_name)
        try:
            # Fetch gluster pod after delete
            pod_name = self._get_newly_deployed_gluster_pod(g_pod_list_before)

            # Check if pod name is empty i.e no new pod come up so use old pod
            openshift_ops.wait_for_pod_be_ready(
                self._master,
                pod_name[0] if pod_name else g_pod_list_before[0], timeout=1)
        except exceptions.ExecutionError:
            # Force delete and wait for new pod to come up
            openshift_ops.oc_delete(
                self._master, 'pod', g_pod_list_before[0], is_force=True)
            openshift_ops.wait_for_resource_absence(
                self._master, 'pod', g_pod_list_before[0])

            # Fetch gluster pod after force delete
            g_new_pod = self._get_newly_deployed_gluster_pod(g_pod_list_before)
            openshift_ops.wait_for_pod_be_ready(self._master, g_new_pod[0])

    @pytest.mark.tier3
    def test_validate_logging_pods_and_pvc(self):
        """Validate metrics pods and PVC"""

        # Wait for kibana pod to be ready
        kibana_pod = openshift_ops.get_pod_name_from_dc(
            self._master, self._logging_kibana_dc)
        openshift_ops.wait_for_pod_be_ready(self._master, kibana_pod)

        # Wait for fluentd pods to be ready
        fluentd_custom = [":.status.desiredNumberScheduled",
                          ":.spec.template.metadata.labels"]
        count_and_selector = openshift_ops.oc_get_custom_resource(
            self._master, "ds", fluentd_custom, self._logging_fluentd_ds)
        selector = count_and_selector[1][4:].replace(":", "=")
        openshift_ops.wait_for_pods_be_ready(
            self._master, int(count_and_selector[0]), selector)

        # Wait for PVC to be bound and elasticsearch pod to be ready
        es_pod = openshift_ops.get_pod_name_from_dc(
            self._master, self._logging_es_dc)
        pvc_custom = ":.spec.volumes[*].persistentVolumeClaim.claimName"
        pvc_name = openshift_ops.oc_get_custom_resource(
            self._master, "pod", pvc_custom, es_pod)[0]
        openshift_ops.verify_pvc_status_is_bound(self._master, pvc_name)
        openshift_ops.wait_for_pod_be_ready(self._master, es_pod)

        # Validate iscsi and multipath
        self.verify_iscsi_sessions_and_multipath(
            pvc_name, self._logging_es_dc,
            heketi_server_url=self._registry_heketi_server_url,
            is_registry_gluster=True)

    @pytest.mark.tier3
    def test_logging_es_pod_pvc_all_freespace_utilization(self):
        """Validate logging by utilizing all the free space of block PVC bound
           to elsaticsearch pod"""

        # Fetch pod and validate iscsi and multipath
        es_pod, _ = self._get_es_pod_and_verify_iscsi_sessions()

        # Get the available free space
        mount_point = '/elasticsearch/persistent'
        cmd_free_space = (
            "df -kh {} | awk '{{print $4}}' | tail -1".format(mount_point))
        old_available_space = openshift_ops.oc_rsh(
            self._master, es_pod, cmd_free_space)[1]

        # Fill the all the available space
        file_name = '{}/file'.format(mount_point)
        cmd_fill_space = (
            "fallocate -l {} {}".format(old_available_space, file_name))
        with self.assertRaises(AssertionError):
            openshift_ops.oc_rsh(self._master, es_pod, cmd_fill_space)

            # Cleanup the filled space
            cmd_remove_file = 'rm {}'.format(file_name)
            self.addCleanup(
                openshift_ops.oc_rsh, self._master, es_pod, cmd_remove_file)

    @pytest.mark.tier2
    def test_resping_gluster_pod(self):
        """Validate gluster pod restart with no disruption to elasticsearch pod
        """
        restart_custom = ":status.containerStatuses[0].restartCount"

        # Fetch pod and validate iscsi and multipath
        es_pod, _ = self._get_es_pod_and_verify_iscsi_sessions()

        # Fetch the restart count for the es pod
        restart_count_before = openshift_ops.oc_get_custom_resource(
            self._master, "pod", restart_custom, es_pod)[0]

        # Switch to gluster project
        openshift_ops.switch_oc_project(
            self._master, self._registry_project_name)

        # Fetch the gluster pod list before
        g_pod_list_before = [
            pod["pod_name"]
            for pod in openshift_ops.get_ocp_gluster_pod_details(self._master)]

        # Respin a gluster pod
        openshift_ops.oc_delete(self._master, "pod", g_pod_list_before[0])
        self.addCleanup(self._guster_pod_delete_cleanup, g_pod_list_before)

        # Wait for pod to get absent
        openshift_ops.wait_for_resource_absence(
            self._master, "pod", g_pod_list_before[0])

        # Fetch gluster pod after delete
        g_new_pod = self._get_newly_deployed_gluster_pod(g_pod_list_before)
        openshift_ops.wait_for_pod_be_ready(self._master, g_new_pod[0])

        # Switch to logging project
        openshift_ops.switch_oc_project(
            self._master, self._logging_project_name)

        # Fetch the restart count for the es pod
        restart_count_after = openshift_ops.oc_get_custom_resource(
            self._master, "pod", restart_custom, es_pod)[0]
        self.assertEqual(
            restart_count_before, restart_count_after,
            "Failed disruption to es pod found expecting restart count before"
            " {} and after {} for es pod to be equal after gluster pod"
            " respin".format(restart_count_before, restart_count_after))

    @pytest.mark.tier2
    def test_kill_bhv_fsd_while_es_pod_running(self):
        """Validate killing of bhv fsd won't effect es pod io's"""

        # Fetch pod and PVC names and validate iscsi and multipath
        es_pod, pvc_name = self._get_es_pod_and_verify_iscsi_sessions()

        # Get the bhv name
        gluster_node = list(self._registry_servers_info.keys())[0]
        openshift_ops.switch_oc_project(
            self._master, self._registry_project_name)
        bhv_name = self.get_block_hosting_volume_by_pvc_name(
            pvc_name, heketi_server_url=self._registry_heketi_server_url,
            gluster_node=gluster_node)

        # Get one of the bricks pid of the bhv
        gluster_volume_status = gluster_ops.get_gluster_vol_status(bhv_name)
        pid = None
        for g_node, g_node_data in gluster_volume_status.items():
            if g_node != gluster_node:
                continue
            for process_name, process_data in g_node_data.items():
                if not process_name.startswith("/var"):
                    continue
                pid = process_data["pid"]
                # When birck is down, pid of the brick is returned as -1.
                # Which is unexepeted situation. So, add appropriate assertion.
                self.assertNotEqual(
                    pid, "-1", "Got unexpected PID (-1) for '{}' gluster vol "
                    "on '{}' node.".format(bhv_name, gluster_node))
                break
            self.assertTrue(
                pid, "Could not find 'pid' in Gluster vol data for '{}' "
                "Gluster node. Data: {}".format(
                    gluster_node, gluster_volume_status))
            break

        # Kill gluster vol brick process using found pid
        cmd_kill = "kill -9 {}".format(pid)
        cmd_start_vol = "gluster v start {} force".format(bhv_name)
        openshift_ops.cmd_run_on_gluster_pod_or_node(
            self._master, cmd_kill, gluster_node)
        self.addCleanup(openshift_ops.cmd_run_on_gluster_pod_or_node,
                        self._master, cmd_start_vol, gluster_node)
        self.addCleanup(openshift_ops.switch_oc_project,
                        self._master, self._registry_project_name)

        # Run I/O on ES pod
        openshift_ops.switch_oc_project(
            self._master, self._logging_project_name)
        file_name = '/elasticsearch/persistent/file1'
        cmd_run_io = 'dd if=/dev/urandom of={} bs=4k count=10000'.format(
            file_name)
        cmd_remove_file = 'rm {}'.format(file_name)
        openshift_ops.oc_rsh(self._master, es_pod, cmd_run_io)
        self.addCleanup(
            openshift_ops.oc_rsh, self._master, es_pod, cmd_remove_file)