summaryrefslogtreecommitdiffstats
path: root/tests/functional/afr/test_default_granular_entry_heal.py
blob: 91ca25907cdc1c1fe4de60d7a7a4da9610c947d8 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
#  Copyright (C) 2021 Red Hat, Inc. <http://www.redhat.com>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License along`
#  with this program; if not, write to the Free Software Foundation, Inc.,
#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

from random import choice

from glusto.core import Glusto as g
from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on
from glustolibs.gluster.exceptions import ExecutionError
from glustolibs.gluster.brick_libs import (bring_bricks_offline,
                                           bring_bricks_online,
                                           are_bricks_offline,
                                           are_bricks_online, get_all_bricks)
from glustolibs.gluster.glusterfile import occurences_of_pattern_in_file
from glustolibs.gluster.heal_libs import (monitor_heal_completion,
                                          is_heal_complete)
from glustolibs.gluster.lib_utils import collect_bricks_arequal
from glustolibs.gluster.volume_libs import get_subvols
from glustolibs.gluster.volume_ops import get_volume_options
from glustolibs.io.utils import collect_mounts_arequal


@runs_on([['distributed-replicated', 'replicated',
           'arbiter', 'distributed-arbiter'], ['glusterfs']])
class TestDefaultGranularEntryHeal(GlusterBaseClass):

    def setUp(self):

        self.get_super_method(self, 'setUp')()

        self.first_client = self.mounts[0].client_system
        self.mountpoint = self.mounts[0].mountpoint

        # Setup Volume
        if not self.setup_volume_and_mount_volume([self.mounts[0]]):
            raise ExecutionError("Failed to setup and mount volume")

    def tearDown(self):

        if not self.unmount_volume_and_cleanup_volume([self.mounts[0]]):
            raise ExecutionError("Failed to cleanup Volume")

        # Calling GlusterBaseClass tearDown
        self.get_super_method(self, 'tearDown')()

    def _bring_bricks_offline(self):
        """Brings bricks offline and confirms if they are offline"""
        # Select bricks to bring offline from a replica set
        subvols_dict = get_subvols(self.mnode, self.volname)
        subvols = subvols_dict['volume_subvols']
        self.bricks_to_bring_offline = []
        for subvol in subvols:
            self.bricks_to_bring_offline.append(choice(subvol))

        # Bring bricks offline
        ret = bring_bricks_offline(self.volname, self.bricks_to_bring_offline)
        self.assertTrue(ret, 'Failed to bring bricks %s offline' %
                        self.bricks_to_bring_offline)

        ret = are_bricks_offline(self.mnode, self.volname,
                                 self.bricks_to_bring_offline)
        self.assertTrue(ret, 'Bricks %s are not offline'
                        % self.bricks_to_bring_offline)
        g.log.info('Bringing bricks %s offline is successful',
                   self.bricks_to_bring_offline)

    def _restart_volume_and_bring_all_offline_bricks_online(self):
        """Restart volume and bring all offline bricks online"""

        ret = is_heal_complete(self.mnode, self.volname)
        self.assertFalse(ret, 'Heal is completed')
        g.log.info('Heal is pending')

        ret = bring_bricks_online(self.mnode, self.volname,
                                  self.bricks_to_bring_offline,
                                  bring_bricks_online_methods=[
                                      'volume_start_force'])
        self.assertTrue(ret, 'Failed to bring bricks %s online' %
                        self.bricks_to_bring_offline)

        # Check if bricks are back online or not
        ret = are_bricks_online(self.mnode, self.volname,
                                self.bricks_to_bring_offline)
        self.assertTrue(ret, 'Bricks not online %s even after restart' %
                        self.bricks_to_bring_offline)

        g.log.info('Bringing bricks %s online is successful',
                   self.bricks_to_bring_offline)

    def _wait_for_heal_to_completed(self):
        """Check if heal is completed"""
        ret = monitor_heal_completion(self.mnode, self.volname,
                                      timeout_period=3600)
        self.assertTrue(ret, 'Heal has not yet completed')

    def _check_arequal_on_bricks_with_a_specific_arequal(self, arequal,
                                                         brick_list):
        """
        Compare an inital arequal checksum with bricks from a given brick list
        """
        init_val = arequal[0].splitlines()[-1].split(':')[-1]
        ret, arequals = collect_bricks_arequal(brick_list)
        self.assertTrue(ret, 'Failed to get arequal on bricks')
        for brick_arequal in arequals:
            brick_total = brick_arequal.splitlines()[-1].split(':')[-1]
            self.assertEqual(init_val, brick_total, 'Arequals not matching')

    @staticmethod
    def _add_dir_path_to_brick_list(brick_list):
        """Add test_self_heal at the end of brick path"""
        dir_brick_list = []
        for brick in brick_list:
            dir_brick_list.append('{}/{}'.format(brick, 'mydir'))
        return dir_brick_list

    def _check_arequal_checksum_for_the_volume(self):
        """
        Check if arequals of mount point and bricks are
        are the same.
        """
        if self.volume_type == "replicated":
            # Check arequals for "replicated"
            brick_list = get_all_bricks(self.mnode, self.volname)
            dir_brick_list = self._add_dir_path_to_brick_list(brick_list)

            # Get arequal before getting bricks offline
            work_dir = '{}/mydir'.format(self.mountpoint)
            ret, arequals = collect_mounts_arequal([self.mounts[0]],
                                                   path=work_dir)
            self.assertTrue(ret, 'Failed to get arequal')
            g.log.info('Getting arequal before getting bricks offline '
                       'is successful')

            # Get arequal on bricks and compare with mount_point_total
            self._check_arequal_on_bricks_with_a_specific_arequal(
                arequals, dir_brick_list)

        # Check arequals for "distributed-replicated"
        if self.volume_type == "distributed-replicated":
            # Get the subvolumes
            subvols_dict = get_subvols(self.mnode, self.volname)
            num_subvols = len(subvols_dict['volume_subvols'])

            # Get arequals and compare
            for i in range(0, num_subvols):
                # Get arequal for first brick
                brick_list = subvols_dict['volume_subvols'][i]
                dir_brick_list = self._add_dir_path_to_brick_list(brick_list)
                ret, arequals = collect_bricks_arequal([dir_brick_list[0]])
                self.assertTrue(ret, 'Failed to get arequal on first brick')

                # Get arequal for every brick and compare with first brick
                self._check_arequal_on_bricks_with_a_specific_arequal(
                    arequals, dir_brick_list)

    def test_default_granular_entry_heal(self):
        """
        Test case:
        1. Create a cluster.
        2. Create volume start it and mount it.
        3. Check if cluster.granular-entry-heal is ON by default or not.
        4. Check /var/lib/glusterd/<volname>/info for
           cluster.granular-entry-heal=on.
        5. Check if option granular-entry-heal is present in the
           volume graph or not.
        6. Kill one or two bricks of the volume depending on volume type.
        7. Create all types of files on the volume like text files, hidden
           files, link files, dirs, char device, block device and so on.
        8. Bring back the killed brick by restarting the volume.
        9. Wait for heal to complete.
        10. Check arequal-checksum of all the bricks and see if it's proper or
            not.
        """
        # Check if cluster.granular-entry-heal is ON by default or not
        ret = get_volume_options(self.mnode, self.volname,
                                 'granular-entry-heal')
        self.assertEqual(ret['cluster.granular-entry-heal'], 'on',
                         "Value of cluster.granular-entry-heal not on "
                         "by default")

        # Check var/lib/glusterd/<volname>/info for
        # cluster.granular-entry-heal=on
        ret = occurences_of_pattern_in_file(self.mnode,
                                            'cluster.granular-entry-heal=on',
                                            '/var/lib/glusterd/vols/{}/info'
                                            .format(self.volname))
        self.assertEqual(ret, 1, "Failed get cluster.granular-entry-heal=on in"
                         " info file")

        # Check if option granular-entry-heal is present in the
        # volume graph or not
        ret = occurences_of_pattern_in_file(self.first_client,
                                            'option granular-entry-heal on',
                                            "/var/log/glusterfs/mnt-{}_{}.log"
                                            .format(self.volname,
                                                    self.mount_type))
        self.assertTrue(ret > 0,
                        "Failed to find granular-entry-heal in volume graph")
        g.log.info("granular-entry-heal properly set to ON by default")

        # Kill one or two bricks of the volume depending on volume type
        self._bring_bricks_offline()

        # Create all types of files on the volume like text files, hidden
        # files, link files, dirs, char device, block device and so on
        cmd = ("cd {};mkdir mydir;cd mydir;mkdir dir;mkdir .hiddendir;"
               "touch file;touch .hiddenfile;mknod blockfile b 1 5;"
               "mknod charfile b 1 5; mkfifo pipefile;touch fileforhardlink;"
               "touch fileforsoftlink;ln fileforhardlink hardlinkfile;"
               "ln -s fileforsoftlink softlinkfile".format(self.mountpoint))
        ret, _, _ = g.run(self.first_client, cmd)
        self.assertFalse(ret, "Failed to create files of all types")

        # Bring back the killed brick by restarting the volume	Bricks should
        # be online again
        self._restart_volume_and_bring_all_offline_bricks_online()

        # Wait for heal to complete
        self._wait_for_heal_to_completed()

        # Check arequal-checksum of all the bricks and see if it's proper or
        # not
        self._check_arequal_checksum_for_the_volume()