[Test+Lib] Add tests to check self heal

Test scenarios added:
1. Test to check entry self heal.
2. Test to check meta data self heal.
3. Test self heal when files are removed and dirs
   created with the same name.

Additional libraries added:
1. group_del(): Deletes groups created
2. enable_granular_heal(): Enables granular heal
3. disable_granular_heal(): Disables granular heal

Change-Id: Iffa9a100fddaecae99c384afe3aaeaf13dd37e0d
Signed-off-by: kshithijiyer <kshithij.ki@gmail.com>

1 files changed, 600 insertions, 0 deletions

diff --git a/tests/functional/afr/heal/test_self_heal_with_meta_data_entry_and_files_removed.py b/tests/functional/afr/heal/test_self_heal_with_meta_data_entry_and_files_removed.py
new file mode 100644
index 000000000..37bd2ec52
--- /dev/null
+++ b/tests/functional/afr/heal/test_self_heal_with_meta_data_entry_and_files_removed.py
@@ -0,0 +1,600 @@
+#  Copyright (C) 2021 Red Hat, Inc. <http://www.redhat.com>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along`
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from glusto.core import Glusto as g
+from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on
+from glustolibs.gluster.brick_libs import (bring_bricks_offline,
+                                           bring_bricks_online,
+                                           are_bricks_offline,
+                                           are_bricks_online,
+                                           get_all_bricks)
+from glustolibs.gluster.glusterdir import mkdir
+from glustolibs.gluster.exceptions import ExecutionError
+from glustolibs.gluster.heal_libs import (monitor_heal_completion,
+                                          is_volume_in_split_brain,
+                                          is_heal_complete,
+                                          enable_granular_heal,
+                                          disable_granular_heal)
+from glustolibs.gluster.lib_utils import (add_user, del_user, group_del,
+                                          group_add, collect_bricks_arequal)
+from glustolibs.gluster.volume_ops import get_volume_options
+from glustolibs.gluster.volume_libs import get_subvols
+from glustolibs.io.utils import collect_mounts_arequal
+
+
+@runs_on([['distributed-replicated', 'replicated'], ['glusterfs']])
+class TestHealWithLinkFiles(GlusterBaseClass):
+
+    def setUp(self):
+
+        self.get_super_method(self, 'setUp')()
+
+        self.first_client = self.mounts[0].client_system
+        self.mountpoint = self.mounts[0].mountpoint
+        self.user_group_created = False
+
+        # If test case running is test_self_heal_meta_data
+        # create user and group
+        test_name_splitted = self.id().split('.')
+        test_id = test_name_splitted[len(test_name_splitted) - 1]
+        if test_id == 'test_self_heal_meta_data':
+
+            # Create non-root group
+            if not group_add(self.first_client, 'qa_all'):
+                raise ExecutionError("Failed to create group qa_all")
+
+            # Create non-root users
+            self.users = ('qa_func', 'qa_system', 'qa_perf')
+            for user in self.users:
+                if not add_user(self.first_client, user, group='qa_all'):
+                    raise ExecutionError("Failed to create user {}"
+                                         .format(user))
+
+            self.user_group_created = True
+            g.log.info("Successfully created all users.")
+
+        # Setup Volume
+        if not self.setup_volume_and_mount_volume([self.mounts[0]]):
+            raise ExecutionError("Failed to setup and mount volume")
+
+    def tearDown(self):
+
+        # Delete non-root users and group if created
+        if self.user_group_created:
+
+            # Delete non-root users
+            for user in self.users:
+                del_user(self.first_client, user)
+            g.log.info("Successfully deleted all users")
+
+            # Delete non-root group
+            group_del(self.first_client, 'qa_all')
+
+        if not self.unmount_volume_and_cleanup_volume([self.mounts[0]]):
+            raise ExecutionError("Failed to cleanup Volume")
+
+        # Calling GlusterBaseClass tearDown
+        self.get_super_method(self, 'tearDown')()
+
+    def _set_granular_heal_to_on_or_off(self, enabled=False):
+        """Set granular heal to ON or OFF"""
+        granular = get_volume_options(self.mnode, self.volname,
+                                      'granular-entry-heal')
+        if enabled:
+            if granular['cluster.granular-entry-heal'] != 'on':
+                ret = enable_granular_heal(self.mnode, self.volname)
+                self.assertTrue(ret,
+                                "Unable to set granular-entry-heal to on")
+        else:
+            if granular['cluster.granular-entry-heal'] == 'on':
+                ret = disable_granular_heal(self.mnode, self.volname)
+                self.assertTrue(ret,
+                                "Unable to set granular-entry-heal to off")
+
+    def _run_cmd(self, io_cmd, err_msg):
+        """Run cmd and show error message if it fails"""
+        cmd = ("cd {}/test_self_heal;{}".format(self.mountpoint, io_cmd))
+        ret, _, _ = g.run(self.first_client, cmd)
+        self.assertFalse(ret, err_msg)
+
+    def _create_files_and_dirs_on_mount_point(self, index, second_set=False):
+        """A function to create files and dirs on mount point"""
+        # Create a parent directory test_self_heal on mount point
+        if not second_set:
+            ret = mkdir(self.first_client, '{}/{}'.format(
+                self.mountpoint, 'test_self_heal'))
+            self.assertTrue(ret, "Failed to create dir test_self_heal")
+
+        # Create dirctories and files inside directory test_self_heal
+        io_cmd = ("for i in `seq 1 50`; do mkdir dir.$i; dd if=/dev/random"
+                  " of=file.$i count=1K bs=$i; done",
+
+                  "for i in `seq 1 100`; do mkdir dir.$i; for j in `seq 1 5`;"
+                  " do dd if=/dev/random of=dir.$i/file.$j bs=1K count=$j"
+                  ";done;done",
+
+                  "for i in `seq 1 10`; do mkdir l1_dir.$i; for j in `seq "
+                  "1 5`; do mkdir l1_dir.$i/l2_dir.$j; for k in `seq 1 10`;"
+                  " do dd if=/dev/random of=l1_dir.$i/l2_dir.$j/test.$k"
+                  " bs=1k count=$k; done; done; done;",
+
+                  "for i in `seq 51 100`; do mkdir new_dir.$i; for j in `seq"
+                  " 1 10`; do dd if=/dev/random of=new_dir.$i/new_file.$j "
+                  "bs=1K count=$j; done; dd if=/dev/random of=new_file.$i"
+                  " count=1K bs=$i; done ;")
+        self._run_cmd(
+            io_cmd[index], "Failed to create dirs and files inside")
+
+    def _delete_files_and_dirs(self):
+        """Delete files and dirs from mount point"""
+        io_cmd = ("for i in `seq 1 50`; do rm -rf dir.$i; rm -f file.$i;done")
+        self._run_cmd(io_cmd, "Failed to delete dirs and files")
+
+    def _rename_files_and_dirs(self):
+        """Rename files and dirs from mount point"""
+        io_cmd = ("for i in `seq 51 100`; do mv new_file.$i renamed_file.$i;"
+                  " for j in `seq 1 10`; do mv new_dir.$i/new_file.$j "
+                  "new_dir.$i/renamed_file.$j ; done ; mv new_dir.$i "
+                  "renamed_dir.$i; done;")
+        self._run_cmd(io_cmd, "Failed to rename dirs and files")
+
+    def _change_meta_deta_of_dirs_and_files(self):
+        """Change meta data of dirs and files"""
+        cmds = (
+            # Change permission
+            "for i in `seq 1 100`; do chmod 555 dir.$i; done; "
+            "for i in `seq 1 50`; do for j in `seq 1 5`; do chmod 666 "
+            "dir.$i/file.$j; done; done; for i in `seq 51 100`; do for "
+            "j in `seq 1 5`;do chmod 444 dir.$i/file.$j; done; done;",
+
+            # Change ownership
+            "for i in `seq 1 35`; do chown -R qa_func dir.$i; done; "
+            "for i in `seq 36 70`; do chown -R qa_system dir.$i; done; "
+            "for i in `seq 71 100`; do chown -R qa_perf dir.$i; done;",
+
+            # Change group
+            "for i in `seq 1 100`; do chgrp -R qa_all dir.$i; done;")
+
+        for io_cmd in cmds:
+            self._run_cmd(io_cmd,
+                          "Failed to change meta data on dirs and files")
+        g.log.info("Successfully changed meta data on dirs and files")
+
+    def _verify_meta_data_of_files_and_dirs(self):
+        """Verify meta data of files and dirs"""
+        cmds = (
+            # Verify permissions
+            "for i in `seq 1 50`; do stat -c %a dir.$i | grep -F \"555\";"
+            " if [ $? -ne 0 ]; then exit 1; fi; for j in `seq 1 5` ; do "
+            "stat -c %a dir.$i/file.$j | grep -F \"666\"; if [ $? -ne 0 ]"
+            ";  then exit 1; fi; done; done; for i in `seq 51 100`; do "
+            "stat -c %a dir.$i | grep -F \"555\";if [ $? -ne 0 ]; then "
+            "exit 1; fi; for j in `seq 1 5`; do stat -c %a dir.$i/file.$j"
+            " | grep -F \"444\"; if [ $? -ne 0 ]; then exit 1; fi; done;"
+            "done;",
+
+            # Verify ownership
+            "for i in `seq 1 35`; do stat -c %U dir.$i | grep -F "
+            "\"qa_func\"; if [ $? -ne 0 ]; then exit 1; fi; for j in "
+            "`seq 1 5`; do stat -c %U dir.$i/file.$j | grep -F "
+            "\"qa_func\"; if [ $? -ne 0 ]; then exit 1; fi; done; done;"
+            " for i in `seq 36 70` ; do stat -c %U dir.$i | grep -F "
+            "\"qa_system\"; if [ $? -ne 0 ]; then exit 1; fi; for j in "
+            "`seq 1 5`; do stat -c %U dir.$i/file.$j | grep -F "
+            "\"qa_system\"; if [ $? -ne 0 ]; then exit 1; fi; done; done;"
+            " for i in `seq 71 100` ; do stat -c %U dir.$i | grep -F "
+            "\"qa_perf\"; if [ $? -ne 0 ]; then exit 1; fi; for j in "
+            "`seq 1 5`; do stat -c %U dir.$i/file.$j | grep -F "
+            "\"qa_perf\"; if [ $? -ne 0 ]; then exit 1; fi; done; done;",
+
+            # Verify group
+            "for i in `seq 1 100`; do stat -c %G dir.$i | grep -F "
+            "\"qa_all\"; if [ $? -ne 0 ]; then exit 1; fi; for j in "
+            "`seq 1 5`; do stat -c %G dir.$i/file.$j | grep -F "
+            "\"qa_all\"; if [ $? -ne 0 ]; then exit 1; fi; done; done;")
+
+        for io_cmd in cmds:
+            self._run_cmd(io_cmd, "Meta data of dirs and files not proper")
+
+    def _set_and_remove_extended_attributes(self, remove=False):
+        """Set and remove extended attributes"""
+        # Command to set extended attribute to files and dirs
+        io_cmd = ("for i in `seq 1 100`; do setfattr -n trusted.name -v "
+                  "testing_xattr_selfheal_on_dirs dir.$i; for j in `seq 1 "
+                  "5`;do setfattr -n trusted.name -v "
+                  "testing_xattr_selfheal_on_files dir.$i/file.$j; done; "
+                  "done;")
+        err_msg = "Failed to set extended attributes to files and dirs"
+        if remove:
+            # Command to remove extended attribute set on files and dirs
+            io_cmd = ("for i in `seq 1 100`; do setfattr -x trusted.name "
+                      "dir.$i; for j in `seq 1 5`; do setfattr -x "
+                      "trusted.name dir.$i/file.$j ; done ; done ;")
+            err_msg = "Failed to remove extended attributes to files and dirs"
+
+        self._run_cmd(io_cmd, err_msg)
+
+    def _verify_if_extended_attributes_are_proper(self, remove=False):
+        """Verify if extended attributes are set or remove properly"""
+        io_cmd = ("for i in `seq 1 100`; do getfattr -n trusted.name -e text "
+                  "dir.$i | grep -F 'testing_xattr_selfheal_on_dirs'; if [ $? "
+                  "-ne 0 ]; then exit 1 ; fi ; for j in `seq 1 5` ; do "
+                  "getfattr -n trusted.name -e text dir.$i/file.$j | grep -F "
+                  "'testing_xattr_selfheal_on_files'; if [ $? -ne 0 ]; then "
+                  "exit 1; fi; done; done;")
+        err_msg = "Extended attributes on files and dirs are not proper"
+        if remove:
+            io_cmd = ("for i in `seq 1 100`; do getfattr -n trusted.name -e "
+                      "text dir.$i; if [ $? -eq 0 ]; then exit 1; fi; for j in"
+                      " `seq 1 5`; do getfattr -n trusted.name -e text "
+                      "dir.$i/file.$j; if [ $? -eq 0]; then exit 1; fi; done; "
+                      "done;")
+            err_msg = "Extended attributes set to files and dirs not removed"
+        self._run_cmd(io_cmd, err_msg)
+
+    def _remove_files_and_create_dirs_with_the_same_name(self):
+        """Remove files and create dirs with the same name"""
+        io_cmd = ("for i in `seq 1 10`; do for j in `seq 1 5`; do for k in "
+                  "`seq 1 10`; do rm -f l1_dir.$i/l2_dir.$j/test.$k; mkdir "
+                  "l1_dir.$i/l2_dir.$j/test.$k; done; done; done;")
+        self._run_cmd(io_cmd,
+                      "Failed to remove files and create dirs with same name")
+
+    def _verify_if_dirs_are_proper_or_not(self):
+        """Verify if dirs are proper or not"""
+        io_cmd = ("for i in `seq 1 10`; do for j in `seq 1 5`; do for k in "
+                  "`seq 1 10`; do stat -c %F l1_dir.$i/l2_dir.$j/test.$k | "
+                  "grep -F 'directory'; if [ $? -ne 0 ]; then exit 1; fi; "
+                  "done; done; done;")
+        self._run_cmd(io_cmd, "Dirs created instead of files aren't proper")
+
+    def _bring_bricks_offline(self):
+        """Brings bricks offline and confirms if they are offline"""
+        # Select bricks to bring offline from a replica set
+        subvols_dict = get_subvols(self.mnode, self.volname)
+        subvols = subvols_dict['volume_subvols']
+        self.bricks_to_bring_offline = []
+        for subvol in subvols:
+            self.bricks_to_bring_offline.append(subvol[0])
+
+        # Bring bricks offline
+        ret = bring_bricks_offline(self.volname, self.bricks_to_bring_offline)
+        self.assertTrue(ret, 'Failed to bring bricks %s offline' %
+                        self.bricks_to_bring_offline)
+
+        ret = are_bricks_offline(self.mnode, self.volname,
+                                 self.bricks_to_bring_offline)
+        self.assertTrue(ret, 'Bricks %s are not offline'
+                        % self.bricks_to_bring_offline)
+        g.log.info('Bringing bricks %s offline is successful',
+                   self.bricks_to_bring_offline)
+
+    def _restart_volume_and_bring_all_offline_bricks_online(self):
+        """Restart volume and bring all offline bricks online"""
+        ret = bring_bricks_online(self.mnode, self.volname,
+                                  self.bricks_to_bring_offline,
+                                  bring_bricks_online_methods=[
+                                      'volume_start_force'])
+        self.assertTrue(ret, 'Failed to bring bricks %s online' %
+                        self.bricks_to_bring_offline)
+
+        # Check if bricks are back online or not
+        ret = are_bricks_online(self.mnode, self.volname,
+                                self.bricks_to_bring_offline)
+        self.assertTrue(ret, 'Bricks not online %s even after restart' %
+                        self.bricks_to_bring_offline)
+
+        g.log.info('Bringing bricks %s online is successful',
+                   self.bricks_to_bring_offline)
+
+    def _check_arequal_on_bricks_with_a_specific_arequal(self, arequal,
+                                                         brick_list):
+        """
+        Compare an inital arequal checksum with bricks from a given brick list
+        """
+        init_val = arequal[0].splitlines()[-1].split(':')[-1]
+        ret, arequals = collect_bricks_arequal(brick_list)
+        self.assertTrue(ret, 'Failed to get arequal on bricks')
+        for brick_arequal in arequals:
+            brick_total = brick_arequal.splitlines()[-1].split(':')[-1]
+            self.assertEqual(init_val, brick_total, 'Arequals not matching')
+
+    @staticmethod
+    def _add_dir_path_to_brick_list(brick_list):
+        """Add test_self_heal at the end of brick path"""
+        dir_brick_list = []
+        for brick in brick_list:
+            dir_brick_list.append('{}/{}'.format(brick, 'test_self_heal'))
+        return dir_brick_list
+
+    def _check_arequal_checksum_for_the_volume(self):
+        """
+        Check if arequals of mount point and bricks are
+        are the same.
+        """
+        if self.volume_type == "replicated":
+            # Check arequals for "replicated"
+            brick_list = get_all_bricks(self.mnode, self.volname)
+            dir_brick_list = self._add_dir_path_to_brick_list(brick_list)
+
+            # Get arequal before getting bricks offline
+            work_dir = '{}/test_self_heal'.format(self.mountpoint)
+            ret, arequals = collect_mounts_arequal([self.mounts[0]],
+                                                   path=work_dir)
+            self.assertTrue(ret, 'Failed to get arequal')
+            g.log.info('Getting arequal before getting bricks offline '
+                       'is successful')
+
+            # Get arequal on bricks and compare with mount_point_total
+            self._check_arequal_on_bricks_with_a_specific_arequal(
+                arequals, dir_brick_list)
+
+        # Check arequals for "distributed-replicated"
+        if self.volume_type == "distributed-replicated":
+            # Get the subvolumes
+            subvols_dict = get_subvols(self.mnode, self.volname)
+            num_subvols = len(subvols_dict['volume_subvols'])
+
+            # Get arequals and compare
+            for i in range(0, num_subvols):
+                # Get arequal for first brick
+                brick_list = subvols_dict['volume_subvols'][i]
+                dir_brick_list = self._add_dir_path_to_brick_list(brick_list)
+                ret, arequals = collect_bricks_arequal([dir_brick_list[0]])
+                self.assertTrue(ret, 'Failed to get arequal on first brick')
+
+                # Get arequal for every brick and compare with first brick
+                self._check_arequal_on_bricks_with_a_specific_arequal(
+                    arequals, dir_brick_list)
+
+    def _check_heal_is_completed_and_not_in_split_brain(self):
+        """Check if heal is completed and volume not in split brain"""
+        # Check if heal is completed
+        ret = is_heal_complete(self.mnode, self.volname)
+        self.assertTrue(ret, 'Heal is not complete')
+        g.log.info('Heal is completed successfully')
+
+        # Check if volume is in split brian or not
+        ret = is_volume_in_split_brain(self.mnode, self.volname)
+        self.assertFalse(ret, 'Volume is in split-brain state')
+        g.log.info('Volume is not in split-brain state')
+
+    def _check_if_there_are_files_and_dirs_to_be_healed(self):
+        """Check if there are files and dirs to be healed"""
+        ret = is_heal_complete(self.mnode, self.volname)
+        self.assertFalse(ret, 'Heal is completed')
+        g.log.info('Heal is pending')
+
+    def _wait_for_heal_is_completed(self):
+        """Check if heal is completed"""
+        ret = monitor_heal_completion(self.mnode, self.volname,
+                                      timeout_period=3600)
+        self.assertTrue(ret, 'Heal has not yet completed')
+
+    def _check_heal_status_restart_vol_wait_and_check_data(self):
+        """
+        Perform repatative steps mentioned below:
+        1 Check if heal info is showing all the files and dirs to be healed
+        2 Bring back all brick processes which were killed
+        3 Wait for heal to complete on the volume
+        4 Check if heal is complete and check if volume is in split brain
+        5 Collect and compare arequal-checksum according to the volume type
+          for bricks
+        """
+        # Check if heal info is showing all the files and dirs to be healed
+        self._check_if_there_are_files_and_dirs_to_be_healed()
+
+        # Bring back all brick processes which were killed
+        self._restart_volume_and_bring_all_offline_bricks_online()
+
+        # Wait for heal to complete on the volume
+        self._wait_for_heal_is_completed()
+
+        # Check if heal is complete and check if volume is in split brain
+        self._check_heal_is_completed_and_not_in_split_brain()
+
+        # Collect and compare arequal-checksum according to the volume type
+        # for bricks
+        self._check_arequal_checksum_for_the_volume()
+
+    def _run_test_self_heal_entry_heal(self):
+        """Run steps of test_self_heal_entry_heal"""
+        # Create a directory and create files and directories inside it on
+        # mount point
+        self._create_files_and_dirs_on_mount_point(0)
+
+        # Collect and compare arequal-checksum according to the volume type
+        # for bricks
+        self._check_arequal_checksum_for_the_volume()
+
+        # Bring down brick processes accoding to the volume type
+        self._bring_bricks_offline()
+
+        # Create a new set of files and directories on mount point
+        self._create_files_and_dirs_on_mount_point(3, second_set=True)
+
+        self._check_heal_status_restart_vol_wait_and_check_data()
+
+        # Bring down brick processes accoding to the volume type
+        self._bring_bricks_offline()
+
+        # Delete files and directories from mount point
+        self._delete_files_and_dirs()
+
+        self._check_heal_status_restart_vol_wait_and_check_data()
+
+        # Bring down brick processes accoding to the volume type
+        self._bring_bricks_offline()
+
+        # Rename the existing files and dirs
+        self._rename_files_and_dirs()
+
+        self._check_heal_status_restart_vol_wait_and_check_data()
+
+    def test_self_heal_entry_heal(self):
+        """
+        Test case:
+        1. Create a volume, start it and mount it.
+        2. Create a directory and create files and directories inside it
+           on mount point.
+        3. Collect and compare arequal-checksum according to the volume type
+           for bricks.
+        4. Bring down brick processes accoding to the volume type.
+        5. Create a new set of files and directories on mount point.
+        6. Check if heal info is showing all the files and dirs to be healed.
+        7. Bring back all brick processes which were killed.
+        8. Wait for heal to complete on the volume.
+        9. Check if heal is complete and check if volume is in split brain.
+        10. Collect and compare arequal-checksum according to the volume type
+            for bricks.
+        11. Bring down brick processes accoding to the volume type.
+        12. Delete files and directories from mount point.
+        13. Check if heal info is showing all the files and dirs to be healed.
+        14. Bring back all brick processes which were killed.
+        15. Wait for heal to complete on the volume.
+        16. Check if heal is complete and check if volume is in split brain.
+        17. Collect and compare arequal-checksum according to the volume type
+            for bricks.
+        18. Bring down brick processes accoding to the volume type.
+        19. Rename the existing files and dirs.
+        20. Check if heal info is showing all the files and dirs to be healed.
+        21. Bring back all brick processes which were killed.
+        22. Wait for heal to complete on the volume.
+        23. Check if heal is complete and check if volume is in split brain.
+        24. Collect and compare arequal-checksum according to the volume type
+            for bricks.
+
+        Note:
+        Do this test with both Granular-entry-heal set enable and disable.
+        """
+        for value in (False, True):
+            if value:
+                # Cleanup old data from mount point
+                ret, _, _ = g.run(self.first_client,
+                                  'rm -rf {}/*'.format(self.mountpoint))
+                self.assertFalse(ret, 'Failed to cleanup mount point')
+                g.log.info("Testing with granular heal set to enabled")
+            self._set_granular_heal_to_on_or_off(enabled=value)
+            self._run_test_self_heal_entry_heal()
+
+    def test_self_heal_meta_data(self):
+        """
+        Test case:
+        1. Create a volume, start it and mount it.
+        2. Create a directory and create files and directories inside it
+           on mount point.
+        3. Collect and compare arequal-checksum according to the volume type
+           for bricks.
+        4. Bring down brick processes accoding to the volume type.
+        5. Change the meta data of files and dirs.
+        6. Check if heal info is showing all the files and dirs to be healed.
+        7. Bring back all brick processes which were killed.
+        8. Wait for heal to complete on the volume.
+        9. Check if heal is complete and check if volume is in split brain.
+        10. Collect and compare arequal-checksum according to the volume type
+            for bricks.
+        11. Verify if the meta data of files and dirs.
+        12. Bring down brick processes accoding to the volume type.
+        13. Set extended attributes on the files and dirs.
+        14. Verify if the extended attributes are set properly or not.
+        15. Check if heal info is showing all the files and dirs to be healed.
+        16. Bring back all brick processes which were killed.
+        17. Wait for heal to complete on the volume.
+        18. Check if heal is complete and check if volume is in split brain.
+        19. Collect and compare arequal-checksum according to the volume type
+            for bricks.
+        20. Verify if extended attributes are consitent or not.
+        21. Bring down brick processes accoding to the volume type
+        22. Remove extended attributes on the files and dirs.
+        23. Verify if extended attributes were removed properly.
+        24. Check if heal info is showing all the files and dirs to be healed.
+        25. Bring back all brick processes which were killed.
+        26. Wait for heal to complete on the volume.
+        27. Check if heal is complete and check if volume is in split brain.
+        28. Collect and compare arequal-checksum according to the volume type
+            for bricks.
+        29. Verify if extended attributes are removed or not.
+        """
+        # Create a directory and create files and directories inside it
+        # on mount point
+        self._create_files_and_dirs_on_mount_point(1)
+
+        # Collect and compare arequal-checksum according to the volume type
+        # for bricks
+        self._check_arequal_checksum_for_the_volume()
+
+        # Bring down brick processes accoding to the volume type
+        self._bring_bricks_offline()
+
+        # Change the meta data of files and dirs
+        self._change_meta_deta_of_dirs_and_files()
+
+        self._check_heal_status_restart_vol_wait_and_check_data()
+
+        # Verify if the meta data of files and dirs
+        self._verify_meta_data_of_files_and_dirs()
+
+        for value in (False, True):
+            # Bring down brick processes accoding to the volume type
+            self._bring_bricks_offline()
+
+            # Set or remove extended attributes on the files and dirs
+            self._set_and_remove_extended_attributes(remove=value)
+
+            # Verify if the extended attributes are set properly or not
+            self._verify_if_extended_attributes_are_proper(remove=value)
+
+            self._check_heal_status_restart_vol_wait_and_check_data()
+
+            # Verify if extended attributes are consitent or not
+            self._verify_if_extended_attributes_are_proper(remove=value)
+
+    def test_self_heal_of_dir_with_files_removed(self):
+        """
+        Test case:
+        1. Create a volume, start it and mount it.
+        2. Create a directory and create files and directories inside it
+           on mount point.
+        3. Collect and compare arequal-checksum according to the volume type
+           for bricks.
+        4. Bring down brick processes accoding to the volume type.
+        5. Remove all files and create dir which have name of files.
+        6. Check if heal info is showing all the files and dirs to be healed.
+        7. Bring back all brick processes which were killed.
+        8. Wait for heal to complete on the volume.
+        9. Check if heal is complete and check if volume is in split brain.
+        10. Collect and compare arequal-checksum according to the volume type
+            for bricks.
+        11. Verify if dirs are healed properly or not.
+        """
+        # Create a directory and create files and directories inside it
+        # on mount point
+        self._create_files_and_dirs_on_mount_point(2)
+
+        # Collect and compare arequal-checksum according to the volume type
+        # for bricks
+        self._check_arequal_checksum_for_the_volume()
+
+        # Bring down brick processes accoding to the volume type
+        self._bring_bricks_offline()
+
+        # Remove all files and create dir which have name of files
+        self._remove_files_and_create_dirs_with_the_same_name()
+
+        self._check_heal_status_restart_vol_wait_and_check_data()
+
+        # Verify if dirs are healed properly or not
+        self._verify_if_dirs_are_proper_or_not()