From 10e5edfb5b2187c64bb65429aeffc2ba3206743f Mon Sep 17 00:00:00 2001 From: sayaleeraut Date: Mon, 5 Oct 2020 20:10:37 +0530 Subject: [Test] Validate creation of different file types This test script covers below scenarios: 1) Creation of various file types - regular, block, character and pipe file 2) Hard link create, validate 3) Symbolic link create, validate Issue : Fails on CI due to- https://github.com/gluster/glusterfs/issues/1461 Change-Id: If50b8d697115ae7c23b4d30e0f8946e9fe705ece Signed-off-by: sayaleeraut --- tests/functional/dht/test_file_creation.py | 494 +++++++++++++++++++++++++++++ 1 file changed, 494 insertions(+) create mode 100644 tests/functional/dht/test_file_creation.py (limited to 'tests/functional/dht') diff --git a/tests/functional/dht/test_file_creation.py b/tests/functional/dht/test_file_creation.py new file mode 100644 index 000000000..5671cb84b --- /dev/null +++ b/tests/functional/dht/test_file_creation.py @@ -0,0 +1,494 @@ +# Copyright (C) 2020 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from glusto.core import Glusto as g +from glustolibs.gluster.exceptions import ExecutionError +from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on +from glustolibs.gluster.glusterfile import (get_file_stat, get_pathinfo, + file_exists, create_link_file, + get_md5sum, get_fattr) +from glustolibs.gluster.lib_utils import append_string_to_file + + +@runs_on([['distributed', 'distributed-arbiter', + 'distributed-replicated', 'distributed-dispersed'], + ['glusterfs']]) +class TestFileCreation(GlusterBaseClass): + def setUp(self): + + # Calling GlusterBaseClass setUp + self.get_super_method(self, 'setUp')() + + # Setup Volume and Mount Volume + ret = self.setup_volume_and_mount_volume([self.mounts[0]]) + if not ret: + raise ExecutionError("Failed to Setup_Volume and Mount_Volume") + g.log.info("Successful in Setup Volume and Mount Volume") + self.client, self.m_point = (self.mounts[0].client_system, + self.mounts[0].mountpoint) + + def tearDown(self): + + # Unmount and cleanup original volume + ret = self.unmount_volume_and_cleanup_volume(mounts=[self.mounts[0]]) + if not ret: + raise ExecutionError("Failed to umount the vol & cleanup Volume") + g.log.info("Successful in umounting the volume and Cleanup") + + # Calling GlusterBaseClass tearDown + self.get_super_method(self, 'tearDown')() + + def _create_file_using_touch(self, file_name): + """Creates a regular empty file""" + cmd = "touch {}/{}".format(self.m_point, file_name) + ret, _, _ = g.run(self.client, cmd) + self.assertEqual(ret, 0, "Failed to create file {}".format(file_name)) + g.log.info("Successfully created file %s", file_name) + + def _check_file_stat_on_mountpoint(self, file_name, file_type): + """Check the file-type on mountpoint""" + file_stat = (get_file_stat(self.client, "{}/{}".format( + self.m_point, file_name + )))['filetype'] + self.assertEqual(file_stat, file_type, + "File is not a {}".format(file_type)) + g.log.info("File is %s", file_type) + + def _is_file_present_on_brick(self, file_name): + """Check if file is created on the backend-bricks as per + the value of trusted.glusterfs.pathinfo xattr""" + brick_list = get_pathinfo(self.client, "{}/{}".format( + self.m_point, file_name)) + self.assertNotEqual( + brick_list, 0, "Failed to get bricklist for {}".format(file_name)) + + for brick in brick_list['brickdir_paths']: + host, path = brick.split(':') + ret = file_exists(host, path) + self.assertTrue(ret, "File {} is not present on {}".format( + file_name, brick + )) + g.log.info("File %s is present on %s", file_name, brick) + + def _compare_file_permissions(self, file_name, + file_info_mnt=None, file_info_brick=None): + """Check if the file's permission are same on mountpoint and + backend-bricks""" + if (file_info_mnt is None and file_info_brick is None): + file_info_mnt = (get_file_stat(self.client, "{}/{}".format( + self.m_point, file_name + )))['access'] + self.assertIsNotNone( + file_info_mnt, "Failed to get access time for {}".format( + file_name)) + brick_list = get_pathinfo(self.client, "{}/{}".format( + self.m_point, file_name)) + self.assertNotEqual( + brick_list, 0, "Failed to get bricklist for {}".format( + file_name)) + file_info_brick = [] + for brick in brick_list['brickdir_paths']: + host, path = brick.split(':') + info_brick = (get_file_stat(host, path))['access'] + file_info_brick.append(info_brick) + + for info in file_info_brick: + self.assertEqual(info, file_info_mnt, + "File details for {} are diffrent on" + " backend-brick".format(file_name)) + g.log.info("Details for file %s is correct" + " on backend-bricks", file_name) + + def _check_change_time_mnt(self, file_name): + """Find out the modification time for file on mountpoint""" + file_ctime_mnt = (get_file_stat(self.client, "{}/{}".format( + self.m_point, file_name + )))['epoch_ctime'] + return file_ctime_mnt + + def _check_change_time_brick(self, file_name): + """Find out the modification time for file on backend-bricks""" + brick_list = get_pathinfo(self.client, "{}/{}".format( + self.m_point, file_name)) + self.assertNotEqual(brick_list, 0, + "Failed to get bricklist for {}".format(file_name)) + + brick_mtime = [] + for brick in brick_list['brickdir_paths']: + host, path = brick.split(':') + cmd = "ls -lR {}".format(path) + ret, _, _ = g.run(host, cmd) + self.assertEqual(ret, 0, "Lookup failed on" + " brick:{}".format(path)) + file_ctime_brick = (get_file_stat(host, path))['epoch_ctime'] + brick_mtime.append(file_ctime_brick) + return brick_mtime + + def _compare_file_perm_mnt(self, mtime_before, mtime_after, + file_name): + """Compare the file permissions before and after appending data""" + self.assertNotEqual(mtime_before, mtime_after, "Unexpected:" + "The ctime has not been changed") + g.log.info("The modification time for %s has been" + " changed as expected", file_name) + + def _collect_and_compare_file_info_on_mnt( + self, link_file_name, values, expected=True): + """Collect the files's permissions on mountpoint and compare""" + stat_test_file = get_file_stat( + self.client, "{}/test_file".format(self.m_point)) + self.assertIsNotNone(stat_test_file, "Failed to get stat of test_file") + stat_link_file = get_file_stat( + self.client, "{}/{}".format(self.m_point, link_file_name)) + self.assertIsNotNone(stat_link_file, "Failed to get stat of {}".format( + link_file_name)) + + for key in values: + if expected is True: + self.assertEqual(stat_test_file[key], stat_link_file[key], + "The {} is not same for test_file" + " and {}".format(key, link_file_name)) + g.log.info("The %s for test_file and %s is same on mountpoint", + key, link_file_name) + else: + self.assertNotEqual(stat_test_file[key], stat_link_file[key], + "Unexpected : The {} is same for test_file" + " and {}".format(key, link_file_name)) + g.log.info("The %s for test_file and %s is different" + " on mountpoint", key, link_file_name) + + def _compare_file_md5sum_on_mnt(self, link_file_name): + """Collect and compare the md5sum for file on mountpoint""" + md5sum_test_file, _ = (get_md5sum( + self.client, "{}/test_file".format(self.m_point))).split() + self.assertIsNotNone( + md5sum_test_file, "Failed to get md5sum for test_file") + + md5sum_link_file, _ = get_md5sum( + self.client, "{}/{}".format(self.m_point, link_file_name)).split() + self.assertIsNotNone(md5sum_link_file, "Failed to get" + " md5sum for {}".format(link_file_name)) + self.assertEqual(md5sum_test_file, md5sum_link_file, + "The md5sum for test_file and {} is" + " not same".format(link_file_name)) + g.log.info("The md5sum is same for test_file and %s" + " on mountpoint", link_file_name) + + def _compare_file_md5sum_on_bricks(self, link_file_name): + """Collect and compare md5sum for file on backend-bricks""" + brick_list_test_file = get_pathinfo(self.client, "{}/test_file".format( + self.m_point)) + md5sum_list_test_file = [] + for brick in brick_list_test_file['brickdir_paths']: + host, path = brick.split(':') + md5sum_test_file, _ = (get_md5sum(host, path)).split() + md5sum_list_test_file.append(md5sum_test_file) + + brick_list_link_file = get_pathinfo(self.client, "{}/{}".format( + self.m_point, link_file_name)) + md5sum_list_link_file = [] + for brick in brick_list_link_file['brickdir_paths']: + md5sum_link_file, _ = (get_md5sum(host, path)).split() + md5sum_list_link_file.append(md5sum_link_file) + + self.assertEqual(md5sum_test_file, md5sum_link_file, + "The md5sum for test_file and {} is" + " not same on brick {}".format(link_file_name, brick)) + g.log.info("The md5sum for test_file and %s is same" + " on backend brick %s", link_file_name, brick) + + def _compare_gfid_xattr_on_files(self, link_file_name, expected=True): + """Collect and compare the value of trusted.gfid xattr for file + on backend-bricks""" + brick_list_test_file = get_pathinfo(self.client, "{}/test_file".format( + self.m_point)) + xattr_list_test_file = [] + for brick in brick_list_test_file['brickdir_paths']: + host, path = brick.split(':') + xattr_test_file = get_fattr(host, path, "trusted.gfid") + xattr_list_test_file.append(xattr_test_file) + + brick_list_link_file = get_pathinfo(self.client, "{}/{}".format( + self.m_point, link_file_name)) + xattr_list_link_file = [] + for brick in brick_list_link_file['brickdir_paths']: + host, path = brick.split(':') + xattr_link_file = get_fattr(host, path, "trusted.gfid") + xattr_list_link_file.append(xattr_link_file) + + if expected is True: + self.assertEqual(xattr_list_test_file, xattr_list_link_file, + "Unexpected: The xattr trusted.gfid is not same " + "for test_file and {}".format(link_file_name)) + g.log.info("The xattr trusted.gfid is same for test_file" + " and %s", link_file_name) + else: + self.assertNotEqual(xattr_list_test_file, xattr_list_link_file, + "Unexpected: The xattr trusted.gfid is same " + "for test_file and {}".format(link_file_name)) + g.log.info("The xattr trusted.gfid is not same for test_file" + " and %s", link_file_name) + + def test_special_file_creation(self): + """ + Description : check creation of different types of files. + + Steps: + 1) From mount point, Create a regular file + eg: + touch f1 + - From mount point, create character, block device and pipe files + mknod c + mknod b + mkfifo + 2) Stat on the files created in Step-2 from mount point + 3) Verify that file is stored on only one bricks which is mentioned in + trusted.glusterfs.pathinfo xattr + On mount point - + " getfattr -n trusted.glusterfs.pathinfo + On all bricks + " ls / " + 4) Verify that file permissions are same on mount point and sub-volumes + " stat " + 5) Append some data to the file. + 6) List content of file to verify that data has been appended. + " cat " + 7) Verify that file change time and size has been updated + accordingly(from mount point and sub-volume) + " stat / " + """ + # pylint: disable=too-many-statements + # pylint: disable=too-many-locals + # Create a regular file + self._create_file_using_touch("regfile") + + # Create a character and block file + for (file_name, parameter) in [ + ("blockfile", "b"), ("charfile", "c")]: + cmd = "mknod {}/{} {} 1 5".format(self.m_point, file_name, + parameter) + ret, _, _ = g.run(self.client, cmd) + self.assertEqual( + ret, 0, "Failed to create {} file".format(file_name)) + g.log.info("%s file created successfully", file_name) + + # Create a pipe file + cmd = "mkfifo {}/pipefile".format(self.m_point) + ret, _, _ = g.run(self.client, cmd) + self.assertEqual(ret, 0, "Failed to create pipe file") + g.log.info("Pipe file is created successfully") + + # Stat all the files created on mount-point + for (file_name, check_string) in [ + ("regfile", "regular empty file"), + ("charfile", "character special file"), + ("blockfile", "block special file"), + ("pipefile", "fifo")]: + self._check_file_stat_on_mountpoint(file_name, check_string) + + # Verify files are stored on backend bricks as per + # the trusted.glusterfs.pathinfo + file_types = ["regfile", "charfile", "blockfile", "pipefile"] + + for file_name in file_types: + self._is_file_present_on_brick(file_name) + + # Verify that the file permissions are same on + # mount-point and bricks + for file_name in file_types: + self._compare_file_permissions(file_name) + + # Note the modification time on mount and bricks + # for all files. Also it should be same on mnt and bricks + reg_mnt_ctime_1 = self._check_change_time_mnt("regfile") + char_mnt_ctime_1 = self._check_change_time_mnt("charfile") + block_mnt_ctime_1 = self._check_change_time_mnt("blockfile") + fifo_mnt_ctime_1 = self._check_change_time_mnt("pipefile") + + reg_brick_ctime_1 = self._check_change_time_brick("regfile") + char_brick_ctime_1 = self._check_change_time_brick("charfile") + block_brick_ctime_1 = self._check_change_time_brick("blockfile") + fifo_brick_ctime_1 = self._check_change_time_brick("pipefile") + + for (file_name, mnt_ctime, brick_ctime) in [ + ("regfile", reg_mnt_ctime_1, reg_brick_ctime_1), + ("charfile", char_mnt_ctime_1, char_brick_ctime_1), + ("blockfile", block_mnt_ctime_1, block_brick_ctime_1), + ("pipefile", fifo_mnt_ctime_1, fifo_brick_ctime_1)]: + self._compare_file_permissions( + file_name, mnt_ctime, brick_ctime) + + # Append some data to the files + for (file_name, data_str) in [ + ("regfile", "regular"), + ("charfile", "character special"), + ("blockfile", "block special")]: + ret = append_string_to_file( + self.client, "{}/{}".format(self.m_point, file_name), + "Welcome! This is a {} file".format(data_str)) + self.assertTrue( + ret, "Failed to append data to {}".format(file_name)) + g.log.info( + "Successfully appended data to %s", file_name) + + # Check if the data has been appended + check = "Welcome! This is a regular file" + cmd = "cat {}/{}".format(self.m_point, "regfile") + ret, out, _ = g.run(self.client, cmd) + self.assertEqual(out.strip(), check, "No data present at regfile") + + # Append data to pipefile and check if it has been appended + g.run_async(self.client, "echo 'Hello' > {}/{} ".format( + self.m_point, "pipefile")) + ret, out, _ = g.run( + self.client, "cat < {}/{}".format(self.m_point, "pipefile")) + self.assertEqual( + ret, 0, "Unable to fetch datat on other terimnal") + self.assertEqual( + "Hello", out.split('\n')[0], + "Hello not recieved on the second terimnal") + + # Lookup on mount-point + cmd = "ls -lR {}".format(self.m_point) + ret, _, _ = g.run(self.client, cmd) + self.assertEqual(ret, 0, "Lookup on mountpoint failed") + + # Collect ctime on mount point after appending data + reg_mnt_ctime_2 = self._check_change_time_mnt("regfile") + + # After appending data the ctime for file should change + self.assertNotEqual(reg_mnt_ctime_1, reg_mnt_ctime_2, "Unexpected:" + "The ctime has not been changed") + g.log.info("The modification time for regfile has been" + " changed as expected") + + # Collect the ctime on bricks + reg_brick_ctime_2 = self._check_change_time_brick("regfile") + + # Check if the ctime has changed on bricks as per mount + self._compare_file_permissions( + "regfile", reg_mnt_ctime_2, reg_brick_ctime_2) + + def test_hard_link_file(self): + """ + Description: link file create, validate and access file + using it + + Steps: + 1) From mount point, create a regular file + 2) Verify that file is stored on only on bricks which is + mentioned in trusted.glusterfs.pathinfo xattr + 3) From mount point create hard-link file for the created file + 4) From mount point stat on the hard-link file and original file; + file inode, permission, size should be same + 5) From mount point, verify that file contents are same + "md5sum" + 6) Verify "trusted.gfid" extended attribute of the file + on sub-vol + 7) From sub-volume stat on the hard-link file and original file; + file inode, permission, size should be same + 8) From sub-volume verify that content of file are same + """ + # Create a regular file + self._create_file_using_touch("test_file") + + # Check file is create on bricks as per trusted.glusterfs.pathinfo + self._is_file_present_on_brick("test_file") + + # Create a hard-link file for the test_file + ret = create_link_file( + self.client, "{}/test_file".format(self.m_point), + "{}/hardlink_file".format(self.m_point)) + self.assertTrue(ret, "Failed to create hard link file for" + " test_file") + g.log.info("Successfully created hardlink_file") + + # On mountpoint perform stat on original and hard-link file + values = ["inode", "access", "size"] + self._collect_and_compare_file_info_on_mnt( + "hardlink_file", values, expected=True) + + # Check the md5sum on original and hard-link file on mountpoint + self._compare_file_md5sum_on_mnt("hardlink_file") + + # Compare the value of trusted.gfid for test_file and hard-link file + # on backend-bricks + self._compare_gfid_xattr_on_files("hardlink_file") + + # On backend bricks perform stat on original and hard-link file + values = ["inode", "access", "size"] + self._collect_and_compare_file_info_on_mnt("hardlink_file", values) + + # On backend bricks check the md5sum + self._compare_file_md5sum_on_bricks("hardlink_file") + + def test_symlink_file(self): + """ + Description: Create symbolic link file, validate and access file + using it + + Steps: + 1) From mount point, create a regular file + 2) Verify that file is stored on only on bricks which is + mentioned in trusted.glusterfs.pathinfo xattr + 3) From mount point create symbolic link file for the created file + 4) From mount point stat on the symbolic link file and original file; + file inode should be different + 5) From mount point, verify that file contents are same + "md5sum" + 6) Verify "trusted.gfid" extended attribute of the file + on sub-vol + 7) Verify readlink on symbolic link from mount point + "readlink " + 8) From sub-volume verify that content of file are same + """ + # Create a regular file on mountpoint + self._create_file_using_touch("test_file") + + # Check file is create on bricks as per trusted.glusterfs.pathinfo + self._is_file_present_on_brick("test_file") + + # Create a symbolic-link file for the test_file + ret = create_link_file( + self.client, "{}/test_file".format(self.m_point), + "{}/softlink_file".format(self.m_point), soft=True) + self.assertTrue(ret, "Failed to create symbolic link file for" + " test_file") + g.log.info("Successfully created softlink_file") + + # On mountpoint perform stat on original and symbolic-link file + # The value of inode should be different + values = ["inode"] + self._collect_and_compare_file_info_on_mnt( + "softlink_file", values, expected=False) + + # Check the md5sum on original and symbolic-link file on mountpoint + self._compare_file_md5sum_on_mnt("softlink_file") + + # Compare the value of trusted.gfid for test_file and + # symbolic-link file on backend-bricks + self._compare_gfid_xattr_on_files("softlink_file") + + # Verify readlink on symbolic-link from mount point + cmd = "readlink {}/softlink_file".format(self.m_point) + ret, out, _ = g.run(self.client, cmd) + self.assertEqual( + out.strip(), "{}/test_file".format(self.m_point), + "Symbolic link points to incorrect file") + g.log.info("Symbolic link points to correct file") + + # Check the md5sum on original and symbolic-link file on backend bricks + self._compare_file_md5sum_on_bricks("softlink_file") -- cgit