From b08b734f4afd15e9d18fee7b4b96ef62cf27be6e Mon Sep 17 00:00:00 2001 From: srijan-sivakumar Date: Wed, 16 Sep 2020 11:50:04 +0530 Subject: [Test] Rebalance with brick down in replica Steps- 1. Create a Replica volume. 2. Bring down one of the brick down in the replica pair 3. Do some IO and create files on the mount point 4. Add a pair of bricks to the volume 5. Initiate rebalance 6. Bring back the brick which was down 7. After self heal happens, all the files should be present. Change-Id: I78a42866d585b00c40a2712c4ae8f2ab3552adca Signed-off-by: srijan-sivakumar --- .../dht/test_rebalance_with_brick_down.py | 171 +++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 tests/functional/dht/test_rebalance_with_brick_down.py (limited to 'tests/functional/dht') diff --git a/tests/functional/dht/test_rebalance_with_brick_down.py b/tests/functional/dht/test_rebalance_with_brick_down.py new file mode 100644 index 000000000..fd0a0ffe2 --- /dev/null +++ b/tests/functional/dht/test_rebalance_with_brick_down.py @@ -0,0 +1,171 @@ +# Copyright (C) 2020 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along` +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-131 USA. + +""" +Description: + Rebalance with one brick down in replica +""" + +from random import choice + +from glusto.core import Glusto as g + +from glustolibs.gluster.exceptions import ExecutionError +from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on +from glustolibs.gluster.rebalance_ops import (rebalance_start, + wait_for_rebalance_to_complete) +from glustolibs.gluster.volume_libs import ( + expand_volume, + log_volume_info_and_status, + volume_start) +from glustolibs.gluster.brick_libs import ( + get_all_bricks, + bring_bricks_offline) +from glustolibs.gluster.heal_libs import monitor_heal_completion +from glustolibs.io.utils import ( + wait_for_io_to_complete, + collect_mounts_arequal) +from glustolibs.misc.misc_libs import upload_scripts + + +@runs_on([['distributed-arbiter', 'distributed-replicated', + 'distributed-dispersed'], ['glusterfs']]) +class TestRebalanceWithBrickDown(GlusterBaseClass): + """ Rebalance with brick down in replica""" + + def setUp(self): + """Setup Volume""" + # Calling GlusterBaseClass setUp + self.get_super_method(self, 'setUp')() + self.all_mounts_procs = [] + + # Setup and mount the volume + g.log.info("Starting to setup and mount the volume") + ret = self.setup_volume_and_mount_volume(mounts=self.mounts) + if not ret: + raise ExecutionError("Failed to Setup Volume and Mount it") + + # Upload IO script for running IO on mounts + self.script_upload_path = ("/usr/share/glustolibs/io/scripts/" + "file_dir_ops.py") + ret = upload_scripts(self.mounts[0].client_system, + self.script_upload_path) + if not ret: + raise ExecutionError("Failed to upload IO scripts to clients") + + def test_rebalance_with_brick_down(self): + """ + Rebalance with brick down in replica + - Create a Replica volume. + - Bring down one of the brick down in the replica pair + - Do some IO and create files on the mount point + - Add a pair of bricks to the volume + - Initiate rebalance + - Bring back the brick which was down. + - After self heal happens, all the files should be present. + """ + # Log the volume info and status before brick is down. + log_volume_info_and_status(self.mnode, self.volname) + + # Bring one fo the bricks offline + brick_list = get_all_bricks(self.mnode, self.volname) + ret = bring_bricks_offline(self.volname, choice(brick_list)) + + # Log the volume info and status after brick is down. + log_volume_info_and_status(self.mnode, self.volname) + + # Create files at mountpoint. + cmd = ( + "/usr/bin/env python %s create_files " + "-f 2000 --fixed-file-size 1k --base-file-name file %s" + % (self.script_upload_path, self.mounts[0].mountpoint)) + proc = g.run_async( + self.mounts[0].client_system, cmd, user=self.mounts[0].user) + self.all_mounts_procs.append(proc) + + # Wait for IO to complete. + self.assertTrue(wait_for_io_to_complete(self.all_mounts_procs, + self.mounts[0]), + "IO failed on some of the clients") + g.log.info("IO completed on the clients") + + # Compute the arequal checksum before bringing all bricks online + arequal_before_all_bricks_online = collect_mounts_arequal(self.mounts) + + # Log the volume info and status before expanding volume. + log_volume_info_and_status(self.mnode, self.volname) + + # Expand the volume. + ret = expand_volume(self.mnode, self.volname, self.servers, + self.all_servers_info) + self.assertTrue(ret, ("Failed to expand the volume %s", self.volname)) + g.log.info("Expanding volume is successful on " + "volume %s", self.volname) + + # Log the voluem info after expanding volume. + log_volume_info_and_status(self.mnode, self.volname) + + # Start Rebalance. + ret, _, _ = rebalance_start(self.mnode, self.volname) + self.assertEqual(ret, 0, ("Failed to start rebalance on the volume " + "%s", self.volname)) + g.log.info("Successfully started rebalance on the volume %s", + self.volname) + + # Wait for rebalance to complete + ret = wait_for_rebalance_to_complete(self.mnode, self.volname) + self.assertTrue(ret, ("Rebalance is not yet complete on the volume " + "%s", self.volname)) + g.log.info("Rebalance is successfully complete on the volume %s", + self.volname) + + # Log the voluem info and status before bringing all bricks online + log_volume_info_and_status(self.mnode, self.volname) + + # Bring all bricks online. + ret, _, _ = volume_start(self.mnode, self.volname, force=True) + self.assertEqual(ret, 0, "Not able to start volume with force option") + g.log.info("Volume start with force option successful.") + + # Log the volume info and status after bringing all beicks online + log_volume_info_and_status(self.mnode, self.volname) + + # Monitor heal completion. + ret = monitor_heal_completion(self.mnode, self.volname) + self.assertTrue(ret, "heal has not yet completed") + g.log.info("Self heal completed") + + # Compute the arequal checksum after all bricks online. + arequal_after_all_bricks_online = collect_mounts_arequal(self.mounts) + + # Comparing arequal checksum before and after the operations. + self.assertEqual(arequal_before_all_bricks_online, + arequal_after_all_bricks_online, + "arequal checksum is NOT MATCHING") + g.log.info("arequal checksum is SAME") + + def tearDown(self): + """tear Down callback""" + # Unmount Volume and cleanup. + g.log.info("Starting to Unmount Volume and Cleanup") + ret = self.unmount_volume_and_cleanup_volume(mounts=self.mounts) + if not ret: + raise ExecutionError("Filed to Unmount Volume and " + "Cleanup Volume") + g.log.info("Successful in Unmount Volume and cleanup.") + + # Calling GlusterBaseClass tearDown + self.get_super_method(self, 'tearDown')() -- cgit