From d557d097851d335effe0a2e810ca3f664c30e93f Mon Sep 17 00:00:00 2001 From: Krutika Dhananjay Date: Fri, 18 Nov 2016 15:38:00 +0530 Subject: features/index: Delete granular entry indices of already healed directories during crawl Backport of: http://review.gluster.org/15880 If granular name indices are already in existence for a volume, and before they are healed, granular entry heal be disabled, a crawl on indices/xattrop will clear the changelogs on these directories. When their corresponding entry-changes indices are crawled subsequently, if it is found that the directories don't need heal anymore, the granular indices are not cleaned up. This patch fixes that problem by ensuring that the zero-xattrop also deletes the stale indices at the level of index translator. Change-Id: Iae0a560c1c9d37b083cad89f16d3dcf83c4f7dc7 BUG: 1398501 Signed-off-by: Krutika Dhananjay Reviewed-on: http://review.gluster.org/15927 Smoke: Gluster Build System NetBSD-regression: NetBSD Build System CentOS-regression: Gluster Build System Reviewed-by: Pranith Kumar Karampuri --- .../granular-indices-but-non-granular-heal.t | 76 ++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tests/basic/afr/granular-esh/granular-indices-but-non-granular-heal.t (limited to 'tests/basic') diff --git a/tests/basic/afr/granular-esh/granular-indices-but-non-granular-heal.t b/tests/basic/afr/granular-esh/granular-indices-but-non-granular-heal.t new file mode 100644 index 00000000000..2da90a98c76 --- /dev/null +++ b/tests/basic/afr/granular-esh/granular-indices-but-non-granular-heal.t @@ -0,0 +1,76 @@ +#!/bin/bash + +. $(dirname $0)/../../../include.rc +. $(dirname $0)/../../../volume.rc +. $(dirname $0)/../../../afr.rc + +cleanup + +TESTS_EXPECTED_IN_LOOP=4 + +TEST glusterd +TEST pidof glusterd + +TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1} +TEST $CLI volume start $V0 +TEST $CLI volume set $V0 cluster.data-self-heal off +TEST $CLI volume set $V0 cluster.metadata-self-heal off +TEST $CLI volume set $V0 cluster.entry-self-heal off +TEST $CLI volume set $V0 self-heal-daemon off +TEST $CLI volume set $V0 granular-entry-heal on + +TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0 + +# Kill brick-0. +TEST kill_brick $V0 $H0 $B0/${V0}0 + +# Create files under root +for i in {1..2} +do + echo $i > $M0/f$i +done + +# Test that the index associated with '/' is created on B1. +TEST stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID + +# Check for successful creation of granular entry indices +for i in {1..2} +do + TEST_IN_LOOP stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID/f$i +done + +# Now disable granular-entry-heal +TEST $CLI volume set $V0 granular-entry-heal off + +# Start the brick that was down +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 + +# Enable shd +TEST gluster volume set $V0 cluster.self-heal-daemon on +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 + +# Now the indices created are granular but the heal is going to be of the +# normal kind. We test to make sure that heal still completes fine and that +# the stale granular indices are going to be deleted + +TEST $CLI volume heal $V0 + +# Wait for heal to complete +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +# Test if data was healed +for i in {1..2} +do + TEST_IN_LOOP diff $B0/${V0}0/f$i $B0/${V0}1/f$i +done + +# Now verify that there are no name indices left after self-heal +TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID/f1 +TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID/f2 +TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID + +cleanup -- cgit From e06f4590616dfe7b93e8ac147ed812756df4f22d Mon Sep 17 00:00:00 2001 From: Krutika Dhananjay Date: Thu, 22 Sep 2016 16:48:54 +0530 Subject: cluster/afr: CLI for granular entry heal enablement/disablement Backport of: http://review.gluster.org/15747 When there are already existing non-granular indices created that are yet to be healed, if granular-entry-heal option is toggled from 'off' to 'on', AFR self-heal whenever it kicks in, will try to look for granular indices in 'entry-changes'. Because of the absence of name indices, granular entry healing logic will fail to heal these directories, and worse yet unset pending extended attributes with the assumption that are no entries that need heal. To get around this, a new CLI is introduced which will invoke glfsheal program to figure whether at the time an attempt is made to enable granular entry heal, there are pending heals on the volume OR there are one or more bricks that are down. If either of them is true, the command will be failed with the appropriate error. New CLI: gluster volume heal granular-entry-heal {enable,disable} Change-Id: I342e0390f847fcb015a50ef58aedfcbcb58f4ed3 BUG: 1398501 Signed-off-by: Krutika Dhananjay Reviewed-on: http://review.gluster.org/15942 NetBSD-regression: NetBSD Build System Smoke: Gluster Build System CentOS-regression: Gluster Build System Reviewed-by: Pranith Kumar Karampuri --- tests/basic/afr/granular-esh/add-brick.t | 2 +- tests/basic/afr/granular-esh/cli.t | 142 +++++++++++++++++++++ tests/basic/afr/granular-esh/conservative-merge.t | 4 +- tests/basic/afr/granular-esh/granular-esh.t | 2 +- .../granular-indices-but-non-granular-heal.t | 4 +- tests/basic/afr/granular-esh/replace-brick.t | 2 +- 6 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 tests/basic/afr/granular-esh/cli.t (limited to 'tests/basic') diff --git a/tests/basic/afr/granular-esh/add-brick.t b/tests/basic/afr/granular-esh/add-brick.t index f3125d7fe7d..270cf1d32a6 100644 --- a/tests/basic/afr/granular-esh/add-brick.t +++ b/tests/basic/afr/granular-esh/add-brick.t @@ -14,7 +14,7 @@ TEST $CLI volume set $V0 cluster.data-self-heal off TEST $CLI volume set $V0 cluster.metadata-self-heal off TEST $CLI volume set $V0 cluster.entry-self-heal off TEST $CLI volume set $V0 self-heal-daemon off -TEST $CLI volume set $V0 granular-entry-heal on +TEST $CLI volume heal $V0 granular-entry-heal enable TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0 diff --git a/tests/basic/afr/granular-esh/cli.t b/tests/basic/afr/granular-esh/cli.t new file mode 100644 index 00000000000..a655180a095 --- /dev/null +++ b/tests/basic/afr/granular-esh/cli.t @@ -0,0 +1,142 @@ +#!/bin/bash + +. $(dirname $0)/../../../include.rc +. $(dirname $0)/../../../volume.rc +. $(dirname $0)/../../../afr.rc + +cleanup + +TESTS_EXPECTED_IN_LOOP=4 + +TEST glusterd +TEST pidof glusterd + +TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1} +# Test that enabling the option should work on a newly created volume +TEST $CLI volume set $V0 cluster.granular-entry-heal on +TEST $CLI volume set $V0 cluster.granular-entry-heal off + +######################### +##### DISPERSE TEST ##### +######################### +# Execute the same command on a disperse volume and make sure it fails. +TEST $CLI volume create $V1 disperse 3 redundancy 1 $H0:$B0/${V1}{0,1,2} +TEST $CLI volume start $V1 +TEST ! $CLI volume heal $V1 granular-entry-heal enable +TEST ! $CLI volume heal $V1 granular-entry-heal disable + +####################### +###### TIER TEST ###### +####################### +# Execute the same command on a disperse + replicate tiered volume and make +# sure the option is set on the replicate leg of the volume +TEST $CLI volume attach-tier $V1 replica 2 $H0:$B0/${V1}{3,4} +TEST $CLI volume heal $V1 granular-entry-heal enable +EXPECT "enable" volume_get_field $V1 cluster.granular-entry-heal +TEST $CLI volume heal $V1 granular-entry-heal disable +EXPECT "disable" volume_get_field $V1 cluster.granular-entry-heal + +# Kill a disperse brick and make heal be pending on the volume. +TEST kill_brick $V1 $H0 $B0/${V1}0 + +# Now make sure that one offline brick in disperse does not affect enabling the +# option on the volume. +TEST $CLI volume heal $V1 granular-entry-heal enable +EXPECT "enable" volume_get_field $V1 cluster.granular-entry-heal +TEST $CLI volume heal $V1 granular-entry-heal disable +EXPECT "disable" volume_get_field $V1 cluster.granular-entry-heal + +# Now kill a replicate brick. +TEST kill_brick $V1 $H0 $B0/${V1}3 +# Now make sure that one offline brick in replicate causes the command to be +# failed. +TEST ! $CLI volume heal $V1 granular-entry-heal enable +EXPECT "disable" volume_get_field $V1 cluster.granular-entry-heal + +###################### +### REPLICATE TEST ### +###################### +TEST $CLI volume start $V0 +TEST $CLI volume set $V0 cluster.data-self-heal off +TEST $CLI volume set $V0 cluster.metadata-self-heal off +TEST $CLI volume set $V0 cluster.entry-self-heal off +TEST $CLI volume set $V0 self-heal-daemon off +# Test that the volume-set way of enabling the option is disallowed +TEST ! $CLI volume set $V0 granular-entry-heal on +# Test that the volume-heal way of enabling the option is allowed +TEST $CLI volume heal $V0 granular-entry-heal enable +# Volume-reset of the option should be allowed +TEST $CLI volume reset $V0 granular-entry-heal +TEST $CLI volume heal $V0 granular-entry-heal enable + +EXPECT "enable" volume_option $V0 cluster.granular-entry-heal + +TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0 + +# Kill brick-0. +TEST kill_brick $V0 $H0 $B0/${V0}0 + +# Disabling the option should work even when one or more bricks are down +TEST $CLI volume heal $V0 granular-entry-heal disable +# When a brick is down, 'enable' attempt should be failed +TEST ! $CLI volume heal $V0 granular-entry-heal enable + +# Restart the killed brick +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 + +# When all bricks are up, it should be possible to enable the option +TEST $CLI volume heal $V0 granular-entry-heal enable + +# Kill brick-0 again +TEST kill_brick $V0 $H0 $B0/${V0}0 + +# Create files under root +for i in {1..2} +do + echo $i > $M0/f$i +done + +# Test that the index associated with '/' is created on B1. +TEST stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID + +# Check for successful creation of granular entry indices +for i in {1..2} +do + TEST_IN_LOOP stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID/f$i +done + +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 + +TEST gluster volume set $V0 cluster.self-heal-daemon on +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 + +TEST $CLI volume heal $V0 + +# Wait for heal to complete +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +# Test if data was healed +for i in {1..2} +do + TEST_IN_LOOP diff $B0/${V0}0/f$i $B0/${V0}1/f$i +done + +# Now verify that there are no name indices left after self-heal +TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID/f1 +TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID/f2 +TEST ! stat $B0/${V0}1/.glusterfs/indices/entry-changes/$ROOT_GFID + +# Perform a volume-reset-all-options operation +TEST $CLI volume reset $V0 +# Ensure that granular entry heal is also disabled +EXPECT "no" volume_get_field $V0 cluster.granular-entry-heal +EXPECT "on" volume_get_field $V0 cluster.entry-self-heal + +cleanup +#G_TESTDEF_TEST_STATUS_NETBSD7=BAD_TEST,BUG=1399038 diff --git a/tests/basic/afr/granular-esh/conservative-merge.t b/tests/basic/afr/granular-esh/conservative-merge.t index b566a0ea4d3..b170e47e0cb 100644 --- a/tests/basic/afr/granular-esh/conservative-merge.t +++ b/tests/basic/afr/granular-esh/conservative-merge.t @@ -11,13 +11,13 @@ TESTS_EXPECTED_IN_LOOP=4 TEST glusterd TEST pidof glusterd TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1} +TEST $CLI volume start $V0 TEST $CLI volume set $V0 self-heal-daemon off TEST $CLI volume set $V0 data-self-heal off TEST $CLI volume set $V0 metadata-self-heal off TEST $CLI volume set $V0 entry-self-heal off -TEST $CLI volume set $V0 granular-entry-heal on +TEST $CLI volume heal $V0 granular-entry-heal enable -TEST $CLI volume start $V0 TEST $GFS --volfile-id=$V0 -s $H0 $M0 TEST mkdir $M0/dir diff --git a/tests/basic/afr/granular-esh/granular-esh.t b/tests/basic/afr/granular-esh/granular-esh.t index ee53878e004..de0e8f4290b 100644 --- a/tests/basic/afr/granular-esh/granular-esh.t +++ b/tests/basic/afr/granular-esh/granular-esh.t @@ -16,7 +16,7 @@ TEST $CLI volume set $V0 cluster.data-self-heal off TEST $CLI volume set $V0 cluster.metadata-self-heal off TEST $CLI volume set $V0 cluster.entry-self-heal off TEST $CLI volume set $V0 self-heal-daemon off -TEST $CLI volume set $V0 granular-entry-heal on +TEST $CLI volume heal $V0 granular-entry-heal enable TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0 diff --git a/tests/basic/afr/granular-esh/granular-indices-but-non-granular-heal.t b/tests/basic/afr/granular-esh/granular-indices-but-non-granular-heal.t index 2da90a98c76..1b5421bf4b6 100644 --- a/tests/basic/afr/granular-esh/granular-indices-but-non-granular-heal.t +++ b/tests/basic/afr/granular-esh/granular-indices-but-non-granular-heal.t @@ -17,7 +17,7 @@ TEST $CLI volume set $V0 cluster.data-self-heal off TEST $CLI volume set $V0 cluster.metadata-self-heal off TEST $CLI volume set $V0 cluster.entry-self-heal off TEST $CLI volume set $V0 self-heal-daemon off -TEST $CLI volume set $V0 granular-entry-heal on +TEST $CLI volume heal $V0 granular-entry-heal enable TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0 @@ -40,7 +40,7 @@ do done # Now disable granular-entry-heal -TEST $CLI volume set $V0 granular-entry-heal off +TEST $CLI volume heal $V0 granular-entry-heal disable # Start the brick that was down TEST $CLI volume start $V0 force diff --git a/tests/basic/afr/granular-esh/replace-brick.t b/tests/basic/afr/granular-esh/replace-brick.t index aaa54da2a2c..639ed81b95c 100644 --- a/tests/basic/afr/granular-esh/replace-brick.t +++ b/tests/basic/afr/granular-esh/replace-brick.t @@ -12,7 +12,7 @@ TEST $CLI volume set $V0 cluster.data-self-heal off TEST $CLI volume set $V0 cluster.metadata-self-heal off TEST $CLI volume set $V0 cluster.entry-self-heal off TEST $CLI volume set $V0 self-heal-daemon off -TEST $CLI volume set $V0 granular-entry-heal on +TEST $CLI volume heal $V0 granular-entry-heal enable TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0; -- cgit From 67feb849ef96431ad869ce904c5de727195758cf Mon Sep 17 00:00:00 2001 From: Ravishankar N Date: Wed, 14 Dec 2016 22:48:20 +0530 Subject: glfsheal: Explicitly enable self-heal xlator options Enable data, metadata and entry self-heal as xlator-options so that glfs-heal.c can heal split-brain files even if they are disabled on the volume via volume set commands. > Reviewed-on: http://review.gluster.org/11333 > Smoke: Gluster Build System > Reviewed-by: Pranith Kumar Karampuri > Tested-by: Pranith Kumar Karampuri > NetBSD-regression: NetBSD Build System > CentOS-regression: Gluster Build System (cherry picked from commit 209c2d447be874047cb98d86492b03fa807d1832) Change-Id: Ic191a1017131db1ded94d97c932079d7bfd79457 BUG: 1405130 Signed-off-by: Ravishankar N Reviewed-on: http://review.gluster.org/16144 Reviewed-by: Pranith Kumar Karampuri Smoke: Gluster Build System NetBSD-regression: NetBSD Build System CentOS-regression: Gluster Build System --- tests/basic/afr/split-brain-healing.t | 3 +++ 1 file changed, 3 insertions(+) (limited to 'tests/basic') diff --git a/tests/basic/afr/split-brain-healing.t b/tests/basic/afr/split-brain-healing.t index 302a3e6144b..c66bb5d44df 100644 --- a/tests/basic/afr/split-brain-healing.t +++ b/tests/basic/afr/split-brain-healing.t @@ -31,6 +31,9 @@ TEST glusterd TEST pidof glusterd TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4} TEST $CLI volume set $V0 cluster.self-heal-daemon off +TEST $CLI volume set $V0 cluster.data-self-heal off +TEST $CLI volume set $V0 cluster.metadata-self-heal off +TEST $CLI volume set $V0 cluster.entry-self-heal off TEST $CLI volume start $V0 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 -- cgit From cb8bc3396d16e777d9a2683886fefd43e747e8a3 Mon Sep 17 00:00:00 2001 From: Ravishankar N Date: Thu, 29 Dec 2016 12:10:00 +0000 Subject: tests: Fix split-brain-favorite-child-policy.t failures Problem: In CentOS-7, the file was receving an extra removexattr(security.ima) FOP which changed its ctime, breaking the assumption that a particular brick had the latest ctime based on the writevs done in the .t Fix: 1. Compare the ctime of both files in the backend and pick the one with the latest ctime for the fav-child policy. Also unmount the volume before comparing, to avoid any further FOPS on the file that can possibly modify the timestamps. 2. Added floating point handling in stat function. Thanks to Pranith for the helping debugging the regex. > Reviewed-on: http://review.gluster.org/16288 > Smoke: Gluster Build System > NetBSD-regression: NetBSD Build System > CentOS-regression: Gluster Build System > Reviewed-by: Pranith Kumar Karampuri (cherry picked from commit 76fff8cb2a164b596ca67e65c99623f5b68361fd) Change-Id: I06041a0f39a29d2593b867af8685d65c7cd99150 BUG: 1410073 Signed-off-by: Ravishankar N Reviewed-on: http://review.gluster.org/16324 Smoke: Gluster Build System NetBSD-regression: NetBSD Build System CentOS-regression: Gluster Build System Reviewed-by: Pranith Kumar Karampuri --- tests/basic/afr/split-brain-favorite-child-policy.t | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'tests/basic') diff --git a/tests/basic/afr/split-brain-favorite-child-policy.t b/tests/basic/afr/split-brain-favorite-child-policy.t index 7a14852685c..3df8e718bf0 100644 --- a/tests/basic/afr/split-brain-favorite-child-policy.t +++ b/tests/basic/afr/split-brain-favorite-child-policy.t @@ -42,8 +42,15 @@ TEST $CLI volume heal $V0 cat $M0/file > /dev/null EXPECT "1" echo $? -#We know that the first brick has latest ctime. -LATEST_CTIME_MD5=$(md5sum $B0/${V0}0/file | cut -d\ -f1) +# Umount to prevent further FOPS on the file, then find the brick with latest ctime. +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +ctime1=`stat -c "%.Z" $B0/${V0}0/file` +ctime2=`stat -c "%.Z" $B0/${V0}1/file` +if (( $(echo "$ctime1 > $ctime2" | bc -l) )); then + LATEST_CTIME_MD5=$(md5sum $B0/${V0}0/file | cut -d\ -f1) +else + LATEST_CTIME_MD5=$(md5sum $B0/${V0}1/file | cut -d\ -f1) +fi TEST $CLI volume set $V0 cluster.favorite-child-policy ctime TEST $CLI volume start $V0 force EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status @@ -51,10 +58,13 @@ EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST $CLI volume heal $V0 EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 +B0_MD5=$(md5sum $B0/${V0}0/file | cut -d\ -f1) +B1_MD5=$(md5sum $B0/${V0}1/file | cut -d\ -f1) +TEST [ "$LATEST_CTIME_MD5" == "$B0_MD5" ] +TEST [ "$LATEST_CTIME_MD5" == "$B1_MD5" ] +TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 cat $M0/file > /dev/null EXPECT "0" echo $? -HEALED_MD5=$(md5sum $B0/${V0}1/file | cut -d\ -f1) -TEST [ "$LATEST_CTIME_MD5" == "$HEALED_MD5" ] ############ Healing using favorite-child-policy = mtime ################# TEST $CLI volume set $V0 cluster.favorite-child-policy none -- cgit