From 243d61575c093c03b9beb014bf9d097646836e95 Mon Sep 17 00:00:00 2001 From: Jeff Darcy Date: Wed, 7 May 2014 19:31:30 +0000 Subject: dht: make lookup-unhashed=auto do something actually useful The key concept here is to determine whether a directory is "clean" by comparing its last-known-good topology to the current one for the volume. These are stored as "commit hashes" on the directory and the volume root respectively. The volume's commit hash changes whenever a brick is added or removed, and a fix-layout is done. A directory's commit hash changes only when a full rebalance (not just fix-layout) is done on it. If all bricks are present and have a directory commit hash that matches the volume commit hash, then we can assume that every file is in its "proper" place. Therefore, if we look for a file in that proper place and don't find it, we can assume it's not on any other subvolume and *safely* skip the global (broadcast to all) lookup. Change-Id: Id6ce4593ba1f7daffa74cfab591cb45960629ae3 BUG: 1220064 Reviewed-on-master: http://review.gluster.org/#/c/7702/ Signed-off-by: Jeff Darcy Signed-off-by: Shyam Reviewed-on: http://review.gluster.org/10729 Tested-by: Gluster Build System Reviewed-by: Krishnan Parthasarathi Reviewed-by: Vijay Bellur --- tests/bugs/distribute/bug-907072.t | 18 ++++--- tests/bugs/distribute/bug-921408.t | 4 +- tests/bugs/glusterd/bug-1070734.t | 7 ++- tests/features/unhashed-auto.t | 99 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 12 deletions(-) create mode 100755 tests/features/unhashed-auto.t (limited to 'tests') diff --git a/tests/bugs/distribute/bug-907072.t b/tests/bugs/distribute/bug-907072.t index 1e8bd280f32..a4d98831380 100755 --- a/tests/bugs/distribute/bug-907072.t +++ b/tests/bugs/distribute/bug-907072.t @@ -17,10 +17,11 @@ TEST glusterfs -s $H0 --volfile-id $V0 $M0; TEST mkdir $M0/test; -OLD_LAYOUT0=`get_layout $B0/${V0}0/test`; -OLD_LAYOUT1=`get_layout $B0/${V0}1/test`; -OLD_LAYOUT2=`get_layout $B0/${V0}2/test`; -OLD_LAYOUT3=`get_layout $B0/${V0}3/test`; +# Extract the layout sans the commit hash +OLD_LAYOUT0=`get_layout $B0/${V0}0/test | cut -c11-34`; +OLD_LAYOUT1=`get_layout $B0/${V0}1/test | cut -c11-34`; +OLD_LAYOUT2=`get_layout $B0/${V0}2/test | cut -c11-34`; +OLD_LAYOUT3=`get_layout $B0/${V0}3/test | cut -c11-34`; TEST killall glusterfsd; @@ -36,10 +37,11 @@ EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 TEST glusterfs -s $H0 --volfile-id $V0 $M0; TEST stat $M0/test; -NEW_LAYOUT0=`get_layout $B0/${V0}0/test`; -NEW_LAYOUT1=`get_layout $B0/${V0}1/test`; -NEW_LAYOUT2=`get_layout $B0/${V0}2/test`; -NEW_LAYOUT3=`get_layout $B0/${V0}3/test`; +# Extract the layout sans the commit hash +NEW_LAYOUT0=`get_layout $B0/${V0}0/test | cut -c11-34`; +NEW_LAYOUT1=`get_layout $B0/${V0}1/test | cut -c11-34`; +NEW_LAYOUT2=`get_layout $B0/${V0}2/test | cut -c11-34`; +NEW_LAYOUT3=`get_layout $B0/${V0}3/test | cut -c11-34`; EXPECT $OLD_LAYOUT0 echo $NEW_LAYOUT0; EXPECT $OLD_LAYOUT1 echo $NEW_LAYOUT1; diff --git a/tests/bugs/distribute/bug-921408.t b/tests/bugs/distribute/bug-921408.t index b1887f8ae22..559114bb85a 100755 --- a/tests/bugs/distribute/bug-921408.t +++ b/tests/bugs/distribute/bug-921408.t @@ -37,7 +37,7 @@ addbr_rebal_till_layout_change() then break fi - NEW_LAYOUT=`get_layout $B0/${V0}0` + NEW_LAYOUT=`get_layout $B0/${V0}0 | cut -c11-34` if [ $OLD_LAYOUT == $NEW_LAYOUT ] then i=`expr $i + 1`; @@ -64,7 +64,7 @@ TEST touch $M0/test/test fd=`fd_available` TEST fd_open $fd "rw" $M0/test/test -OLD_LAYOUT=`get_layout $B0/${V0}0` +OLD_LAYOUT=`get_layout $B0/${V0}0 | cut -c11-34` addbr_rebal_till_layout_change 1 diff --git a/tests/bugs/glusterd/bug-1070734.t b/tests/bugs/glusterd/bug-1070734.t index b5a53c24cab..5db60e0cfe6 100755 --- a/tests/bugs/glusterd/bug-1070734.t +++ b/tests/bugs/glusterd/bug-1070734.t @@ -65,8 +65,11 @@ TEST [ -f ${OTHERBRICK}/DIR/file ] #Check the DIR on HASHED should have got zeroed layout and the \ #OTHERBRICK should have got full layout -EXPECT "0x00000001000000000000000000000000" dht_get_layout $HASHED/DIR ; -EXPECT "0x000000010000000000000000ffffffff" dht_get_layout $OTHERBRICK/DIR; +shorter_layout () { + dht_get_layout $1 | cut -c 19- +} +EXPECT "0000000000000000" shorter_layout $HASHED/DIR ; +EXPECT "00000000ffffffff" shorter_layout $OTHERBRICK/DIR; ## Before killing daemon to avoid deadlocks EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" umount_nfs $N0 diff --git a/tests/features/unhashed-auto.t b/tests/features/unhashed-auto.t new file mode 100755 index 00000000000..97663c20e10 --- /dev/null +++ b/tests/features/unhashed-auto.t @@ -0,0 +1,99 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc +. $(dirname $0)/../dht.rc + +NFILES=100 + +touch_files () { + for i in $(seq 1 $NFILES); do + touch $(printf $M0/dir/file%02d $i) + done +} + +count_files () { + found=0 + for i in $(seq 1 $NFILES); do + if [ -f $(printf $M0/dir/file%02d $i) ]; then + found=$((found+1)) + fi + done + echo "found $found files" > /dev/tty + echo $found +} + +wait_for_rebalance () { + while true; do + tmp=$(rebalance_completed) + if [ $tmp -eq 1 ]; then + sleep 1 + else + break + fi + done +} + +get_xattr () { + cmd="getfattr --absolute-names --only-values -n trusted.glusterfs.dht" + $cmd $1 | od -tx1 -An | tr -d ' ' +} + +cleanup + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info + +TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2} +EXPECT "$V0" volinfo_field $V0 'Volume Name' +EXPECT 'Created' volinfo_field $V0 'Status' + +TEST $CLI volume set $V0 cluster.lookup-unhashed auto + +TEST $CLI volume start $V0 +EXPECT 'Started' volinfo_field $V0 'Status' + +# Create some files for later tests. +TEST $GFS -s $H0 --volfile-id $V0 $M0 +TEST mkdir $M0/dir +TEST touch_files +TEST umount $M0 + +# Add a brick and do the fix-layout part of rebalance to update directory layouts +# (including their directory commit hashes). +TEST $CLI volume add-brick $V0 $H0:$B0/${V0}3 +EXPECT '3' brick_count $V0 +TEST $CLI volume rebalance $V0 fix-layout start +TEST wait_for_rebalance + +# Now for the sneaky part. *Undo* the part of rebalance that updated the volume +# commit hash, forcing a false match between that and the directory commit hashes. +TEST setfattr -x trusted.glusterfs.dht.commithash $B0/${V0}1 +TEST setfattr -x trusted.glusterfs.dht.commithash $B0/${V0}2 +TEST setfattr -x trusted.glusterfs.dht.commithash $B0/${V0}3 + +# Mount and check that we do *not* see all of the files. This indicates that we +# correctly skipped the broadcast lookup that would have found them. +TEST $GFS -s $H0 --volfile-id $V0 $M0 +TEST [ $(count_files) -ne 100 ] +TEST umount $M0 + +# Do the fix-layout again to generate a new volume commit hash. +TEST $CLI volume rebalance $V0 fix-layout start +TEST wait_for_rebalance + +# Mount and check that we *do* see all of the files. This indicates that we saw +# the mismatch and did the broadcast lookup this time. +TEST $GFS -s $H0 --volfile-id $V0 $M0 +TEST [ $(count_files) -eq 100 ] +TEST umount $M0 + +# Do a *full* rebalance and verify that the directory commit hash changed. +old_val=$(get_xattr $B0/${V0}1/dir) +TEST $CLI volume rebalance $V0 start +TEST wait_for_rebalance +new_val=$(get_xattr $B0/${V0}1/dir) +TEST [ ! x"$old_val" = x"$new_val" ] + +cleanup -- cgit