blob: 390579c02880587047cce072beb95ab30079f87f (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
|
#!/bin/bash
#Test the split-brain resolution CLI commands.
. $(dirname $0)/../../include.rc
. $(dirname $0)/../../volume.rc
function get_replicate_subvol_number {
local filename=$1
#get_backend_paths
if [ -f $B0/${V0}1/$filename ]
then
echo 0
elif [ -f $B0/${V0}3/$filename ]
then echo 1
else
echo -1
fi
}
cleanup;
AREQUAL_PATH=$(dirname $0)/../../utils
CFLAGS=""
test "`uname -s`" != "Linux" && {
CFLAGS="$CFLAGS -I$(dirname $0)/../../../contrib/argp-standalone ";
CFLAGS="$CFLAGS -L$(dirname $0)/../../../contrib/argp-standalone -largp ";
CFLAGS="$CFLAGS -lintl";
}
build_tester $AREQUAL_PATH/arequal-checksum.c $CFLAGS
TEST glusterd
TEST pidof glusterd
TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4}
TEST $CLI volume set $V0 cluster.self-heal-daemon off
TEST $CLI volume start $V0
TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
cd $M0
for i in {1..10}
do
echo "Initial content">>file$i
done
replica_0_files_list=(`ls $B0/${V0}1|grep -v '^\.'`)
replica_1_files_list=(`ls $B0/${V0}3|grep -v '^\.'`)
############ Create data split-brain in the files. ###########################
TEST kill_brick $V0 $H0 $B0/${V0}1
for file in ${!replica_0_files_list[*]}
do
echo "B1 is down">>${replica_0_files_list[$file]}
done
TEST kill_brick $V0 $H0 $B0/${V0}3
for file in ${!replica_1_files_list[*]}
do
echo "B3 is down">>${replica_1_files_list[$file]}
done
SMALLER_FILE_SIZE=$(stat -c %s file1)
TEST $CLI volume start $V0 force
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2
TEST kill_brick $V0 $H0 $B0/${V0}2
for file in ${!replica_0_files_list[*]}
do
echo "B2 is down">>${replica_0_files_list[$file]}
echo "appending more content to make it the bigger file">>${replica_0_files_list[$file]}
done
TEST kill_brick $V0 $H0 $B0/${V0}4
for file in ${!replica_1_files_list[*]}
do
echo "B4 is down">>${replica_1_files_list[$file]}
echo "appending more content to make it the bigger file">>${replica_1_files_list[$file]}
done
BIGGER_FILE_SIZE=$(stat -c %s file1)
TEST $CLI volume start $V0 force
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 3
############### Acessing the files should now give EIO. ###############################
TEST ! cat file1
TEST ! cat file2
TEST ! cat file3
TEST ! cat file4
TEST ! cat file5
TEST ! cat file6
TEST ! cat file7
TEST ! cat file8
TEST ! cat file9
TEST ! cat file10
###################
TEST $CLI volume set $V0 cluster.self-heal-daemon on
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status
EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0
EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1
EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 2
EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 3
################ Heal file1 using the bigger-file option ##############
$CLI volume heal $V0 split-brain bigger-file /file1
EXPECT "0" echo $?
EXPECT $BIGGER_FILE_SIZE stat -c %s file1
################ Heal file2 using the bigger-file option and its gfid ##############
subvolume=$(get_replicate_subvol_number file2)
if [ $subvolume == 0 ]
then
GFID=$(gf_get_gfid_xattr $B0/${V0}1/file2)
elif [ $subvolume == 1 ]
then
GFID=$(gf_get_gfid_xattr $B0/${V0}3/file2)
fi
GFIDSTR="gfid:$(gf_gfid_xattr_to_str $GFID)"
$CLI volume heal $V0 split-brain bigger-file $GFIDSTR
EXPECT "0" echo $?
################ Heal file3 using the source-brick option ##############
################ Use the brick having smaller file size as source #######
subvolume=$(get_replicate_subvol_number file3)
if [ $subvolume == 0 ]
then
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}2 /file3
elif [ $subvolume == 1]
then
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}4 /file3
fi
EXPECT "0" echo $?
EXPECT $SMALLER_FILE_SIZE stat -c %s file3
################ Heal file4 using the source-brick option and it's gfid ##############
################ Use the brick having smaller file size as source #######
subvolume=$(get_replicate_subvol_number file4)
if [ $subvolume == 0 ]
then
GFID=$(gf_get_gfid_xattr $B0/${V0}1/file4)
GFIDSTR="gfid:$(gf_gfid_xattr_to_str $GFID)"
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}2 $GFIDSTR
elif [ $subvolume == 1]
then
GFID=$(gf_get_gfid_xattr $B0/${V0}3/file4)
GFIDSTR="gfid:$(gf_gfid_xattr_to_str $GFID)"
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}4 $GFIDSTR
fi
EXPECT "0" echo $?
EXPECT $SMALLER_FILE_SIZE stat -c %s file4
################ Heal file5 using the latest-mtime option ##############
subvolume=$(get_replicate_subvol_number file5)
if [ $subvolume == 0 ]
then
mtime1=$(stat -c %Y $B0/${V0}1/file5)
mtime2=$(stat -c %Y $B0/${V0}2/file5)
LATEST_MTIME=$(($mtime1 > $mtime2 ? $mtime1:$mtime2))
elif [ $subvolume == 1 ]
then
mtime1=$(stat -c %Y $B0/${V0}3/file5)
mtime2=$(stat -c %Y $B0/${V0}4/file5)
LATEST_MTIME=$(($mtime1 > $mtime2 ? $mtime1:$mtime2))
fi
$CLI volume heal $V0 split-brain latest-mtime /file5
EXPECT "0" echo $?
#TODO: Uncomment the below after posix_do_utimes() supports utimensat(2) accuracy
#TEST [ $LATEST_MTIME -eq $mtime1 ]
#TEST [ $LATEST_MTIME -eq $mtime2 ]
################ Heal file6 using the latest-mtime option and its gfid ##############
subvolume=$(get_replicate_subvol_number file6)
if [ $subvolume == 0 ]
then
GFID=$(gf_get_gfid_xattr $B0/${V0}1/file6)
mtime1=$(stat -c %Y $B0/${V0}1/file5)
mtime2=$(stat -c %Y $B0/${V0}2/file5)
LATEST_MTIME=$(($mtime1 > $mtime2 ? $mtime1:$mtime2))
elif [ $subvolume == 1 ]
then
GFID=$(gf_get_gfid_xattr $B0/${V0}3/file6)
mtime1=$(stat -c %Y $B0/${V0}3/file5)
mtime2=$(stat -c %Y $B0/${V0}4/file5)
LATEST_MTIME=$(($mtime1 > $mtime2 ? $mtime1:$mtime2))
fi
GFIDSTR="gfid:$(gf_gfid_xattr_to_str $GFID)"
$CLI volume heal $V0 split-brain latest-mtime $GFIDSTR
EXPECT "0" echo $?
#TODO: Uncomment the below after posix_do_utimes() supports utimensat(2) accuracy
#TEST [ $LATEST_MTIME -eq $mtime1 ]
#TEST [ $LATEST_MTIME -eq $mtime2 ]
################ Heal remaining SB'ed files of replica_0 using B1 as source ##############
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}1
EXPECT "0" echo $?
################ Heal remaining SB'ed files of replica_1 using B3 as source ##############
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}3
EXPECT "0" echo $?
############### Reading the files should now succeed. ###############################
TEST cat file1
TEST cat file2
TEST cat file3
TEST cat file4
TEST cat file5
TEST cat file6
TEST cat file7
TEST cat file8
TEST cat file9
TEST cat file10
################ File contents on the bricks must be same. ################################
TEST diff <(arequal-checksum -p $B0/$V01 -i .glusterfs) <(arequal-checksum -p $B0/$V02 -i .glusterfs)
TEST diff <(arequal-checksum -p $B0/$V03 -i .glusterfs) <(arequal-checksum -p $B0/$V04 -i .glusterfs)
############### Trying to heal files not in SB should fail. ###############################
$CLI volume heal $V0 split-brain bigger-file /file1
EXPECT "1" echo $?
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}4 /file3
EXPECT "1" echo $?
cd -
TEST rm $AREQUAL_PATH/arequal-checksum
cleanup
|
