blob: a2c489a4063447f5f644953000ea676732dde3be (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
#!/bin/bash
. $(dirname $0)/../include.rc
. $(dirname $0)/../volume.rc
cleanup;
TEST glusterd
TEST pidof glusterd
TEST $CLI volume info 2> /dev/null;
# Setup a cluster with 3 replicas, and fav child by majority on
TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{1..3};
TEST $CLI volume set $V0 cluster.choose-local off
TEST $CLI volume set $V0 cluster.self-heal-daemon off
TEST $CLI volume set $V0 cluster.data-self-heal on
TEST $CLI volume set $V0 cluster.metadata-self-heal on
TEST $CLI volume set $V0 cluster.entry-self-heal on
# This would normally be a toxic combination because it allows us to create a
# split brain by writing to 1/3 replicas ... but for testing that's exactly
# what we want.
TEST $CLI volume set $V0 cluster.quorum-type fixed
TEST $CLI volume set $V0 cluster.quorum-count 1
TEST $CLI volume start $V0
sleep 5
# Mount the volume
TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 \
--attribute-timeout=0 --entry-timeout=0
# Write some random data into a file
dd if=/dev/urandom of=$M0/splitfile bs=128k count=5 2>/dev/null
# Create a split-brain by downing a brick, writing some data
# then downing the other two, write some more data and bring
# everything back up.
TEST kill_brick $V0 $H0 $B0/${V0}1
TEST dd if=/dev/urandom of=$M0/splitfile bs=128k count=5 oflag=append 2>/dev/null
MAJORITY_MD5=$(md5sum $M0/splitfile | cut -d\ -f1)
# Bring bricks back up
TEST $CLI volume start $V0 force
TEST kill_brick $V0 $H0 $B0/${V0}2
TEST kill_brick $V0 $H0 $B0/${V0}3
EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
TEST dd if=/dev/urandom of=$M0/splitfile bs=128k count=5 oflag=append 2>/dev/null
# Bring all bricks up
TEST $CLI volume start $V0 force
EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
EXPECT_WITHIN 20 "1" afr_child_up_status $V0 2
# First do a test to prove the file is splitbrained without
# favorite-child support.
umount $M0
# Mount the volume
TEST glusterfs --log-level DEBUG --volfile-id=/$V0 --volfile-server=$H0 $M0 \
--attribute-timeout=0 --entry-timeout=0
sleep 1
#EST ! md5sum $M0/splitfile
# Ok now turn the favorite-child option and we should be able to read it.
# Compare MD5's, the healed file should be that of the file which is
# on 2/3 bricks.
umount $M0
TEST $CLI volume set $V0 cluster.favorite-child-by-majority on
sleep 1
# Mount the volume
TEST glusterfs --log-level DEBUG --volfile-id=/$V0 --volfile-server=$H0 $M0 \
--attribute-timeout=0 --entry-timeout=0
sleep 2
HEALED_MD5=$(md5sum $M0/splitfile | cut -d\ -f1)
TEST [ "$MAJORITY_MD5" == "$HEALED_MD5" ]
cleanup
TEST glusterd
TEST pidof glusterd
TEST $CLI volume info 2> /dev/null;
# Setup a cluster with 3 replicas, and fav child by majority on
TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{1..3};
TEST $CLI volume set $V0 cluster.choose-local off
TEST $CLI volume set $V0 cluster.self-heal-daemon off
TEST $CLI volume set $V0 cluster.data-self-heal off
TEST $CLI volume set $V0 cluster.metadata-self-heal off
TEST $CLI volume set $V0 cluster.entry-self-heal off
TEST $CLI volume set $V0 cluster.quorum-type fixed
TEST $CLI volume set $V0 cluster.quorum-count 1
TEST $CLI volume start $V0
sleep 5
# Mount the volume
TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 \
--attribute-timeout=0 --entry-timeout=0
mkdir $M0/d
# Write some random data into a file
dd if=/dev/urandom of=$M0/d/splitfile bs=128k count=5 2>/dev/null
# Create a split-brain by downing a brick, writing some data
# then downing the other two, write some more data and bring
# everything back up.
TEST kill_brick $V0 $H0 $B0/${V0}1
TEST dd if=/dev/urandom of=$M0/d/splitfile bs=128k count=5 oflag=append 2>/dev/null
MAJORITY_MD5=$(md5sum $M0/d/splitfile | cut -d\ -f1)
# Bring bricks back up
TEST $CLI volume start $V0 force
TEST kill_brick $V0 $H0 $B0/${V0}2
TEST kill_brick $V0 $H0 $B0/${V0}3
EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
TEST dd if=/dev/urandom of=$M0/d/splitfile bs=128k count=5 oflag=append 2>/dev/null
# Bring all bricks up
TEST $CLI volume start $V0 force
EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
EXPECT_WITHIN 20 "1" afr_child_up_status $V0 2
# First do a test to prove the file is splitbrained without
# favorite-child support.
umount $M0
# Mount the volume
TEST glusterfs --log-level DEBUG --volfile-id=/$V0 --volfile-server=$H0 $M0 \
--attribute-timeout=0 --entry-timeout=0
sleep 1
#EST ! md5sum $M0/d/splitfile
# Ok now turn the favorite-child option and we should be able to read it.
# Compare MD5's, the healed file should be that of the file which is
# on 2/3 bricks.
umount $M0
TEST $CLI volume set $V0 cluster.favorite-child-by-majority on
TEST $CLI volume set $V0 cluster.self-heal-daemon on
sleep 1
/etc/init.d/glusterd restart_shd
EXPECT_WITHIN 60 "0" get_pending_heal_count $V0
sleep 1
# Mount the volume
TEST glusterfs --log-level DEBUG --volfile-id=/$V0 --volfile-server=$H0 $M0 \
--attribute-timeout=0 --entry-timeout=0
sleep 2
HEALED_MD5=$(md5sum $M0/d/splitfile | cut -d\ -f1)
TEST [ "$MAJORITY_MD5" == "$HEALED_MD5" ]
cleanup
|
