summaryrefslogtreecommitdiffstats
path: root/tests/basic/ec/self-heal.t
blob: d217559db1ab695f1f0cda709c30667085741623 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
#!/bin/bash

SCRIPT_TIMEOUT=300

. $(dirname $0)/../../include.rc
. $(dirname $0)/../../volume.rc

# This test checks self-healing feature of dispersed volumes

cleanup

function check_mount_dir
{
    getfattr -d -m. -e hex $M0 2>&1 > /dev/null
    for i in {1..20}; do
        ls -l $M0/ | grep "dir1"
        if [ $? -ne 0 ]; then
            return 1
        fi
    done

    return 0
}

function check_size
{
    cat $M0/$1 2>&1 > /dev/null
    for i in "${brick[@]}"; do
        res=`stat -c "%s" $i/$1`
        if [ "$res" != "$2" ]; then
            echo "N"
            return 0
        fi
    done
    echo "Y"
    return 0
}

function check_mode
{
    getfattr -d -m. -e hex $M0/$1 2>&1 > /dev/null
    for i in "${brick[@]}"; do
        res=`stat -c "%A" $i/$1`
        if [ "$res" != "$2" ]; then
            echo "N"
            return 0
        fi
    done
    echo "Y"
    return 0
}

function check_date
{
    getfattr -d -m. -e hex $M0/$1 2>&1 > /dev/null
    for i in "${brick[@]}"; do
        res=`stat -c "%Y" $i/$1`
        if [ "$res" != "$2" ]; then
            echo "N"
            return 0
        fi
    done
    echo "Y"
    return 0
}

function check_xattr
{
    getfattr -d -m. -e hex $M0/$1 2>&1 > /dev/null
    for i in "${brick[@]}"; do
        getfattr -n $2 $i/$1 2>/dev/null
        if [ $? -eq 0 ]; then
            echo "N"
            return 0
        fi
    done
    echo "Y"
    return 0
}

function check_dir
{
    getfattr -m. -d $M0/dir1 2>&1 > /dev/null
    for i in "${brick[@]}"; do
        if [ ! -d $i/dir1 ]; then
            echo "N"
            return 0
        fi
    done
    echo "Y"
    return 0
}

function check_soft_link
{
    getfattr -d -m. -e hex $M0/test3 2>&1 > /dev/null
    for i in "${brick[@]}"; do
        if [ ! -h $i/test3 ]; then
            echo "N"
            return 0
        fi
    done
    echo "Y"
    return 0
}

function check_hard_link
{
    getfattr -d -m. -e hex $M0/test4 2>&1 > /dev/null
    for i in "${brick[@]}"; do
        res=`stat -c "%h" $i/test4`
        if [ "$res" != "3" ]; then
            echo "N"
            return 0
        fi
    done
    echo "Y"
    return 0
}

tmp=`mktemp -d -t ${0##*/}.XXXXXX`
if [ ! -d $tmp ]; then
    exit 1
fi

TESTS_EXPECTED_IN_LOOP=194

TEST glusterd
TEST pidof glusterd
TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5}
TEST $CLI volume set $V0 client-log-level DEBUG
#Write-behind has a bug where lookup can race over write which leads to size mismatch on the mount after a 'cp'
TEST $CLI volume set $V0 performance.write-behind off
EXPECT "Created" volinfo_field $V0 'Status'
TEST $CLI volume start $V0
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status'
#direct-io-mode is to make sure 'cat' leads to READ fop which triggers heal
TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 --direct-io-mode=yes $M0;
# Wait until all 6 childs have been recognized by the ec xlator
EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0

TEST dd if=/dev/urandom of=$tmp/test bs=1024 count=1024

cs=$(sha1sum $tmp/test | awk '{ print $1 }')

TEST df -h $M0
TEST stat $M0

for idx in {0..5}; do
    brick[$idx]=$(gf_get_gfid_backend_file_path $B0/$V0$idx)
done

TEST cp $tmp/test $M0/test
TEST chmod 644 $M0/test
TEST touch -d "@946681200" $M0/test
EXPECT "-rw-r--r--" stat -c "%A" $M0/test
EXPECT "946681200" stat -c "%Y" $M0/test

for idx1 in {0..5}; do
    TEST chmod 666 ${brick[$idx1]}/test
    TEST truncate -s 0 ${brick[$idx1]}/test
    TEST setfattr -n user.test -v "test1" ${brick[$idx1]}/test
    EXPECT "-rw-r--r--" stat -c "%A" $M0/test
    EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_size test "262144"
    EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_mode test "-rw-r--r--"
    EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_date test "946681200"
    EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_xattr test "user.test"
done

for idx1 in {0..4}; do
    for idx2 in `seq $(($idx1 + 1)) 5`; do
        if [ $idx1 -ne $idx2 ]; then
            TEST chmod 666 ${brick[$idx1]}/test
            TEST chmod 600 ${brick[$idx2]}/test
            TEST truncate -s 0 ${brick[$idx1]}/test
            TEST truncate -s 2097152 ${brick[$idx2]}/test
            TEST setfattr -n user.test -v "test1" ${brick[$idx1]}/test
            TEST setfattr -n user.test -v "test2" ${brick[$idx2]}/test
            EXPECT "-rw-r--r--" stat -c "%A" $M0/test
            EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_size test "262144"
            EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_mode test "-rw-r--r--"
            EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_date test "946681200"
            EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_xattr test "user.test"
        fi
    done
done

sleep 2

TEST kill_brick $V0 $H0 $B0/${V0}0
TEST kill_brick $V0 $H0 $B0/${V0}1
TEST cp $tmp/test $M0/test2
EXPECT "1048576" stat -c "%s" $M0/test2
TEST chmod 777 $M0/test2
EXPECT "-rwxrwxrwx" stat -c "%A" $M0/test2

TEST mkdir $M0/dir1
TEST ls -al $M0/dir1

TEST ln -s test2 $M0/test3
TEST [ -h $M0/test3 ]

TEST ln $M0/test2 $M0/test4
TEST [ -f $M0/test4 ]
EXPECT "2" stat -c "%h" $M0/test2
EXPECT "2" stat -c "%h" $M0/test4

sleep 2

TEST $CLI volume start $V0 force
# Wait until the killed bricks have been started and recognized by the ec
# xlator
EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0

TEST check_mount_dir

EXPECT "1048576" stat -c "%s" $M0/test2
EXPECT "-rwxrwxrwx" stat -c "%A" $M0/test2
EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_size test2 "262144"
EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_mode test2 "-rwxrwxrwx"

TEST ls -al $M0/dir1
EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_dir

TEST [ -h $M0/test3 ]
EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_soft_link

EXPECT "2" stat -c "%h" $M0/test4
EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_hard_link

TEST rm -rf $tmp

cleanup