cli/glusterd: Added support for dispersed volumes

Two new options have been added to the 'create' command of the cli
interface:

    disperse [<count>] redundancy <count>

Both are optional. A dispersed volume is created by specifying, at
least, one of them. If 'disperse' is missing or it's present but
'<count>' does not, the number of bricks enumerated in the command
line is taken as the disperse count.

If 'redundancy' is missing, the lowest optimal value is assumed. A
configuration is considered optimal (for most workloads) when the
disperse count - redundancy count is a power of 2. If the resulting
redundancy is 1, the volume is created normally, but if it's greater
than 1, a warning is shown to the user and he/she must answer yes/no
to continue volume creation. If there isn't any optimal value for
the given number of bricks, a warning is also shown and, if the user
accepts, a redundancy of 1 is used.

If 'redundancy' is specified and the resulting volume is not optimal,
another warning is shown to the user.

A distributed-disperse volume can be created using a number of bricks
multiple of the disperse count.

Change-Id: Iab93efbe78e905cdb91f54f3741599f7ea6645e4
BUG: 1118629
Signed-off-by: Xavier Hernandez <xhernandez@datalab.es>
Reviewed-on: http://review.gluster.org/7782
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

10 files changed, 597 insertions, 0 deletions

diff --git a/tests/basic/ec/ec-12-4.t b/tests/basic/ec/ec-12-4.t
new file mode 100644
index 00000000000..9ab47018617
--- /dev/null
+++ b/tests/basic/ec/ec-12-4.t
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+# This test checks basic dispersed volume functionality and cli interface
+
+DISPERSE=12
+REDUNDANCY=4
+
+# This must be equal to 44 * $DISPERSE + 106
+TESTS_EXPECTED_IN_LOOP=634
+
+. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-3-1.t b/tests/basic/ec/ec-3-1.t
new file mode 100644
index 00000000000..5769c202289
--- /dev/null
+++ b/tests/basic/ec/ec-3-1.t
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+# This test checks basic dispersed volume functionality and cli interface
+
+DISPERSE=3
+REDUNDANCY=1
+
+# This must be equal to 44 * $DISPERSE + 106
+TESTS_EXPECTED_IN_LOOP=238
+
+. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-4-1.t b/tests/basic/ec/ec-4-1.t
new file mode 100644
index 00000000000..d34e1fb4e95
--- /dev/null
+++ b/tests/basic/ec/ec-4-1.t
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+# This test checks basic dispersed volume functionality and cli interface
+
+DISPERSE=4
+REDUNDANCY=1
+
+# This must be equal to 44 * $DISPERSE + 106
+TESTS_EXPECTED_IN_LOOP=282
+
+. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-5-1.t b/tests/basic/ec/ec-5-1.t
new file mode 100644
index 00000000000..61d1cb6ce48
--- /dev/null
+++ b/tests/basic/ec/ec-5-1.t
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+# This test checks basic dispersed volume functionality and cli interface
+
+DISPERSE=5
+REDUNDANCY=1
+
+# This must be equal to 44 * $DISPERSE + 106
+TESTS_EXPECTED_IN_LOOP=326
+
+. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-5-2.t b/tests/basic/ec/ec-5-2.t
new file mode 100644
index 00000000000..4dc1c186f02
--- /dev/null
+++ b/tests/basic/ec/ec-5-2.t
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+# This test checks basic dispersed volume functionality and cli interface
+
+DISPERSE=5
+REDUNDANCY=2
+
+# This must be equal to 44 * $DISPERSE + 106
+TESTS_EXPECTED_IN_LOOP=326
+
+. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-6-2.t b/tests/basic/ec/ec-6-2.t
new file mode 100644
index 00000000000..23ec84e60e9
--- /dev/null
+++ b/tests/basic/ec/ec-6-2.t
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+# This test checks basic dispersed volume functionality and cli interface
+
+DISPERSE=6
+REDUNDANCY=2
+
+# This must be equal to 44 * $DISPERSE + 106
+TESTS_EXPECTED_IN_LOOP=370
+
+. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-7-3.t b/tests/basic/ec/ec-7-3.t
new file mode 100644
index 00000000000..4ebba2a1de3
--- /dev/null
+++ b/tests/basic/ec/ec-7-3.t
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+# This test checks basic dispersed volume functionality and cli interface
+
+DISPERSE=7
+REDUNDANCY=3
+
+# This must be equal to 44 * $DISPERSE + 106
+TESTS_EXPECTED_IN_LOOP=414
+
+. $(dirname $0)/ec-common
diff --git a/tests/basic/ec/ec-common b/tests/basic/ec/ec-common
new file mode 100644
index 00000000000..95f53f250bc
--- /dev/null
+++ b/tests/basic/ec/ec-common
@@ -0,0 +1,143 @@
+
+SIZE_LIST="1048576 1000 12345 0"
+
+LAST_BRICK=$(($DISPERSE - 1))
+
+function fragment_size
+{
+    local fragments=$(($DISPERSE - $REDUNDANCY))
+    local block_size=$((128 * $fragments))
+    local size=$(($1 + $block_size - 1))
+
+    echo $((( $size - ( $size ) % $block_size ) / $fragments))
+}
+
+cleanup
+
+tmp=`mktemp -d`
+if [ ! -d $tmp ]; then
+    exit 1
+fi
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 redundancy $REDUNDANCY $H0:$B0/${V0}{0..$LAST_BRICK}
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+
+TEST dd if=/dev/urandom of=$tmp/small bs=1024 count=1
+TEST dd if=/dev/urandom of=$tmp/big bs=1024 count=4096
+
+cs_small=$(sha1sum $tmp/small | awk '{ print $1 }')
+cs_big=$(sha1sum $tmp/big | awk '{ print $1 }')
+cp $tmp/small $tmp/small1
+for size in $SIZE_LIST; do
+    truncate -s $size $tmp/small1
+    eval cs_small_truncate[$size]=$(sha1sum $tmp/small1 | awk '{ print $1 }')
+done
+cp $tmp/big $tmp/big1
+for size in $SIZE_LIST; do
+    truncate -s $size $tmp/big1
+    eval cs_big_truncate[$size]=$(sha1sum $tmp/big1 | awk '{ print $1 }')
+done
+
+TEST df -h
+TEST stat $M0
+
+for idx in `seq 0 $LAST_BRICK`; do
+    brick[$idx]=$(gf_get_gfid_backend_file_path $B0/$V0$idx)
+done
+
+cd $M0
+EXPECT "2" echo $(ls -a1 | wc -l)
+TEST mkdir dir1
+TEST [ -d dir1 ]
+TEST touch file1
+TEST [ -f file1 ]
+
+for dir in . dir1; do
+    TEST cp $tmp/small $dir/small
+    TEST [ -f $dir/small ]
+    fsize=$(fragment_size 1024)
+    EXPECT "1024" stat -c "%s" $dir/small
+    for idx in `seq 0 $LAST_BRICK`; do
+        EXPECT "$fsize" stat -c "%s" ${brick[$idx]}/$dir/small
+    done
+
+    EXPECT "$cs_small" echo $(sha1sum $dir/small | awk '{ print $1 }')
+
+    TEST cp $tmp/big $dir/big
+    TEST [ -f $dir/big ]
+    fsize=$(fragment_size 4194304)
+    EXPECT "4194304" stat -c "%s" $dir/big
+    for idx in `seq 0 $LAST_BRICK`; do
+        EXPECT "$fsize" stat -c "%s" ${brick[$idx]}/$dir/big
+    done
+
+    EXPECT "$cs_big" echo $(sha1sum $dir/big | awk '{ print $1 }')
+
+    for idx in `seq 0 $LAST_BRICK`; do
+        TEST kill_brick $V0 $H0 $B0/$V0$idx
+
+        EXPECT "1024" stat -c "%s" $dir/small
+        EXPECT "4194304" stat -c "%s" $dir/big
+        EXPECT "$cs_small" echo $(sha1sum $dir/small | awk '{ print $1 }')
+        EXPECT "$cs_big" echo $(sha1sum $dir/big | awk '{ print $1 }')
+
+        cd
+        TEST umount $M0
+        TEST $CLI volume stop $V0 force
+        TEST $CLI volume start $V0
+        TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+        cd $M0
+    done
+
+    for size in $SIZE_LIST; do
+        TEST truncate -s $size $dir/small
+        TEST [ -f $dir/small ]
+        fsize=$(fragment_size $size)
+        EXPECT "$size" stat -c "%s" $dir/small
+        for idx in `seq 0 $LAST_BRICK`; do
+            EXPECT "$fsize" stat -c "%s" ${brick[$idx]}/$dir/small
+        done
+
+        EXPECT "${cs_small_truncate[$size]}" echo $(sha1sum $dir/small | awk '{ print $1 }')
+
+        TEST truncate -s $size $dir/big
+        TEST [ -f $dir/big ]
+        EXPECT "$size" stat -c "%s" $dir/big
+        for idx in `seq 0 $LAST_BRICK`; do
+            EXPECT "$fsize" stat -c "%s" ${brick[$idx]}/$dir/big
+        done
+
+        EXPECT "${cs_big_truncate[$size]}" echo $(sha1sum $dir/big | awk '{ print $1 }')
+    done
+
+    TEST rm -f $dir/small
+    TEST [ ! -e $dir/small ]
+    for idx in `seq 0 $LAST_BRICK`; do
+        TEST [ ! -e ${brick[$idx]}/$dir/small ]
+    done
+
+    TEST rm -f $dir/big
+    TEST [ ! -e $dir/big ]
+    for idx in `seq 0 $LAST_BRICK`; do
+        TEST [ ! -e ${brick[$idx]}/$dir/big ]
+    done
+done
+
+TEST rmdir dir1
+TEST [ ! -e dir1 ]
+for idx in `seq 0 $LAST_BRICK`; do
+    TEST [ ! -e ${brick[$idx]}/dir1 ]
+done
+
+TEST rm -f file1
+TEST [ ! -e file1 ]
+for idx in `seq 0 $LAST_BRICK`; do
+    TEST [ ! -e ${brick[$idx]}/file1 ]
+done
+
+rm -rf $tmp
+
+cleanup
diff --git a/tests/basic/ec/ec.t b/tests/basic/ec/ec.t
new file mode 100644
index 00000000000..e81de0d97bd
--- /dev/null
+++ b/tests/basic/ec/ec.t
@@ -0,0 +1,233 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+TEST_USER=test-ec-user
+TEST_UID=27341
+
+function my_getfattr {
+    getfattr --only-values -e text $* 2> /dev/null
+}
+
+function get_rep_count {
+    v=$(my_getfattr -n trusted.nsr.rep-count $1)
+    #echo $v > /dev/tty
+    echo $v
+}
+
+function create_file {
+    dd if=/dev/urandom of=$1 bs=4k count=$2 conv=sync 2> /dev/null
+}
+
+function setup_perm_file {
+    mkdir $1/perm_dir               || return 1
+    chown ${TEST_USER} $1/perm_dir              || return 1
+    su ${TEST_USER} -c "touch $1/perm_dir/perm_file"    || return 1
+    return 0
+}
+
+# Functions to check repair for specific operation types.
+
+function check_create_write {
+    for b in $*; do
+        cmp $tmpdir/create-write $b/create-write || return 1
+    done
+    return 0
+}
+
+function check_truncate {
+    truncate --size=8192 $tmpdir/truncate
+    for b in $*; do
+        cmp $tmpdir/truncate $b/truncate || return 1
+    done
+    return 0
+}
+
+function check_hard_link {
+    for b in $*; do
+        inum1=$(ls -i $b/hard-link-1 | cut -d' ' -f1)
+        inum2=$(ls -i $b/hard-link-2 | cut -d' ' -f1)
+        [ "$inum1" = "$inum2" ] || return 1
+    done
+    echo "Y"
+    return 0
+}
+
+function check_soft_link {
+    for b in $*; do
+        [ "$(readlink $b/soft-link)" = "soft-link-tgt" ] || return 1
+    done
+    echo "Y"
+    return 0
+}
+
+function check_unlink {
+    for b in $*; do
+        [ ! -e $b/unlink ] || return 1
+    done
+    echo "Y"
+    return 0
+}
+
+function check_mkdir {
+    for b in $*; do
+        [ -d $b/mkdir ] || return 1
+    done
+    echo "Y"
+    return 0
+}
+
+function check_rmdir {
+    for b in $*; do
+        [ ! -e $b/rmdir ] || return 1
+    done
+    echo "Y"
+    return 0
+}
+
+function check_setxattr {
+    for b in $*; do
+        v=$(my_getfattr -n user.foo $b/setxattr)
+        [ "$v" = "ash_nazg_durbatuluk" ] || return 1
+    done
+    echo "Y"
+    return 0
+}
+
+function check_removexattr {
+    for b in $*; do
+        my_getfattr -n user.bar $b/removexattr 2> /dev/null
+        [ $? = 0 ] && return 1
+    done
+    echo "Y"
+    return 0
+}
+
+function check_perm_file {
+    b1=$1
+    shift 1
+    ftext=$(stat -c "%u %g %a" $b1/perm_dir/perm_file)
+    #echo "first u/g/a = $ftext" > /dev/tty
+    for b in $*; do
+        btext=$(stat -c "%u %g %a" $b/perm_dir/perm_file)
+        #echo "  next u/a/a = $btext" > /dev/tty
+        if [ x"$btext" != x"$ftext" ]; then
+            return 1
+        fi
+    done
+    echo "Y"
+    return 0
+}
+
+cleanup
+
+TEST useradd -o -M -u ${TEST_UID} ${TEST_USER}
+trap "userdel --force ${TEST_USER}" EXIT
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+
+TEST mkdir -p $B0/${V0}{0,1,2,3,4,5,6,7,8,9}
+TEST $CLI volume create $V0 disperse 10 redundancy 2 $H0:$B0/${V0}{0,1,2,3,4,5,6,7,8,9}
+
+EXPECT "$V0" volinfo_field $V0 'Volume Name'
+EXPECT 'Created' volinfo_field $V0 'Status'
+EXPECT '10' brick_count $V0
+
+TEST $CLI volume start $V0
+EXPECT 'Started' volinfo_field $V0 'Status'
+
+# Mount FUSE with caching disabled
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0
+
+# Create local files for comparisons etc.
+tmpdir=$(mktemp -d)
+trap "rm -rf $tmpdir" EXIT
+TEST create_file $tmpdir/create-write 10
+TEST create_file $tmpdir/truncate 10
+
+# Prepare files and directories we'll need later.
+TEST cp $tmpdir/truncate $M0/
+TEST touch $M0/hard-link-1
+TEST touch $M0/unlink
+TEST mkdir $M0/rmdir
+TEST touch $M0/setxattr
+TEST touch $M0/removexattr
+TEST setfattr -n user.bar -v "ash_nazg_gimbatul" $M0/removexattr
+
+# Kill a couple of bricks and allow some time for things to settle.
+TEST kill_brick $V0 $H0 $B0/${V0}3
+TEST kill_brick $V0 $H0 $B0/${V0}8
+sleep 10
+
+# Test create+write
+TEST cp $tmpdir/create-write $M0/
+# Test truncate
+TEST truncate --size=8192 $M0/truncate
+# Test hard link
+TEST ln $M0/hard-link-1 $M0/hard-link-2
+# Test soft link
+TEST ln -s soft-link-tgt $M0/soft-link
+# Test unlink
+TEST rm $M0/unlink
+# Test rmdir
+TEST rmdir $M0/rmdir
+# Test mkdir
+TEST mkdir $M0/mkdir
+# Test setxattr
+TEST setfattr -n user.foo -v "ash_nazg_durbatuluk" $M0/setxattr
+# Test removexattr
+TEST setfattr -x user.bar $M0/removexattr
+# Test uid/gid behavior
+TEST setup_perm_file $M0
+
+# Unmount/remount so that create/write and truncate don't see cached data.
+TEST umount $M0
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0
+
+# Test create/write and truncate *before* the bricks are brought back.
+TEST check_create_write $M0
+TEST check_truncate $M0
+
+# Restart the bricks and allow repair to occur.
+TEST $CLI volume start $V0 force
+sleep 10
+
+# Unmount/remount again, same reason as before.
+TEST umount $M0
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0
+
+# Make sure everything is as it should be.  Most tests check for consistency
+# between the bricks and the front end.  This is not valid for disperse, so we
+# check the mountpoint state instead.
+
+TEST check_create_write $M0
+TEST check_truncate $M0
+
+TEST stat $M0/hard-link-1
+TEST stat $M0/hard-link-2
+TEST stat $M0/soft-link
+TEST ! stat $M0/unlink
+TEST ! stat $M0/rmdir
+TEST stat $M0/mkdir
+TEST stat $M0/setxattr
+TEST stat $M0/removexattr
+TEST stat $M0/perm_dir
+TEST stat $M0/perm_dir/perm_file
+
+EXPECT_WITHIN 5 "Y" check_hard_link $B0/${V0}{0..9}
+EXPECT_WITHIN 5 "Y" check_soft_link $B0/${V0}{0..9}
+EXPECT_WITHIN 5 "Y" check_unlink $B0/${V0}{0..9}
+EXPECT_WITHIN 5 "Y" check_rmdir $B0/${V0}{0..9}
+EXPECT_WITHIN 5 "Y" check_mkdir $B0/${V0}{0..9}
+EXPECT_WITHIN 5 "Y" check_setxattr $B0/${V0}{0..9}
+EXPECT_WITHIN 5 "Y" check_removexattr $B0/${V0}{0..9}
+EXPECT_WITHIN 5 "Y" check_perm_file $B0/${V0}{0..9}
+
+rm -rf $tmpdir
+userdel --force ${TEST_USER}
+
+cleanup
+
diff --git a/tests/basic/ec/self-heal.t b/tests/basic/ec/self-heal.t
new file mode 100644
index 00000000000..99cfd9420aa
--- /dev/null
+++ b/tests/basic/ec/self-heal.t
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+# This test checks self-healing feature of dispersed volumes
+
+cleanup
+
+tmp=`mktemp -d`
+if [ ! -d $tmp ]; then
+    exit 1
+fi
+
+TESTS_EXPECTED_IN_LOOP=85
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5}
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+
+TEST dd if=/dev/urandom of=$tmp/test bs=1024 count=1024
+
+cs=$(sha1sum $tmp/test | awk '{ print $1 }')
+
+TEST df -h
+TEST stat $M0
+
+for idx in {0..5}; do
+    brick[$idx]=$(gf_get_gfid_backend_file_path $B0/$V0$idx)
+done
+
+cd $M0
+TEST cp $tmp/test test
+TEST chmod 644 test
+EXPECT "-rw-r--r--" stat -c "%A" test
+
+for idx1 in {0..5}; do
+    TEST chmod 666 ${brick[$idx1]}/test
+    sleep 1
+    EXPECT "-rw-r--r--" stat -c "%A" test
+    EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test
+done
+
+for idx1 in {0..4}; do
+    for idx2 in `seq $(($idx1 + 1)) 5`; do
+        if [ $idx1 -ne $idx2 ]; then
+            TEST chmod 666 ${brick[$idx1]}/test
+            TEST chmod 600 ${brick[$idx2]}/test
+            sleep 1
+            EXPECT "-rw-r--r--" stat -c "%A" test
+            EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test
+            EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx2]}/test
+        fi
+    done
+done
+
+TEST truncate -s 0 ${brick[0]}/test
+TEST truncate -s 2097152 ${brick[1]}/test
+TEST setfattr -n user.test -v "test1" ${brick[0]}/test
+TEST setfattr -n user.test -v "test2" ${brick[1]}/test
+TEST chmod 600 ${brick[0]}/test
+TEST chmod 666 ${brick[1]}/test
+sleep 1
+
+EXPECT "1048576" stat -c "%s" test
+TEST ! getfattr -n user.test test
+
+EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[0]}/test
+EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[1]}/test
+TEST ! getfattr -n user.test ${brick[0]}/test
+TEST ! getfattr -n user.test ${brick[1]}/test
+EXPECT "-rw-r--r--" stat -c "%A" ${brick[0]}/test
+EXPECT "-rw-r--r--" stat -c "%A" ${brick[1]}/test
+
+TEST kill_brick $V0 $H0 $B0/${V0}0
+TEST kill_brick $V0 $H0 $B0/${V0}1
+TEST cp $tmp/test test2
+EXPECT "1048576" stat -c "%s" test2
+TEST chmod 777 test2
+EXPECT "-rwxrwxrwx" stat -c "%A" test2
+
+TEST mkdir dir1
+TEST ls -al dir1
+
+TEST ln -s test2 test3
+TEST [ -h test3 ]
+
+TEST ln test2 test4
+TEST [ -f test4 ]
+EXPECT "2" stat -c "%h" test2
+EXPECT "2" stat -c "%h" test4
+
+cd
+TEST umount $M0
+TEST $CLI volume stop $V0 force
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+cd $M0
+
+EXPECT "1048576" stat -c "%s" test2
+EXPECT "-rwxrwxrwx" stat -c "%A" test2
+EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[0]}/test2
+EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[1]}/test2
+EXPECT "-rwxrwxrwx" stat -c "%A" ${brick[0]}/test2
+EXPECT "-rwxrwxrwx" stat -c "%A" ${brick[1]}/test2
+
+TEST ls -al dir1
+EXPECT_WITHIN 5 "1" eval "if [ -d ${brick[0]}/dir1 ]; then echo 1; fi"
+EXPECT_WITHIN 5 "1" eval "if [ -d ${brick[1]}/dir1 ]; then echo 1; fi"
+
+TEST [ -h test3 ]
+EXPECT_WITHIN 5 "1" eval "if [ -h ${brick[0]}/test3 ]; then echo 1; fi"
+EXPECT_WITHIN 5 "1" eval "if [ -h ${brick[1]}/test3 ]; then echo 1; fi"
+
+EXPECT "2" stat -c "%h" test4
+EXPECT_WITHIN 5 "3" stat -c "%h" ${brick[0]}/test4
+EXPECT_WITHIN 5 "3" stat -c "%h" ${brick[1]}/test4
+
+rm -rf $tmp
+
+cleanup