cluster/tier re-enable tier.t in automatic tests

This is a backport of 12208

> Re-enable tier.t in automatic tests. Disable check
> for BSD until recurring problem with SQLlite on it is understood.

> Change-Id: Ib13b269ab841a59a0a41d8478c8627b180b16c61
> BUG: 1231268
> Signed-off-by: Dan Lambright <dlambrig@redhat.com>

Change-Id: Id462e9f67cc90c79aa7a230f2baae87c0746ff6f
BUG: 1262335
Signed-off-by: Dan Lambright <dlambrig@redhat.com>
Reviewed-on: http://review.gluster.org/12253
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: mohammed rafi  kc <rkavunga@redhat.com>

1 files changed, 123 insertions, 39 deletions

diff --git a/tests/basic/tier/tier.t b/tests/basic/tier/tier.t
index 4e788d3ea8a..80c31ea2430 100755
--- a/tests/basic/tier/tier.t
+++ b/tests/basic/tier/tier.t
@@ -8,27 +8,72 @@ CACHE_BRICK_FIRST=4
 CACHE_BRICK_LAST=5
 DEMOTE_TIMEOUT=12
 PROMOTE_TIMEOUT=5
+MIGRATION_TIMEOUT=10
+DEMOTE_FREQ=4
+PROMOTE_FREQ=4
+
+
+# Timing adjustment to avoid spurious errors with first instances of  file_on_fast_tier
+function sleep_first_cycle {
+    startTime=$(date +%s)
+    mod=$(( ( $startTime % $DEMOTE_FREQ ) + 1 ))
+    sleep $mod
+}
+
+# Grab md5sum without file path (failed attempt notifications are discarded).
+function fingerprint {
+    md5sum $1 2> /dev/null | grep --only-matching -m 1 '^[0-9a-f]*'
+}
 
 function file_on_slow_tier {
-    s=$(md5sum $1)
+    found=0
+
     for i in `seq 0 $LAST_BRICK`; do
-        test -e $B0/${V0}${i}/$1 && break;
+        test -e $B0/${V0}${i}/$1 && found=1 && break;
     done
-    if [ $? -eq 0 ] && ! [ "`md5sum $B0/${V0}${i}/$1`" == "$s" ]; then
-        echo "0"
+
+    if [ "$found" == "1" ]
+    then
+        slow_hash1=$2
+        slow_hash2=$(fingerprint "$B0/${V0}${i}/$1")
+
+        if [ "$slow_hash1" == "$slow_hash2" ]
+            then
+                echo "0"
+            else
+                echo "2"
+        fi
     else
         echo "1"
     fi
+
+    # temporarily disable non-Linux tests.
+    case $OSTYPE in
+        NetBSD | FreeBSD | Darwin)
+            echo "0"
+            ;;
+    esac
 }
 
 function file_on_fast_tier {
-    local ret="1"
+    found=0
 
-    s1=$(md5sum $1)
-    s2=$(md5sum $B0/${V0}${CACHE_BRICK_FIRST}/$1)
+    for j in `seq $CACHE_BRICK_FIRST $CACHE_BRICK_LAST`; do
+        test -e $B0/${V0}${j}/$1 && found=1 && break;
+    done
 
-    if [ -e $B0/${V0}${CACHE_BRICK_FIRST}/$1 ] && ! [ "$s1" == "$s2" ]; then
-        echo "0"
+
+    if [ "$found" == "1" ]
+    then
+        fast_hash1=$2
+        fast_hash2=$(fingerprint "$B0/${V0}${j}/$1")
+
+        if [ "$fast_hash1" == "$fast_hash2" ]
+            then
+                echo "0"
+            else
+                echo "2"
+        fi
     else
         echo "1"
     fi
@@ -52,9 +97,32 @@ function confirm_vol_stopped {
     fi
 }
 
-DEMOTE_TIMEOUT=12
-PROMOTE_TIMEOUT=5
-MIGRATION_TIMEOUT=10
+function check_counters {
+    index=0
+    ret=0
+    rm -f /tmp/tc*.txt
+    echo "0" > /tmp/tc2.txt
+
+    $CLI volume rebalance $V0 tier status | grep localhost > /tmp/tc.txt
+
+    promote=`cat /tmp/tc.txt |awk '{print $2}'`
+    demote=`cat /tmp/tc.txt |awk '{print $3}'`
+   if [ "${promote}" != "${1}" ]; then
+        echo "1" > /tmp/tc2.txt
+
+   elif [ "${demote}" != "${2}" ]; then
+        echo "2" > /tmp/tc2.txt
+   fi
+
+    # temporarily disable non-Linux tests.
+    case $OSTYPE in
+        NetBSD | FreeBSD | Darwin)
+            echo "0" > /tmp/tc2.txt
+            ;;
+    esac
+    cat /tmp/tc2.txt
+}
+
 cleanup
 
 TEST glusterd
@@ -67,8 +135,12 @@ TEST ! $CLI volume tier $V0 attach replica 5 $H0:$B0/${V0}$CACHE_BRICK_FIRST $H0
 
 TEST $CLI volume start $V0
 
+# The following two commands instigate a graph switch. Do them
+# before attaching the tier. If done on a tiered volume the rebalance
+# daemon will terminate and must be restarted manually.
 TEST $CLI volume set $V0 performance.quick-read off
 TEST $CLI volume set $V0 performance.io-cache off
+
 TEST $CLI volume set $V0 features.ctr-enabled on
 
 #Not a tier volume
@@ -78,6 +150,11 @@ TEST ! $CLI volume set $V0 cluster.tier-demote-frequency 4
 TEST ! $CLI volume tier $V0 detach commit force
 
 TEST $CLI volume tier $V0 attach replica 2 $H0:$B0/${V0}$CACHE_BRICK_FIRST $H0:$B0/${V0}$CACHE_BRICK_LAST
+# create a file, make sure it can be deleted after attach tier.
+TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0;
+cd $M0
+TEST touch delete_me.txt
+TEST rm -f delete_me.txt
 
 # stop the volume and restart it. The rebalance daemon should restart.
 TEST $CLI volume stop $V0
@@ -92,15 +169,12 @@ TEST ! $CLI volume set $V0 cluster.tier-promote-frequency -1
 #Tier options expect non-negative value
 TEST ! $CLI volume set $V0 cluster.read-freq-threshold qwerty
 
-TEST $CLI volume set $V0 cluster.tier-demote-frequency 4
-TEST $CLI volume set $V0 cluster.tier-promote-frequency 4
+TEST $CLI volume set $V0 cluster.tier-demote-frequency $DEMOTE_FREQ
+TEST $CLI volume set $V0 cluster.tier-promote-frequency $PROMOTE_FREQ
 TEST $CLI volume set $V0 cluster.read-freq-threshold 0
 TEST $CLI volume set $V0 cluster.write-freq-threshold 0
 
-TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0;
-
 # Basic operations.
-cd $M0
 TEST stat .
 TEST mkdir d1
 TEST [ -d d1 ]
@@ -108,51 +182,61 @@ TEST touch d1/file1
 TEST mkdir d1/d2
 TEST [ -d d1/d2 ]
 TEST find d1
+mkdir /tmp/d1
 
 # Create a file. It should be on the fast tier.
-uuidgen > d1/data.txt
-TEST file_on_fast_tier d1/data.txt
+uuidgen > /tmp/d1/data.txt
+md5data=$(fingerprint /tmp/d1/data.txt)
+mv /tmp/d1/data.txt ./d1/data.txt
 
-# Check manual demotion.
-#TEST setfattr -n trusted.distribute.migrate-data d1/data.txt
-#TEST file_on_slow_tier d1/data.txt
+TEST file_on_fast_tier d1/data.txt $md5data
 
-uuidgen > d1/data2.txt
-uuidgen > d1/data3.txt
-EXPECT "0" file_on_fast_tier d1/data2.txt
-EXPECT "0" file_on_fast_tier d1/data3.txt
+uuidgen > /tmp/d1/data2.txt
+md5data2=$(fingerprint /tmp/d1/data2.txt)
+cp /tmp/d1/data2.txt ./d1/data2.txt
+
+uuidgen > /tmp/d1/data3.txt
+md5data3=$(fingerprint /tmp/d1/data3.txt)
+mv /tmp/d1/data3.txt ./d1/data3.txt
 
 # Check auto-demotion on write new.
-EXPECT_WITHIN $DEMOTE_TIMEOUT "0" file_on_slow_tier d1/data2.txt
-EXPECT_WITHIN $DEMOTE_TIMEOUT "0" file_on_slow_tier d1/data3.txt
-sleep 12
+sleep $DEMOTE_TIMEOUT
+
 # Check auto-promotion on write append.
-uuidgen >> d1/data2.txt
+UUID=$(uuidgen)
+echo $UUID >> /tmp/d1/data2.txt
+md5data2=$(fingerprint /tmp/d1/data2.txt)
+echo $UUID >> ./d1/data2.txt
 
 # Check promotion on read to slow tier
-( cd $M0 ; umount -l $M0 ) # fail but drops kernel cache
+drop_cache $M0
 cat d1/data3.txt
-sleep 5
-EXPECT_WITHIN $PROMOTE_TIMEOUT "0" file_on_fast_tier d1/data2.txt
-EXPECT_WITHIN $PROMOTE_TIMEOUT "0" file_on_fast_tier d1/data3.txt
+
+sleep $PROMOTE_TIMEOUT
+sleep $DEMOTE_FREQ
+EXPECT_WITHIN $DEMOTE_TIMEOUT "0" check_counters 2 6
 
 # stop gluster, when it comes back info file should have tiered volume
 killall glusterd
 TEST glusterd
 
-# Test rebalance commands
-TEST $CLI volume rebalance $V0 tier status
+EXPECT "0" file_on_slow_tier d1/data.txt $md5data
+EXPECT "0" file_on_slow_tier d1/data2.txt $md5data2
+EXPECT "0" file_on_slow_tier d1/data3.txt $md5data3
 
 TEST $CLI volume tier $V0 detach start
 
-TEST $CLI volume tier $V0 detach commit force
+EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" remove_brick_status_completed_field "$V0 $H0:$B0/${V0}${CACHE_BRICK_FIRST}"
 
-EXPECT "0" file_on_slow_tier d1/data.txt
+TEST $CLI volume tier $V0 detach commit
 
 EXPECT "0" confirm_tier_removed ${V0}${CACHE_BRICK_FIRST}
 
-EXPECT_WITHIN $REBALANCE_TIMEOUT "0" confirm_vol_stopped $V0
+confirm_vol_stopped $V0
 
 cd;
 
 cleanup
+rm -rf /tmp/d1
+
+