tier : Tier as a service

tierd is implemented by separating from rebalance process.

The commands affected:

1) Attach tier will trigger this process instead of old one
2) tier start and tier start force will also trigger this process.
3) volume status [tier] will show tier daemon as a process instead
of task and normal tier status and tier detach status works.
4) tier stop implemented.
5) detach tier implemented separately along with new detach tier
status
6) volume tier volname status will work using the changes.
7) volume set works

This patch has separated the tier translator from the legacy
DHT rebalance code. It now sends the RPCs from the CLI
to glusterd separate to the DHT rebalance code.
The daemon is now a service, similar to the snapshot daemon,
and can be viewed using the volume status command.

The code for the validation and commit phase are the same
as the earlier tier validation code in DHT rebalance.

The “brickop” phase has been changed so that the status
command can use this framework.

The service management framework is now used.
DHT rebalance does not use this framework.

This service framework takes care of :

*) spawning the daemon, killing it and other such processes.
*) volume set options , which are written on the volfile.
*) restart and reconfigure functions. Restart is to restart
the daemon at two points
        1)after gluster goes down and comes up.
        2) to stop detach tier.
*) reconfigure is used to make immediate volfile changes.
By doing this, we don’t restart the daemon.
it has the code to rewrite the volfile for topological
changes too (which comes into place during add and remove brick).

With this patch the log, pid, and volfile are separated
and put into respective directories.

Change-Id: I3681d0d66894714b55aa02ca2a30ac000362a399
BUG: 1313838
Signed-off-by: hari gowtham <hgowtham@redhat.com>
Reviewed-on: http://review.gluster.org/13365
Smoke: Gluster Build System <jenkins@build.gluster.org>
Tested-by: hari gowtham <hari.gowtham005@gmail.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Dan Lambright <dlambrig@redhat.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

8 files changed, 36 insertions, 14 deletions

diff --git a/tests/basic/tier/legacy-many.t b/tests/basic/tier/legacy-many.t
index a9fcba85209..1782ca70088 100644
--- a/tests/basic/tier/legacy-many.t
+++ b/tests/basic/tier/legacy-many.t
@@ -23,6 +23,10 @@ function read_all {
     done
 }
 
+function tier_status () {
+        $CLI volume tier $V0 status | grep "success" | wc -l
+}
+
 cleanup
 
 TEST glusterd
@@ -47,7 +51,6 @@ wait
 
 # Attach tier
 TEST $CLI volume attach-tier $V0 replica 2 $H0:$B0/${V0}$CACHE_BRICK_FIRST $H0:$B0/${V0}$CACHE_BRICK_LAST
-TEST $CLI volume rebalance $V0 tier status
 
 TEST $CLI volume set $V0 cluster.tier-mode test
 TEST $CLI volume set $V0 cluster.tier-demote-frequency $DEMOTE_FREQ
@@ -56,7 +59,9 @@ TEST $CLI volume set $V0 cluster.read-freq-threshold 0
 TEST $CLI volume set $V0 cluster.write-freq-threshold 0
 
 # wait a little for lookup heal to finish
-sleep 10
+wait_for_tier_start
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" tier_status
 
 # make sure fix layout completed
 CPATH=$B0/${V0}0
diff --git a/tests/basic/tier/new-tier-cmds.t b/tests/basic/tier/new-tier-cmds.t
index dbfac54938e..afc875710ac 100644
--- a/tests/basic/tier/new-tier-cmds.t
+++ b/tests/basic/tier/new-tier-cmds.t
@@ -40,7 +40,7 @@ EXPECT "Tier command failed" $CLI_1 volume tier $V0 detach status
 
 TEST $CLI_1 volume tier $V0 detach start
 
-TEST $CLI_1 volume tier $V0 detach status
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" tier_detach_status
 
 #kill a node
 TEST kill_node 2
@@ -55,7 +55,7 @@ TEST $glusterd_2;
 
 EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers;
 
-TEST $CLI_1 volume tier $V0 detach status
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" tier_detach_status
 
 TEST $CLI_1 volume tier $V0 detach stop
 
diff --git a/tests/basic/tier/tier.t b/tests/basic/tier/tier.t
index 7bac2dd03bb..a5bd09f8028 100755
--- a/tests/basic/tier/tier.t
+++ b/tests/basic/tier/tier.t
@@ -119,6 +119,9 @@ cd /tmp
 umount $M0
 TEST $CLI volume stop $V0
 TEST $CLI volume start $V0
+
+wait_for_tier_start
+
 TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0;
 cd $M0
 
diff --git a/tests/basic/tier/tierd_check.t b/tests/basic/tier/tierd_check.t
index 1f88ea0b72e..6aef1048ee2 100644
--- a/tests/basic/tier/tierd_check.t
+++ b/tests/basic/tier/tierd_check.t
@@ -24,7 +24,7 @@ function tier_status () {
 }
 
 function tier_deamon_kill () {
-pkill -f "rebalance/$V0"
+pkill -f "tierd/$V0"
 echo "$?"
 }
 
@@ -40,6 +40,8 @@ EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers;
 #Create and start a tiered volume
 create_dist_tier_vol
 
+wait_for_tier_start
+
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 tier_daemon_check
 
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" tier_status
@@ -48,6 +50,8 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 tier_deamon_kill
 
 TEST $CLI_1 volume tier $V0 start
 
+wait_for_tier_start
+
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_daemon_check
 
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" tier_status
@@ -56,6 +60,8 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_kill
 
 TEST $CLI_3 volume tier $V0 start force
 
+wait_for_tier_start
+
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_daemon_check
 
 #The pattern progress should occur twice only.
@@ -79,6 +85,8 @@ TEST $CLI_3 volume stop $V0
 
 TEST $CLI_3 volume start $V0
 
+wait_for_tier_start
+
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" tier_status
 
 #check for detach start and stop
diff --git a/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t b/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t
index 75e2d337687..19defe435c1 100644
--- a/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t
+++ b/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t
@@ -40,7 +40,7 @@ TEST $CLI volume status
 #Create and start a tiered volume
 create_dist_tier_vol
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 tier_daemon_check
-sleep 2   #wait for some time to run tier daemon
+sleep 5   #wait for some time to run tier daemon
 time_before_restarting=$(rebalance_run_time $V0);
 
 #checking for elapsed time after sleeping for two seconds.
diff --git a/tests/bugs/replicate/bug-1290965-detect-bitrotten-objects.t b/tests/bugs/replicate/bug-1290965-detect-bitrotten-objects.t
index 393d1448075..1828e6f3493 100644
--- a/tests/bugs/replicate/bug-1290965-detect-bitrotten-objects.t
+++ b/tests/bugs/replicate/bug-1290965-detect-bitrotten-objects.t
@@ -44,7 +44,7 @@ stat $M0/FILE
 # Remove hot-tier
 TEST $CLI volume tier $V0  detach start
 sleep 1
-EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" detach_tier_status_field $V0
+EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" detach_tier_status_field_complete $V0
 TEST $CLI volume tier $V0  detach commit
 #Test that file has migrated to cold tier.
 EXPECT "1024" stat -c "%s" $B0/brick0/FILE
diff --git a/tests/tier.rc b/tests/tier.rc
index 69512c3fb4e..da6b0ee4777 100644
--- a/tests/tier.rc
+++ b/tests/tier.rc
@@ -39,7 +39,7 @@ function check_counters {
     ret=0
     rm -f /tmp/tc*.txt
     echo "0" > /tmp/tc2.txt
-    $CLI volume rebalance $V0 tier status | grep localhost > /tmp/tc.txt
+    $CLI volume tier $V0 status | grep localhost > /tmp/tc.txt
 
     promote=`cat /tmp/tc.txt |awk '{print $2}'`
     demote=`cat /tmp/tc.txt |awk '{print $3}'`
@@ -121,17 +121,16 @@ function sleep_until_mid_cycle {
 }
 
 function tier_daemon_check () {
-    pgrep -f "rebalance/$V0"
+    pgrep -f "tierd/$V0"
     echo "$?"
 }
 
 function rebalance_run_time () {
-    local time=$($CLI volume rebalance $1  status | awk '{print $9}' | sed -n 3p);
+    local time=$($CLI volume tier $1 status | awk '{print $6}' | sed -n 3p);
     local hh=$(echo $time | cut -d ':' -f1);
     local mm=$(echo $time | cut -d ':' -f2);
     local ss=$(echo $time | cut -d ':' -f3);
     local total=$(($hh * 3600 + $mm * 60 + $ss));
-
     echo $total;
 }
 
@@ -144,6 +143,13 @@ function tier_detach_status_node_down () {
 }
 
 function tier_status_node_down () {
-	$CLI_1 volume tier $V0 status | grep "WARNING" | wc -l
+        $CLI_1 volume tier $V0 status | grep "WARNING" | wc -l
+}
+
+function tier_detach_status () {
+        $CLI_1 volume tier $V0 detach status | grep "success" | wc -l
 }
 
+function wait_for_tier_start () {
+        sleep 5
+}
diff --git a/tests/volume.rc b/tests/volume.rc
index 8cad822e9ad..e3ae408f971 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -53,8 +53,8 @@ function fix-layout_status_field {
         $CLI volume rebalance $1 status | awk '{print $2,$3,$4}' |sed -n 3p |tr -d '[^0-9+\.]'|sed 's/ *$//g'
 }
 
-function detach_tier_status_field {
-        $CLI volume tier  $1 detach status | awk '{print $7,$8,$9}' |sed -n 3p |tr -d '[^0-9+\.]'|sed 's/ *$//g'
+function detach_tier_status_field_complete {
+        $CLI volume tier  $1 detach status | awk '{print $7}' |sed -n 4p
 }
 
 function remove_brick_status_completed_field {