mgmt/glusterd: Implement Volume heal enable/disable

For volumes with replicate, disperse xlators, self-heal daemon should do
healing. This patch provides enable/disable functionality for the xlators to be
part of self-heal-daemon. Replicate already had this functionality with
'gluster volume set cluster.self-heal-daemon on/off'. But this patch makes it
uniform for both types of volumes. Internally it still does 'volume set' based
on the volume type.

Change-Id: Ie0f3799b74c2afef9ac658ef3d50dce3e8072b29
BUG: 1177601
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/9358
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Xavier Hernandez <xhernandez@datalab.es>
Tested-by: Krishnan Parthasarathi <kparthas@redhat.com>

2 files changed, 114 insertions, 5 deletions

diff --git a/tests/basic/glusterd/heald.t b/tests/basic/glusterd/heald.t
new file mode 100644
index 00000000000..6683ea2c500
--- /dev/null
+++ b/tests/basic/glusterd/heald.t
@@ -0,0 +1,89 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+# This test contains volume heal commands handled by glusterd.
+# Covers enable/disable at the moment. Will be enhanced later to include
+# the other commands as well.
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+volfile=$(gluster system:: getwd)"/glustershd/glustershd-server.vol"
+#Commands should fail when volume doesn't exist
+TEST ! $CLI volume heal non-existent-volume enable
+TEST ! $CLI volume heal non-existent-volume disable
+
+# Commands should fail when volume is of distribute/stripe type.
+# Glustershd shouldn't be running as long as there are no replicate/disperse
+# volumes
+TEST $CLI volume create dist $H0:$B0/dist
+TEST $CLI volume start dist
+TEST "[ -z $(get_shd_process_pid)]"
+TEST ! $CLI volume heal dist enable
+TEST ! $CLI volume heal dist disable
+TEST $CLI volume create st stripe 3 $H0:$B0/st1 $H0:$B0/st2 $H0:$B0/st3
+TEST $CLI volume start st
+TEST "[ -z $(get_shd_process_pid)]"
+TEST ! $CLI volume heal st
+TEST ! $CLI volume heal st disable
+
+# Commands should work on replicate/disperse volume.
+TEST $CLI volume create r2 replica 2 $H0:$B0/r2_0 $H0:$B0/r2_1
+TEST "[ -z $(get_shd_process_pid)]"
+TEST $CLI volume start r2
+EXPECT "[0-9][0-9]*" get_shd_process_pid
+TEST $CLI volume heal r2 enable
+EXPECT "enable" volume_option r2 "cluster.self-heal-daemon"
+EXPECT "enable" volgen_volume_option $volfile r2-replicate-0 cluster replicate self-heal-daemon
+EXPECT "[0-9][0-9]*" get_shd_process_pid
+TEST $CLI volume heal r2 disable
+EXPECT "disable" volume_option r2 "cluster.self-heal-daemon"
+EXPECT "disable" volgen_volume_option $volfile r2-replicate-0 cluster replicate self-heal-daemon
+EXPECT "[0-9][0-9]*" get_shd_process_pid
+
+# Commands should work on disperse volume.
+TEST $CLI volume create ec2 disperse 3 redundancy 1 $H0:$B0/ec2_0 $H0:$B0/ec2_1 $H0:$B0/ec2_2
+TEST $CLI volume start ec2
+EXPECT "[0-9][0-9]*" get_shd_process_pid
+TEST $CLI volume heal ec2 enable
+EXPECT "enable" volume_option ec2 "cluster.disperse-self-heal-daemon"
+EXPECT "enable" volgen_volume_option $volfile ec2-disperse-0 cluster disperse self-heal-daemon
+EXPECT "[0-9][0-9]*" get_shd_process_pid
+TEST $CLI volume heal ec2 disable
+EXPECT "disable" volume_option ec2 "cluster.disperse-self-heal-daemon"
+EXPECT "disable" volgen_volume_option $volfile ec2-disperse-0 cluster disperse self-heal-daemon
+EXPECT "[0-9][0-9]*" get_shd_process_pid
+
+#Check that shd graph is rewritten correctly on volume stop/start
+EXPECT "Y" volgen_volume_exists $volfile ec2-disperse-0 cluster disperse
+EXPECT "Y" volgen_volume_exists $volfile r2-replicate-0 cluster replicate
+TEST $CLI volume stop r2
+EXPECT "Y" volgen_volume_exists $volfile ec2-disperse-0 cluster disperse
+EXPECT "N" volgen_volume_exists $volfile r2-replicate-0 cluster replicate
+TEST $CLI volume stop ec2
+# When both the volumes are stopped glustershd volfile is not modified just the
+# process is stopped
+TEST "[ -z $(get_shd_process_pid) ]"
+
+TEST $CLI volume start r2
+EXPECT "N" volgen_volume_exists $volfile ec2-disperse-0 cluster disperse
+EXPECT "Y" volgen_volume_exists $volfile r2-replicate-0 cluster replicate
+
+TEST $CLI volume set r2 self-heal-daemon on
+TEST $CLI volume set r2 cluster.self-heal-daemon off
+TEST ! $CLI volume set ec2 self-heal-daemon off
+TEST ! $CLI volume set ec2 cluster.self-heal-daemon on
+TEST ! $CLI volume set dist self-heal-daemon off
+TEST ! $CLI volume set dist cluster.self-heal-daemon on
+
+TEST $CLI volume set ec2 disperse-self-heal-daemon off
+TEST $CLI volume set ec2 cluster.disperse-self-heal-daemon on
+TEST ! $CLI volume set r2 disperse-self-heal-daemon on
+TEST ! $CLI volume set r2 cluster.disperse-self-heal-daemon off
+TEST ! $CLI volume set dist disperse-self-heal-daemon off
+TEST ! $CLI volume set dist cluster.disperse-self-heal-daemon on
+
+cleanup
diff --git a/tests/volume.rc b/tests/volume.rc
index daa9c140fd8..2fd07cd8745 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -135,18 +135,15 @@ function ec_child_up_count {
 }
 
 function get_shd_process_pid {
-        local vol=$1
         ps auxww | grep glusterfs | grep -E "glustershd/run/glustershd.pid" | awk '{print $2}' | head -1
 }
 
 function generate_shd_statedump {
-        local vol=$1
-        generate_statedump $(get_shd_process_pid $vol)
+        generate_statedump $(get_shd_process_pid)
 }
 
 function generate_nfs_statedump {
-        local vol=$1
-        generate_statedump $(get_nfs_pid $vol)
+        generate_statedump $(get_nfs_pid)
 }
 
 function generate_brick_statedump {
@@ -425,3 +422,26 @@ function assign_gfid {
 function get_random_gfid {
         echo "0x"$(uuidgen | awk -F '-' 'BEGIN {OFS=""} {print $1,$2,$3,$4,$5}')
 }
+
+function volgen_volume_exists {
+        local volfile="$1"
+        local xl_vol="$2"
+        local xl_type="$3"
+        local xl_feature="$4"
+        xl=$(sed -e "/./{H;\$!d;}" -e "x;/volume $xl_vol/!d;/type $xl_type\/$xl_feature/!d" $volfile)
+        if [ -z "$xl" ];
+        then
+                echo "N"
+        else
+                echo "Y"
+        fi
+}
+
+function volgen_volume_option {
+        local volfile="$1"
+        local xl_vol="$2"
+        local xl_type="$3"
+        local xl_feature="$4"
+        local xl_option="$5"
+        sed -e "/./{H;\$!d;}" -e "x;/volume $xl_vol/!d;/type $xl_type\/$xl_feature/!d;/option $xl_option/!d" $volfile | grep " $xl_option " | awk '{print $3}'
+}