Adding halo-enable option

Summary:
- Master option for halo geo-replication
- Added prove test for halo mode
- Updated options do values which should work "out of the box" for most use-cases, just run "enable" halo mode and you are done, the options can be tweaked as needed from there.

Test Plan:
- Enabled option, verified halo works, disabled option verified async
  behavior is disabled
- Ran "prove -v tests/basic/halo.t" -> https://phabricator.fb.com/P12074204

Reviewers: jackl, dph, cjh

Reviewed By: cjh

Subscribers: meyering

Differential Revision: https://phabricator.fb.com/D1386675

Tasks: 4117827

Conflicts:
	xlators/cluster/afr/src/afr-self-heal-common.c
	xlators/cluster/afr/src/afr.h

Change-Id: Ib704528d438c3a42150e30974eb6bb01d9e795ae
Signed-off-by: Kevin Vigor <kvigor@fb.com>
Reviewed-on: http://review.gluster.org/16172
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Shreyas Siravara <sshreyas@fb.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

4 files changed, 76 insertions, 7 deletions

diff --git a/tests/basic/halo.t b/tests/basic/halo.t
new file mode 100644
index 00000000000..03fc0f88a19
--- /dev/null
+++ b/tests/basic/halo.t
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Test for the Halo geo-replication feature
+#
+# 1. Create volume w/ 3x replication w/ max-replicas = 2 for clients,
+#    heal daemon is off to start.
+# 2. Write some data
+# 3. Verify at least one of the bricks did not receive the writes.
+# 4. Turn the heal daemon on
+# 5. Within 30 seconds the SHD should async heal the data over
+#    to the 3rd brick.
+#
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2}
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+TEST $CLI volume set $V0 cluster.shd-max-threads 1
+TEST $CLI volume set $V0 cluster.halo-enabled True
+TEST $CLI volume set $V0 cluster.halo-max-replicas 2
+TEST $CLI volume set $V0 cluster.heal-timeout 5
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+TEST $CLI volume set $V0 cluster.eager-lock off
+TEST $CLI volume set $V0 cluster.choose-local off
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+cd $M0
+
+for i in {1..5}
+do
+        dd if=/dev/urandom of=f bs=1M count=1 2>/dev/null
+        mkdir a; cd a;
+done
+
+B0_CNT=$(ls $B0/${V0}0 | wc -l)
+B1_CNT=$(ls $B0/${V0}1 | wc -l)
+B2_CNT=$(ls $B0/${V0}2 | wc -l)
+
+# One of the brick dirs should be empty
+TEST "(($B0_CNT == 0 || $B1_CNT == 0 || $B2_CNT == 0))"
+
+# Ok, turn the heal daemon on and verify it heals it up
+TEST $CLI volume set $V0 cluster.self-heal-daemon on
+EXPECT_WITHIN 30 "0" get_pending_heal_count $V0
+cleanup
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index d1a1e9837da..d9c740dc498 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -176,6 +176,10 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("data-self-heal-algorithm",
                           priv->data_self_heal_algorithm, options, str, out);
 
+        GF_OPTION_RECONF ("halo-enabled",
+                          priv->halo_enabled, options, bool,
+                          out);
+
         GF_OPTION_RECONF ("halo-shd-max-latency",
                           priv->shd.halo_max_latency_msec, options, uint32,
                           out);
@@ -413,6 +417,9 @@ init (xlator_t *this)
 
         GF_OPTION_INIT ("entry-self-heal", priv->entry_self_heal, bool, out);
 
+        GF_OPTION_INIT ("halo-enabled",
+                        priv->halo_enabled, bool, out);
+
         GF_OPTION_INIT ("halo-shd-max-latency", priv->shd.halo_max_latency_msec,
                         uint32, out);
 
@@ -704,21 +711,26 @@ struct volume_options options[] = {
           .type  = GF_OPTION_TYPE_INT,
           .min   = 1,
           .max   = 99999,
-          .default_value = "500",
+          .default_value = "99999",
            .description = "Maximum latency for shd halo replication in msec."
         },
+        { .key   = {"halo-enabled"},
+          .type  = GF_OPTION_TYPE_BOOL,
+          .default_value = "False",
+           .description = "Enable Halo (geo) replication mode."
+        },
         { .key   = {"halo-nfsd-max-latency"},
           .type  = GF_OPTION_TYPE_INT,
           .min   = 1,
           .max   = 99999,
-          .default_value = "10",
-           .description = "Maximum latency for nfsd halo replication in msec."
+          .default_value = "5",
+          .description = "Maximum latency for nfsd halo replication in msec."
         },
         { .key   = {"halo-max-latency"},
           .type  = GF_OPTION_TYPE_INT,
           .min   = 1,
           .max   = 99999,
-          .default_value = "10",
+          .default_value = "5",
            .description = "Maximum latency for halo replication in msec."
         },
         { .key   = {"halo-max-replicas"},
@@ -734,8 +746,8 @@ struct volume_options options[] = {
           .type  = GF_OPTION_TYPE_INT,
           .min   = 1,
           .max   = 99999,
-          .default_value = "1",
-           .description = "The minimmum number of halo replicas, before adding "
+          .default_value = "2",
+           .description = "The minimum number of halo replicas, before adding "
                           "out of region replicas."
          },
          { .key  = {"heal-wait-queue-length"},
@@ -948,7 +960,7 @@ struct volume_options options[] = {
 	},
         { .key  = {"heal-timeout"},
           .type = GF_OPTION_TYPE_INT,
-          .min  = 60,
+          .min  = 5,
           .max  = INT_MAX,
           .default_value = "600",
           .description = "time interval for checking the need to self-heal "
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 913f6a7369d..0a51c38d06e 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -147,6 +147,8 @@ typedef struct _afr_private {
         gf_boolean_t           ensure_durability;
         char                   *sh_domain;
 	char                   *afr_dirty;
+        gf_boolean_t           halo_enabled;
+
         uint32_t               halo_max_latency_msec;
         uint32_t               halo_max_replicas;
         uint32_t               halo_min_replicas;
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 314da8fb7e8..f4dd9fcde71 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -3013,6 +3013,11 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .op_version  = GD_OP_VERSION_3_6_0,
           .type        = NO_DOC,
         },
+        { .key        = "cluster.halo-enabled",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
         { .key        = "cluster.halo-shd-max-latency",
           .voltype    = "cluster/replicate",
           .op_version = 2,