nsr: add first reconciliation test

The test itself fails, because reconciliation never actually seems to
complete.  Instead, it fills up several log files with 16M lines worth
of junk, even though we only did ~200 writes.  Fencing is also disabled
so we can make progress despite this.  Still, this test should provide a
basic framework for testing reconciliation once it's fixed.

This patch also fixes the hard-coded executable path used to start the
recon daemon.  Without this, starting the brick fails on a standard
install (because the hard-coded path was to a non-standard location).

Change-Id: I982967d796c3fb5713c5062912be2d804dcc0149
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>

4 files changed, 91 insertions, 3 deletions

diff --git a/tests/basic/recon.t b/tests/basic/recon.t
new file mode 100755
index 000000000..405fcb5d2
--- /dev/null
+++ b/tests/basic/recon.t
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# Test *very basic* NSR functionality - startup, mount, simplest possible file
+# write.
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+function get_rep_count {
+	v=$(getfattr --only-values -e text -n trusted.nsr.rep-count $1 2> /dev/null)
+	#echo $v > /dev/tty
+	echo $v
+}
+
+function ping_file {
+	dd if=/dev/urandom of=$1 bs=4k count=100 2> /dev/null
+}
+
+function kill_a_brick {
+	for r in /var/lib/glusterd/vols/${V0}/run/*-recon.pid; do
+		rpid=$(cat $r)
+		echo "recon PID = $rpid" > /dev/tty
+		b=$(echo $r | sed '/\(.*\):\(.*\)-recon.pid/s//\1\2.pid/')
+		bpid=$(cat $b)
+		echo "brick PID = $bpid" > /dev/tty
+		kill -9 $bpid $rpid
+		return 0
+	done
+
+	# No bricks?!?
+	return 1
+}
+
+function count_matches {
+	n=0
+	for f in $B0/$V0[12]/$1; do
+		cmp $M0/$1 $f 2> /dev/null
+		if [ $? = 0 ]; then
+			n=$((n+1))
+		fi
+	done
+	echo $n
+}
+
+cleanup
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2}
+
+EXPECT "$V0" volinfo_field $V0 'Volume Name'
+EXPECT 'Created' volinfo_field $V0 'Status'
+EXPECT '2' brick_count $V0
+
+TEST $CLI volume set $V0 cluster.nsr on
+TEST $CLI volume set $V0 cluster.nsr.recon on
+
+TEST $CLI volume start $V0
+EXPECT 'Started' volinfo_field $V0 'Status'
+
+## Mount FUSE with caching disabled (read-only)
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0
+
+# Give the bricks a chance to connect to each other.
+EXPECT_WITHIN 10 "2" get_rep_count $M0
+
+TEST ping_file $M0/probe
+TEST cmp ${M0}/probe ${B0}/${V0}1/probe
+TEST cmp ${M0}/probe ${B0}/${V0}2/probe
+
+TEST kill_a_brick
+sleep 10
+
+# Make sure only one copy makes it while degraded.
+TEST ping_file $M0/probe2
+TEST [ $(count_matches probe2) = 1 ]
+
+# Restart the brick and give reconciliation a chance to run.
+TEST $CLI volume start $V0 force
+sleep 10
+
+# Make sure *both* copies are valid after reconciliation.
+TEST [ $(count_matches probe2) = 2 ]
+
+#cleanup
+#killall -9 etcd
diff --git a/xlators/cluster/nsr-server/src/Makefile.am b/xlators/cluster/nsr-server/src/Makefile.am
index 85a560d09..7153352ad 100644
--- a/xlators/cluster/nsr-server/src/Makefile.am
+++ b/xlators/cluster/nsr-server/src/Makefile.am
@@ -25,7 +25,7 @@ noinst_HEADERS = nsr-internal.h etcd-api.h all-templates.c \
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src
+	-I$(top_srcdir)/rpc/rpc-lib/src -DSBIN_DIR=\"$(sbindir)\"
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/cluster/nsr-server/src/all-templates.c b/xlators/cluster/nsr-server/src/all-templates.c
index 3403aed8c..2f0509a6c 100644
--- a/xlators/cluster/nsr-server/src/all-templates.c
+++ b/xlators/cluster/nsr-server/src/all-templates.c
@@ -92,7 +92,7 @@ nsr_$NAME$ (call_frame_t *frame, xlator_t *this,
         }
 
 
-        if (!priv->leader || priv->fence_io) {
+        if (!priv->leader/* || priv->fence_io*/) {
                 op_errno = EREMOTE;
                 goto err;
         }
diff --git a/xlators/cluster/nsr-server/src/nsr.c b/xlators/cluster/nsr-server/src/nsr.c
index 75412a8f5..9f2338148 100644
--- a/xlators/cluster/nsr-server/src/nsr.c
+++ b/xlators/cluster/nsr-server/src/nsr.c
@@ -664,7 +664,7 @@ nsr_init (xlator_t *this)
         if (!stat(priv->vol_file, &buf)) {
 
                 runinit (&runner);
-                runner_add_args(&runner, "/usr/local/sbin/glusterfs",
+                runner_add_args(&runner, SBIN_DIR "/glusterfs",
                         "-f", recon_file,
                         "-p", recon_pid_file,
                         "-l", recon_log,