summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJeff Darcy <jdarcy@redhat.com>2014-03-25 16:48:37 +0000
committerJeff Darcy <jdarcy@redhat.com>2014-03-25 16:48:37 +0000
commit927483b6fbf2a8e731f9e34dd70dc4179accc079 (patch)
tree4beef24ea3fdeeb402b8b7ff1874e79a09a5bbcc
parentefbb23837761bda6c526baca1b5ea72d227e2ae3 (diff)
nsr: use different etcd keys for each NSR subvolume
This is necessary both for separate volumes using NSR and for DHT volumes composed of multiple NSR subvolumes. Change-Id: Ia269d70b535cc26900f8b6e7f22706087746fbe7 Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
-rwxr-xr-xtests/basic/four-brick.t85
-rw-r--r--xlators/cluster/nsr-recon/src/recon_driver.c1
-rw-r--r--xlators/cluster/nsr-server/src/leader.c12
-rw-r--r--xlators/cluster/nsr-server/src/nsr-internal.h5
-rw-r--r--xlators/cluster/nsr-server/src/nsr.c51
-rw-r--r--xlators/cluster/nsr-server/src/recon_notify.c2
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-volgen.c20
-rw-r--r--xlators/mgmt/glusterd/src/glusterd.h3
8 files changed, 156 insertions, 23 deletions
diff --git a/tests/basic/four-brick.t b/tests/basic/four-brick.t
new file mode 100755
index 000000000..a8d9cd400
--- /dev/null
+++ b/tests/basic/four-brick.t
@@ -0,0 +1,85 @@
+#!/bin/bash
+
+# Test *very basic* NSR functionality - startup, mount, simplest possible file
+# write.
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+function get_rep_count {
+ v=$(getfattr --only-values -e text -n trusted.nsr.rep-count $1 2> /dev/null)
+ #echo $v > /dev/tty
+ echo $v
+}
+
+function ping_file {
+ dd if=/dev/urandom of=$1 bs=4k count=100 2> /dev/null
+}
+
+function kill_brick {
+ bpid=$(cat /var/lib/glusterd/vols/${V0}/run/*-${V0}${1}.pid)
+ rpid=$(cat /var/lib/glusterd/vols/${V0}/run/*-${V0}${1}-recon.pid)
+ echo "brick PID = $bpid" > /dev/tty
+ echo "recon PID = $rpid" > /dev/tty
+ kill -9 $bpid $rpid
+}
+
+function count_matches {
+ n=0
+ for f in $B0/$V0[1234]/$1; do
+ cmp $M0/$1 $f 2> /dev/null
+ if [ $? = 0 ]; then
+ n=$((n+1))
+ fi
+ done
+ echo $n
+}
+
+cleanup
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+
+TEST mkdir -p ${V0}{1,2,3,4}
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4}
+
+EXPECT "$V0" volinfo_field $V0 'Volume Name'
+EXPECT 'Created' volinfo_field $V0 'Status'
+EXPECT '4' brick_count $V0
+
+TEST $CLI volume set $V0 cluster.nsr on
+TEST $CLI volume set $V0 cluster.nsr.recon on
+
+TEST $CLI volume start $V0
+EXPECT 'Started' volinfo_field $V0 'Status'
+
+## Mount FUSE with caching disabled (read-only)
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0
+
+# Give the bricks a chance to connect to each other.
+EXPECT_WITHIN 10 "2" get_rep_count $M0
+
+TEST ping_file $M0/probe
+TEST cmp ${M0}/probe ${B0}/${V0}1/probe
+TEST cmp ${M0}/probe ${B0}/${V0}2/probe
+
+# Kill one brick from each pair.
+TEST kill_brick 1
+TEST kill_brick 3
+sleep 10
+
+# Make sure only one copy makes it while degraded.
+TEST ping_file $M0/probe2
+TEST [ $(count_matches probe2) = 1 ]
+
+# Restart the brick and give reconciliation a chance to run.
+# TBD: figure out why reconciliation takes so $#@! long to run
+TEST $CLI volume start $V0 force
+sleep 20
+
+# Make sure *both* copies are valid after reconciliation.
+TEST [ $(count_matches probe2) = 2 ]
+
+cleanup
+#killall -9 etcd
diff --git a/xlators/cluster/nsr-recon/src/recon_driver.c b/xlators/cluster/nsr-recon/src/recon_driver.c
index b02abf2bb..e0327f81d 100644
--- a/xlators/cluster/nsr-recon/src/recon_driver.c
+++ b/xlators/cluster/nsr-recon/src/recon_driver.c
@@ -3066,6 +3066,7 @@ nsr_reconciliation_driver(void *arg)
nsr_driver_log (this->name, GF_LOG_INFO,
"finished recon work as joiner \n");
+ break;
default:
nsr_driver_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/cluster/nsr-server/src/leader.c b/xlators/cluster/nsr-server/src/leader.c
index 944c85cdc..02a2609c8 100644
--- a/xlators/cluster/nsr-server/src/leader.c
+++ b/xlators/cluster/nsr-server/src/leader.c
@@ -30,7 +30,6 @@
#include "../../nsr-recon/src/recon_driver.h"
#include "../../nsr-recon/src/recon_xlator.h"
-#define NSR_KEY "xyzzy"
#define NSR_TTL 5
static void
@@ -39,20 +38,19 @@ nsr_set_leader (xlator_t *this, etcd_session etcd)
long term = 0;
etcd_result res;
nsr_private_t *priv = this->private;
- char *term_key = priv->term_uuid;
char n_t[sizeof(long)+1];
char *text = NULL;
gf_log (this->name, GF_LOG_INFO, "Just became leader");
- text = etcd_get(etcd, priv->term_uuid);
+ text = etcd_get(etcd, priv->term_key);
if(text == NULL) {
term = 0;
} else {
term = strtol(text, NULL, 10);
}
sprintf(n_t,"%ld",term+1);
- res = etcd_set(etcd, term_key,n_t,text,0);
+ res = etcd_set(etcd, priv->term_key,n_t,text,0);
if(res != ETCD_OK) {
gf_log (this->name, GF_LOG_ERROR, "failed to set term");
return;
@@ -84,7 +82,7 @@ nsr_leader_thread (void *arg)
char *index_out = NULL;
gf_log (this->name, GF_LOG_INFO,
- "calling glfs_open_str on servers %s", priv->etcd_servers);
+ "calling etcd_open_str on servers %s", priv->etcd_servers);
priv->etcd = etcd_open_str(priv->etcd_servers);
if (!(priv->etcd)) {
@@ -98,7 +96,7 @@ nsr_leader_thread (void *arg)
for (;;) {
/* Not leader yet. Try to become leader. */
for (;;) {
- res = etcd_lock (priv->etcd, NSR_KEY, NSR_TTL,
+ res = etcd_lock (priv->etcd, priv->leader_key, NSR_TTL,
index_in, &index_out);
if (res == ETCD_OK) {
break;
@@ -113,7 +111,7 @@ nsr_leader_thread (void *arg)
index_in = index_out;
index_out = NULL;
for (;;) {
- res = etcd_lock (priv->etcd, NSR_KEY, NSR_TTL,
+ res = etcd_lock (priv->etcd, priv->leader_key, NSR_TTL,
index_in, &index_out);
if (index_out && (index_in != index_out)) {
if (index_in) {
diff --git a/xlators/cluster/nsr-server/src/nsr-internal.h b/xlators/cluster/nsr-server/src/nsr-internal.h
index 7825973f7..fc612c136 100644
--- a/xlators/cluster/nsr-server/src/nsr-internal.h
+++ b/xlators/cluster/nsr-server/src/nsr-internal.h
@@ -35,8 +35,9 @@ typedef struct _nsr_recon_notify_ev_s {
typedef struct {
char *etcd_servers;
- char *vol_uuid;
- char *term_uuid;
+ char *subvol_uuid;
+ char *leader_key;
+ char *term_key;
char *brick_uuid;
gf_boolean_t leader;
uint8_t up_children;
diff --git a/xlators/cluster/nsr-server/src/nsr.c b/xlators/cluster/nsr-server/src/nsr.c
index 214c54f18..eda9e555a 100644
--- a/xlators/cluster/nsr-server/src/nsr.c
+++ b/xlators/cluster/nsr-server/src/nsr.c
@@ -500,6 +500,25 @@ out:
extern void *nsr_leader_thread (void *);
+void
+nsr_deallocate_priv (nsr_private_t *priv)
+{
+ if (!priv) {
+ return;
+ }
+
+ if (priv->leader_key) {
+ GF_FREE(priv->leader_key);
+ }
+
+ if (priv->term_key) {
+ GF_FREE(priv->term_key);
+ }
+
+ GF_FREE(priv);
+}
+
+
int32_t
nsr_init (xlator_t *this)
{
@@ -571,12 +590,22 @@ nsr_init (xlator_t *this)
gf_log (this->name, GF_LOG_ERROR, "etcd servers not generated. ???");
goto err;
}
- priv->vol_uuid = "temporary";
+
+ GF_OPTION_INIT ("subvol-uuid", priv->subvol_uuid, str, err);
+ gf_log (this->name, GF_LOG_INFO, "subvol_uuid = %s", priv->subvol_uuid);
+ if (gf_asprintf(&priv->leader_key,"%s:leader",priv->subvol_uuid) <= 0) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "could not generate leader key");
+ goto err;
+ }
+ if (gf_asprintf(&priv->term_key,"%s:term",priv->subvol_uuid) <= 0) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "could not generate term key");
+ goto err;
+ }
uuid_generate(tmp_uuid);
priv->brick_uuid = strdup(uuid_utoa(tmp_uuid));
- priv->term_uuid = "nsr-term";
- gf_log (this->name, GF_LOG_INFO,
- "brick_uuid = %s\n", priv->brick_uuid);
+ gf_log (this->name, GF_LOG_INFO, "brick_uuid = %s\n", priv->brick_uuid);
GF_OPTION_INIT ("my-name", my_name, str, err);
if (!my_name) {
@@ -707,7 +736,10 @@ nsr_init (xlator_t *this)
sleep(1);
}
- (void)pthread_create(&kid,NULL,nsr_leader_thread,this);
+ if (pthread_create(&kid,NULL,nsr_leader_thread,this) != 0) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "failed to start leader thread");
+ }
while (priv->leader_inited == 0) {
sleep(1);
}
@@ -726,9 +758,7 @@ nsr_init (xlator_t *this)
return 0;
err:
- if (priv) {
- GF_FREE(priv);
- }
+ nsr_deallocate_priv(priv);
return -1;
}
@@ -736,6 +766,7 @@ err:
void
nsr_fini (xlator_t *this)
{
+ nsr_deallocate_priv(this->private);
}
class_methods_t class_methods = {
@@ -765,5 +796,9 @@ struct volume_options options[] = {
.type = GF_OPTION_TYPE_STR,
.description = "list of comma seperated etc servers"
},
+ { .key = {"subvol-uuid"},
+ .type = GF_OPTION_TYPE_STR,
+ .description = "UUID for this NSR (sub)volume"
+ },
{ .key = {NULL} },
};
diff --git a/xlators/cluster/nsr-server/src/recon_notify.c b/xlators/cluster/nsr-server/src/recon_notify.c
index 7397192ae..24f7cf2de 100644
--- a/xlators/cluster/nsr-server/src/recon_notify.c
+++ b/xlators/cluster/nsr-server/src/recon_notify.c
@@ -234,7 +234,7 @@ nsr_setup_recon (xlator_t *this)
if (priv->nsr_recon_start == _gf_false)
return 0;
- ctx->fs = glfs_new(priv->vol_uuid);
+ ctx->fs = glfs_new(priv->subvol_uuid);
if (!ctx->fs) {
ret = 1;
gf_log (this->name, GF_LOG_ERROR, "failed to initialise glfs \n");
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index aee655733..8a94309ed 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -1431,9 +1431,14 @@ assign_groups (glusterd_volinfo_t *volinfo)
glusterd_brickinfo_t *brickinfo = NULL;
uint16_t group_num = 0;
int in_group = 0;
+ uuid_t tmp_uuid;
list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+ if (in_group == 0) {
+ uuid_generate(tmp_uuid);
+ }
brickinfo->group = group_num;
+ uuid_copy(brickinfo->nsr_uuid,tmp_uuid);
if (++in_group >= volinfo->replica_count) {
in_group = 0;
++group_num;
@@ -1469,7 +1474,6 @@ add_nsr_stuff (volgen_graph_t *graph, char *volname,
gf_boolean_t enable_recon = _gf_false;
static uint32_t nsr_port = 27000;
-
if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr.recon") > 0) {
enable_recon = _gf_true;
}
@@ -1520,6 +1524,10 @@ add_nsr_stuff (volgen_graph_t *graph, char *volname,
return -1;
if (xlator_set_option(me,"leader",leader_opt))
return -1;
+ if (xlator_set_option(me,"subvol-uuid",
+ uuid_utoa(brickinfo->nsr_uuid))) {
+ return -1;
+ }
#define FILL_REMOTE_NAMES { \
strcat(remote_names, \
@@ -1586,7 +1594,7 @@ add_nsr_stuff (volgen_graph_t *graph, char *volname,
if (enable_recon == _gf_false)
return 0;
- /* Now fill in the various files required for reeconciliation */
+ /* Now fill in the various files required for reconciliation */
snprintf (filepath, PATH_MAX,
"%s-nsr-recon.vol",
dst);
@@ -1898,8 +1906,8 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
/* TBD: conditionalize on NSR being enabled */
if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr") > 0) {
- assign_groups(volinfo);
- ret = add_nsr_stuff (graph, volname, brickinfo, volinfo, changelog_basepath);
+ ret = add_nsr_stuff (graph, volname, brickinfo, volinfo,
+ changelog_basepath);
if (ret) {
return -1;
}
@@ -3774,6 +3782,10 @@ generate_brick_volfiles (glusterd_volinfo_t *volinfo)
}
}
+ if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr") > 0) {
+ assign_groups(volinfo);
+ }
+
list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
gf_log ("", GF_LOG_DEBUG,
"Found a brick - %s:%s", brickinfo->hostname,
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 2b9e0787d..3d810ed97 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -198,7 +198,8 @@ struct glusterd_brickinfo {
* more efficient too, though it would require some further adaptation
* to support more than one layer of hierarchy.
*/
- uint16_t group;
+ uint16_t group;
+ uuid_t nsr_uuid;
};
typedef struct glusterd_brickinfo glusterd_brickinfo_t;