Merge "nsr: use different etcd keys for each NSR subvolume"

7 files changed, 71 insertions, 23 deletions

diff --git a/xlators/cluster/nsr-recon/src/recon_driver.c b/xlators/cluster/nsr-recon/src/recon_driver.c
index b02abf2bb..e0327f81d 100644
--- a/xlators/cluster/nsr-recon/src/recon_driver.c
+++ b/xlators/cluster/nsr-recon/src/recon_driver.c
@@ -3066,6 +3066,7 @@ nsr_reconciliation_driver(void *arg)
 
                         nsr_driver_log (this->name, GF_LOG_INFO,
                                         "finished recon work as joiner \n");
+                        break;
 
                 default:
                         nsr_driver_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/cluster/nsr-server/src/leader.c b/xlators/cluster/nsr-server/src/leader.c
index 944c85cdc..02a2609c8 100644
--- a/xlators/cluster/nsr-server/src/leader.c
+++ b/xlators/cluster/nsr-server/src/leader.c
@@ -30,7 +30,6 @@
 #include "../../nsr-recon/src/recon_driver.h"
 #include "../../nsr-recon/src/recon_xlator.h"
 
-#define NSR_KEY         "xyzzy"
 #define NSR_TTL         5
 
 static void
@@ -39,20 +38,19 @@ nsr_set_leader (xlator_t *this, etcd_session etcd)
         long            term = 0;
         etcd_result     res;
         nsr_private_t   *priv   = this->private;
-        char            *term_key = priv->term_uuid;
         char             n_t[sizeof(long)+1];
         char            *text   = NULL;
 
         gf_log (this->name, GF_LOG_INFO, "Just became leader");
 
-        text = etcd_get(etcd, priv->term_uuid);
+        text = etcd_get(etcd, priv->term_key);
         if(text == NULL) {
                 term = 0;
         } else {
                 term = strtol(text, NULL, 10);
         }
         sprintf(n_t,"%ld",term+1);
-        res = etcd_set(etcd, term_key,n_t,text,0);
+        res = etcd_set(etcd, priv->term_key,n_t,text,0);
         if(res != ETCD_OK) {
 		gf_log (this->name, GF_LOG_ERROR, "failed to set term");
 		return;
@@ -84,7 +82,7 @@ nsr_leader_thread (void *arg)
         char            *index_out      = NULL;
 
         gf_log (this->name, GF_LOG_INFO,
-                "calling glfs_open_str on servers %s", priv->etcd_servers);
+                "calling etcd_open_str on servers %s", priv->etcd_servers);
 
         priv->etcd = etcd_open_str(priv->etcd_servers);
         if (!(priv->etcd)) {
@@ -98,7 +96,7 @@ nsr_leader_thread (void *arg)
         for (;;) {
                 /* Not leader yet.  Try to become leader. */
                 for (;;) {
-                        res = etcd_lock (priv->etcd, NSR_KEY, NSR_TTL,
+                        res = etcd_lock (priv->etcd, priv->leader_key, NSR_TTL,
                                          index_in, &index_out);
                         if (res == ETCD_OK) {
                                 break;
@@ -113,7 +111,7 @@ nsr_leader_thread (void *arg)
                 index_in = index_out;
                 index_out = NULL;
                 for (;;) {
-                        res = etcd_lock (priv->etcd, NSR_KEY, NSR_TTL,
+                        res = etcd_lock (priv->etcd, priv->leader_key, NSR_TTL,
                                          index_in, &index_out);
                         if (index_out && (index_in != index_out)) {
                                 if (index_in) {
diff --git a/xlators/cluster/nsr-server/src/nsr-internal.h b/xlators/cluster/nsr-server/src/nsr-internal.h
index 7825973f7..fc612c136 100644
--- a/xlators/cluster/nsr-server/src/nsr-internal.h
+++ b/xlators/cluster/nsr-server/src/nsr-internal.h
@@ -35,8 +35,9 @@ typedef struct _nsr_recon_notify_ev_s {
 
 typedef struct {
         char                    *etcd_servers;
-        char                    *vol_uuid;
-        char                    *term_uuid;
+        char                    *subvol_uuid;
+        char                    *leader_key;
+        char                    *term_key;
         char                    *brick_uuid;
         gf_boolean_t            leader;
         uint8_t                 up_children;
diff --git a/xlators/cluster/nsr-server/src/nsr.c b/xlators/cluster/nsr-server/src/nsr.c
index 214c54f18..eda9e555a 100644
--- a/xlators/cluster/nsr-server/src/nsr.c
+++ b/xlators/cluster/nsr-server/src/nsr.c
@@ -500,6 +500,25 @@ out:
 
 extern void *nsr_leader_thread (void *);
 
+void
+nsr_deallocate_priv (nsr_private_t *priv)
+{
+        if (!priv) {
+                return;
+        }
+
+        if (priv->leader_key) {
+                GF_FREE(priv->leader_key);
+        }
+
+        if (priv->term_key) {
+                GF_FREE(priv->term_key);
+        }
+
+        GF_FREE(priv);
+}
+
+
 int32_t
 nsr_init (xlator_t *this)
 {
@@ -571,12 +590,22 @@ nsr_init (xlator_t *this)
                 gf_log (this->name, GF_LOG_ERROR, "etcd servers not generated. ???");
                 goto err;
         }
-        priv->vol_uuid = "temporary";
+
+        GF_OPTION_INIT ("subvol-uuid", priv->subvol_uuid, str, err);
+        gf_log (this->name, GF_LOG_INFO, "subvol_uuid = %s", priv->subvol_uuid);
+        if (gf_asprintf(&priv->leader_key,"%s:leader",priv->subvol_uuid) <= 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not generate leader key");
+                goto err;
+        }
+        if (gf_asprintf(&priv->term_key,"%s:term",priv->subvol_uuid) <= 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not generate term key");
+                goto err;
+        }
         uuid_generate(tmp_uuid);
         priv->brick_uuid = strdup(uuid_utoa(tmp_uuid));
-        priv->term_uuid = "nsr-term";
-        gf_log (this->name, GF_LOG_INFO,
-                "brick_uuid = %s\n", priv->brick_uuid);
+        gf_log (this->name, GF_LOG_INFO, "brick_uuid = %s\n", priv->brick_uuid);
 
         GF_OPTION_INIT ("my-name", my_name, str, err);
         if (!my_name) {
@@ -707,7 +736,10 @@ nsr_init (xlator_t *this)
                   sleep(1);
         }
 
-        (void)pthread_create(&kid,NULL,nsr_leader_thread,this);
+        if (pthread_create(&kid,NULL,nsr_leader_thread,this) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to start leader thread");
+        }
         while (priv->leader_inited == 0) {
                   sleep(1);
         }
@@ -726,9 +758,7 @@ nsr_init (xlator_t *this)
 	return 0;
 
 err:
-        if (priv) {
-                GF_FREE(priv);
-        }
+        nsr_deallocate_priv(priv);
         return -1;
 }
 
@@ -736,6 +766,7 @@ err:
 void
 nsr_fini (xlator_t *this)
 {
+        nsr_deallocate_priv(this->private);
 }
 
 class_methods_t class_methods = {
@@ -765,5 +796,9 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_STR,
           .description = "list of comma seperated etc servers"
         },
+        { .key = {"subvol-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "UUID for this NSR (sub)volume"
+        },
 	{ .key = {NULL} },
 };
diff --git a/xlators/cluster/nsr-server/src/recon_notify.c b/xlators/cluster/nsr-server/src/recon_notify.c
index 7397192ae..24f7cf2de 100644
--- a/xlators/cluster/nsr-server/src/recon_notify.c
+++ b/xlators/cluster/nsr-server/src/recon_notify.c
@@ -234,7 +234,7 @@ nsr_setup_recon (xlator_t *this)
         if (priv->nsr_recon_start == _gf_false)
                 return 0;
 
-        ctx->fs = glfs_new(priv->vol_uuid);
+        ctx->fs = glfs_new(priv->subvol_uuid);
         if (!ctx->fs) {
                 ret = 1;
                 gf_log (this->name, GF_LOG_ERROR, "failed to initialise glfs \n");
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index aee655733..8a94309ed 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -1431,9 +1431,14 @@ assign_groups (glusterd_volinfo_t *volinfo)
         glusterd_brickinfo_t    *brickinfo      = NULL;
         uint16_t                group_num       = 0;
         int                     in_group        = 0;
+        uuid_t                  tmp_uuid;
 
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (in_group == 0) {
+                        uuid_generate(tmp_uuid);
+                }
                 brickinfo->group = group_num;
+                uuid_copy(brickinfo->nsr_uuid,tmp_uuid);
                 if (++in_group >= volinfo->replica_count) {
                         in_group = 0;
                         ++group_num;
@@ -1469,7 +1474,6 @@ add_nsr_stuff (volgen_graph_t *graph, char *volname,
         gf_boolean_t            enable_recon = _gf_false;
         static uint32_t         nsr_port = 27000;
 
-
         if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr.recon") > 0) {
                 enable_recon = _gf_true;
         }
@@ -1520,6 +1524,10 @@ add_nsr_stuff (volgen_graph_t *graph, char *volname,
                 return -1;
         if (xlator_set_option(me,"leader",leader_opt))
                 return -1;
+        if (xlator_set_option(me,"subvol-uuid",
+                              uuid_utoa(brickinfo->nsr_uuid))) {
+                return -1;
+        }
 
 #define FILL_REMOTE_NAMES { \
                 strcat(remote_names, \
@@ -1586,7 +1594,7 @@ add_nsr_stuff (volgen_graph_t *graph, char *volname,
         if (enable_recon == _gf_false)
                 return 0;
 
-        /* Now fill in the various files required for reeconciliation */
+        /* Now fill in the various files required for reconciliation */
         snprintf (filepath, PATH_MAX,
                   "%s-nsr-recon.vol",
                   dst);
@@ -1898,8 +1906,8 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
 
         /* TBD: conditionalize on NSR being enabled */
         if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr") > 0) {
-                assign_groups(volinfo);
-                ret = add_nsr_stuff (graph, volname, brickinfo, volinfo, changelog_basepath);
+                ret = add_nsr_stuff (graph, volname, brickinfo, volinfo,
+                                     changelog_basepath);
                 if (ret) {
                         return -1;
                 }
@@ -3774,6 +3782,10 @@ generate_brick_volfiles (glusterd_volinfo_t *volinfo)
                 }
         }
 
+        if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr") > 0) {
+                assign_groups(volinfo);
+        }
+
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 gf_log ("", GF_LOG_DEBUG,
                         "Found a brick - %s:%s", brickinfo->hostname,
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 2b9e0787d..3d810ed97 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -198,7 +198,8 @@ struct glusterd_brickinfo {
          * more efficient too, though it would require some further adaptation
          * to support more than one layer of hierarchy.
          */
-        uint16_t                group;
+        uint16_t           group;
+        uuid_t             nsr_uuid;
 };
 
 typedef struct glusterd_brickinfo glusterd_brickinfo_t;