bring in strict check on export directory being re-used for different volume

Signed-off-by: Amar Tumballi <amar@gluster.com>
Signed-off-by: Anand Avati <avati@gluster.com>

BUG: 3065 (make sure the export directories are not re-used as part of another volume)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=3065

5 files changed, 102 insertions, 5 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 47f2c73c9..5760a4466 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -290,6 +290,8 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
         xlator_t                                *this = NULL;
         glusterd_conf_t                         *priv = NULL;
         char                                    msg[2048] = {0};
+        uuid_t                                  volume_uuid;
+        char                                    *volume_uuid_str;
 
         this = THIS;
         if (!this) {
@@ -329,6 +331,16 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
                 gf_log ("", GF_LOG_ERROR, "Unable to get count");
                 goto out;
         }
+        ret = dict_get_str (dict, "volume-id", &volume_uuid_str);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume id");
+                goto out;
+        }
+        ret = uuid_parse (volume_uuid_str, volume_uuid);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to parse volume id");
+                goto out;
+        }
 
         ret = dict_get_str (dict, "bricks", &bricks);
         if (ret) {
@@ -378,6 +390,7 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
                 if (!uuid_compare (brick_info->uuid, priv->uuid)) {
                         ret = glusterd_brick_create_path (brick_info->hostname,
                                                           brick_info->path,
+                                                          volume_uuid,
                                                           0777, op_errstr);
                         if (ret)
                                 goto out;
@@ -473,6 +486,7 @@ glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
                 if (!uuid_compare (brickinfo->uuid, priv->uuid)) {
                         ret = glusterd_brick_create_path (brickinfo->hostname,
                                                           brickinfo->path,
+                                                          volinfo->volume_id,
                                                           0777, op_errstr);
                         if (ret)
                                 goto out;
@@ -739,6 +753,7 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
                 if (!uuid_compare (brickinfo->uuid, priv->uuid)) {
                         ret = glusterd_brick_create_path (brickinfo->hostname,
                                                           brickinfo->path,
+                                                          volinfo->volume_id,
                                                           0777, op_errstr);
                         if (ret)
                                 goto out;
@@ -1059,7 +1074,9 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                 goto out;
        }
         if (!glusterd_is_local_addr (host)) {
-                ret = glusterd_brick_create_path (host, path, 0777, op_errstr);
+                ret = glusterd_brick_create_path (host, path,
+                                                  volinfo->volume_id, 0777,
+                                                  op_errstr);
                 if (ret)
                         goto out;
         } else {
@@ -1231,7 +1248,8 @@ glusterd_op_stage_log_filename (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        ret = glusterd_brick_create_path (hostname, path, 0777, op_errstr);
+        ret = glusterd_brick_create_path (hostname, path, volinfo->volume_id,
+                                          0777, op_errstr);
         if (ret)
                 goto out;
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 8c042317b..c06b1932f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -2922,12 +2922,14 @@ glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo,
 }
 
 int
-glusterd_brick_create_path (char *host, char *path, mode_t mode,
+glusterd_brick_create_path (char *host, char *path, uuid_t uuid, mode_t mode,
                             char **op_errstr)
 {
         int     ret = -1;
         char    msg[2048] = {0};
         struct  stat st_buf = {0};
+        uuid_t  old_uuid;
+        char    old_uuid_buf[64] = {0,};
 
         ret = stat (path, &st_buf);
         if ((!ret) && (!S_ISDIR (st_buf.st_mode))) {
@@ -2968,6 +2970,37 @@ check_xattr:
                 sys_lremovexattr (path, "trusted.glusterfs.test");
         }
 
+        if (!uuid)
+                goto out;
+
+        /* This 'key' is set when the volume is started for the first time */
+        ret = sys_lgetxattr (path, "trusted.glusterfs.volume-id",
+                             old_uuid, 16);
+        if (ret == 16) {
+                if (uuid_compare (old_uuid, uuid)) {
+                        uuid_utoa_r (old_uuid, old_uuid_buf);
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "%s: mismatching volume-id (%s) recieved. "
+                                "already is a part of volume %s ",
+                                path, uuid_utoa (uuid), old_uuid_buf);
+                        snprintf (msg, sizeof (msg), "'%s:%s' has been part of "
+                                  "a deleted volume with id %s. Please "
+                                  "re-create the brick directory.",
+                                  host, path, old_uuid_buf);
+                        ret = -1;
+                        goto out;
+                }
+                ret = 0;
+        } else if (ret == -1) {
+                /* 'volume-id' not set, seems to be a fresh directory */
+                ret = 0;
+        } else {
+                /* Wrong 'volume-id' is set, it should be error */
+                ret = -1;
+                snprintf (msg, sizeof (msg), "'%s:%s' has wrong entry"
+                          "for 'volume-id'.", host, path);
+        }
+
 out:
         if (msg[0] != '\0')
                 *op_errstr = gf_strdup (msg);
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 380db9057..254557c27 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -232,7 +232,7 @@ int
 glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo,
                           glusterd_brickinfo_t *src_brick, glusterd_brickinfo_t *dst_brick);
 int
-glusterd_brick_create_path (char *host, char *path, mode_t mode,
+glusterd_brick_create_path (char *host, char *path, uuid_t uuid, mode_t mode,
                             char **op_errstr);
 int
 glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 607fe1daf..bd6a79ace 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -1398,6 +1398,11 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (ret)
                 return -1;
 
+        ret = xlator_set_option (xl, "volume-id",
+                                 uuid_utoa (volinfo->volume_id));
+        if (ret)
+                return -1;
+
         xl = volgen_graph_add (graph, "features/access-control", volname);
         if (!xl)
                 return -1;
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 524b700f0..95926bbf5 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -4355,6 +4355,8 @@ init (xlator_t *this)
         int                    ret           = 0;
         int                    op_ret        = -1;
         int32_t                janitor_sleep = 0;
+        uuid_t                 old_uuid;
+        uuid_t                 dict_uuid;
 
         dir_data = dict_get (this->options, "directory");
 
@@ -4389,7 +4391,6 @@ init (xlator_t *this)
                 goto out;
         }
 
-
         /* Check for Extended attribute support, if not present, log it */
         op_ret = sys_lsetxattr (dir_data->data,
                                 "trusted.glusterfs.test", "working", 8, 0);
@@ -4426,6 +4427,46 @@ init (xlator_t *this)
                 }
         }
 
+        tmp_data = dict_get (this->options, "volume-id");
+        if (tmp_data) {
+                op_ret = uuid_parse (tmp_data->data, dict_uuid);
+                if (op_ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "wrong volume-id (%s) set in volume file",
+                                tmp_data->data);
+                        ret = -1;
+                        goto out;
+                }
+                op_ret = sys_lgetxattr (dir_data->data,
+                                        "trusted.glusterfs.volume-id", old_uuid, 16);
+                if (op_ret == 16) {
+                        if (uuid_compare (old_uuid, dict_uuid)) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "mismatching volume-id (%s) recieved. "
+                                        "already is a part of volume %s ",
+                                        tmp_data->data, uuid_utoa (old_uuid));
+                                ret = -1;
+                                goto out;
+                        }
+                } else if (op_ret == -1) {
+                        /* Using the export for first time */
+                        op_ret = sys_lsetxattr (dir_data->data,
+                                                "trusted.glusterfs.volume-id",
+                                                dict_uuid, 16, 0);
+                        if (op_ret == -1) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to set volume id on export");
+                                ret = -1;
+                                goto out;
+                        }
+                } else {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to fetch volume id from export");
+                        goto out;
+                }
+        }
+
         _private = GF_CALLOC (1, sizeof (*_private),
                               gf_posix_mt_posix_private);
         if (!_private) {