Disable brick daemon from incorrect brick directory

Summary:
Currently the bricks can open any mount directory from the given volume. This patch adds a provision to prevent
bricks from opening brick directories that aren't created for them. This will help with operating gluster on large
scale.

We add a new xattr GF_XATTR_BRICK_NAME to the brick directory. When we start a brick daemon, we make sure the path on
disk matches with the config provided. For backward compatibility, we ignore if there is no value for
GF_XATTR_BRICK_NAME and set the current brick daemon's path as value.

We ignore GF_XATTR_BRICK_NAME during healing and reset GF_XATTR_BRICK_NAME on brick replace.

Test Plan: Run fb-smoke

Reviewers: jdarcy, sshreyas

Reviewed By: sshreyas

Differential Revision: https://phabricator.intern.facebook.com/D5448921

Porting note: disabled some checks to deal with the snapshot case

Change-Id: I98e62033dfd07f30ad3b99ac003ce94c8d935e5f
Signed-off-by: Jeff Darcy <jdarcy@fb.com>
Reviewed-on: https://review.gluster.org/18275
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>
Tested-by: Jeff Darcy <jeff@pl.atyp.us>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>

3 files changed, 60 insertions, 0 deletions

diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index dbb33812287..d4ddff2090e 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -4578,3 +4578,59 @@ err:
 out:
         return ret;
 }
+
+
+int
+gf_compare_or_fix_xattr_brick_path (const char* path)
+{
+        xlator_t* this = NULL;
+        char brick_path_ondisk[PATH_MAX] = "";
+        int ret = -1;
+
+        this = THIS;
+
+        GF_ASSERT (path);
+
+        /* Check for brickinfo->path to GF_XATTR_BRICK_PATH xattr
+         * If present compare to brickinfo->path
+         * If not present set GF_XATTR_BRICK_PATH to brickinfo->path
+         */
+         ret = sys_lgetxattr (path, GF_XATTR_BRICK_PATH, brick_path_ondisk,
+                              min (strlen (path), PATH_MAX));
+
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING, "Failed to get "
+                        "extended attribute %s for brick dir %s. "
+                        "Reason : %s. Check ignored.", GF_XATTR_BRICK_PATH,
+                        path, strerror (errno));
+
+                if (errno == ENODATA) {
+                        ret = sys_lsetxattr (path, GF_XATTR_BRICK_PATH,
+                                             path, strlen (path), XATTR_CREATE);
+
+                        if (ret == -1) {
+                                gf_log (this->name, GF_LOG_ERROR, "Error "
+                                        "setting %s xattr. Reason: %s",
+                                        GF_XATTR_BRICK_PATH, strerror (errno));
+                        }
+
+                        ret = 0;
+                        goto out;
+                }
+
+                ret = -1;
+                goto out;
+        }
+
+        GF_ASSERT (ret >= 0);
+        ret = strcmp (brick_path_ondisk, path);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Brick path mismatch for brick " "%s. Found %s",
+                        path, brick_path_ondisk);
+        }
+
+out:
+        return ret;
+}
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index ae96c9bc1a1..d917671fedf 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -849,4 +849,7 @@ gf_bits_index (uint64_t n);
 int
 gf_peerinfo_to_hostname_and_port (const char *peerinfo, char **hostname);
 
+int
+gf_compare_or_fix_xattr_brick_path (const char* path);
+
 #endif /* _COMMON_UTILS_H */
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index dbe18f2c243..0b033d8bfcf 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -85,6 +85,7 @@
 #define GF_XATTR_NODE_UUID_KEY  "trusted.glusterfs.node-uuid"
 #define GF_REBAL_FIND_LOCAL_SUBVOL "glusterfs.find-local-subvol"
 #define GF_XATTR_VOL_ID_KEY   "trusted.glusterfs.volume-id"
+#define GF_XATTR_BRICK_PATH   "trusted.glusterfs.brick-path"
 #define GF_XATTR_LOCKINFO_KEY   "trusted.glusterfs.lockinfo"
 #define GF_META_LOCK_KEY        "glusterfs.lock-migration-meta-lock"
 #define GF_META_UNLOCK_KEY      "glusterfs.lock-migration-meta-unlock"