dht: make lookup-unhashed=auto do something actually useful

The key concept here is to determine whether a directory is "clean" by
comparing its last-known-good topology to the current one for the
volume.  These are stored as "commit hashes" on the directory and the
volume root respectively.  The volume's commit hash changes whenever a
brick is added or removed, and a fix-layout is done.  A directory's
commit hash changes only when a full rebalance (not just fix-layout)
is done on it.  If all bricks are present and have a directory
commit hash that matches the volume commit hash, then we can assume
that every file is in its "proper" place. Therefore, if we look for
a file in that proper place and don't find it, we can assume it's not
on any other subvolume and *safely* skip the global (broadcast to all)
lookup.

Change-Id: Id6ce4593ba1f7daffa74cfab591cb45960629ae3
BUG: 1220064
Reviewed-on-master: http://review.gluster.org/#/c/7702/
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Signed-off-by: Shyam <srangana@redhat.com>
Reviewed-on: http://review.gluster.org/10729
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 78 insertions, 9 deletions

diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 6c0afdbec90..37e07ad77da 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -210,6 +210,7 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
         int              ret = -1;
         dht_layout_t    *layout = NULL;
         dht_conf_t      *conf = NULL;
+        uint32_t         vol_commit_hash = 0;
 
         local = discover_frame->local;
         layout = local->layout;
@@ -279,6 +280,15 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
                         dht_layout_set (this, local->inode, layout);
         }
 
+        if (!conf->vch_forced) {
+                ret = dict_get_uint32 (local->xattr,
+                                       conf->commithash_xattr_name,
+                                       &vol_commit_hash);
+                if (ret == 0) {
+                        conf->vol_commit_hash = vol_commit_hash;
+                }
+        }
+
         DHT_STACK_UNWIND (lookup, main_frame, local->op_ret, local->op_errno,
                           local->inode, &local->stbuf, local->xattr,
                           &local->postparent);
@@ -459,6 +469,12 @@ dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
                         "%s: Failed to set dictionary value:key = %s",
                         loc->path, conf->link_xattr_name);
 
+        if (__is_root_gfid(local->loc.gfid)) {
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->commithash_xattr_name,
+                                       sizeof(uint32_t));
+        }
+
         call_cnt        = conf->subvolume_cnt;
         local->call_cnt = call_cnt;
 
@@ -655,6 +671,7 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         call_frame_t *copy          = NULL;
         dht_local_t  *copy_local    = NULL;
         char gfid[GF_UUID_BUF_SIZE] = {0};
+        uint32_t      vol_commit_hash = 0;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, err);
@@ -667,6 +684,14 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (!conf)
                 goto out;
 
+        if (!conf->vch_forced) {
+                ret = dict_get_uint32 (xattr, conf->commithash_xattr_name,
+                                       &vol_commit_hash);
+                if (ret == 0) {
+                        conf->vol_commit_hash = vol_commit_hash;
+                }
+        }
+
         gf_uuid_unparse (local->loc.gfid, gfid);
 
         LOCK (&frame->lock);
@@ -1852,6 +1877,7 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         call_frame_t *prev          = NULL;
         int           ret           = 0;
         dht_layout_t *parent_layout = NULL;
+        uint32_t      vol_commit_hash = 0;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -1875,6 +1901,14 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       "fresh_lookup returned for %s with op_ret %d and "
                       "op_errno %d", loc->path, op_ret, op_errno);
 
+        if (!conf->vch_forced) {
+                ret = dict_get_uint32 (xattr, conf->commithash_xattr_name,
+                                       &vol_commit_hash);
+                if (ret == 0) {
+                        conf->vol_commit_hash = vol_commit_hash;
+                }
+        }
+
         if (ENTRY_MISSING (op_ret, op_errno)) {
                 gf_msg_debug (this->name, 0,
                               "Entry %s missing on subvol %s",
@@ -1891,7 +1925,10 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                                         &parent_layout);
                         if (ret || !parent_layout)
                                 goto out;
-                        if (parent_layout->search_unhashed) {
+                        if (parent_layout->commit_hash
+                                  != conf->vol_commit_hash) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "hashes don't match, do global lookup");
                                 local->op_errno = ENOENT;
                                 dht_lookup_everywhere (frame, this, loc);
                                 return 0;
@@ -2078,6 +2115,12 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
+        if (__is_root_gfid(loc->gfid)) {
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->commithash_xattr_name,
+                                       sizeof(uint32_t));
+        }
+
         if (!hashed_subvol)
                 hashed_subvol = dht_subvol_get_hashed (this, loc);
         local->hashed_subvol = hashed_subvol;
@@ -3238,8 +3281,9 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
-        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
-                                   op_errno, err);
+        if (!conf->defrag)
+                GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                           op_errno, err);
 
         local = dht_local_init (frame, NULL, fd, GF_FOP_FSETXATTR);
         if (!local) {
@@ -3338,6 +3382,7 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
         char          value[4096] = {0,};
         gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
         int           call_cnt = 0;
+        uint32_t      new_hash = 0;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -3350,8 +3395,10 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
         methods = conf->methods;
         GF_VALIDATE_OR_GOTO (this->name, conf->methods, err);
 
-        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
-                                   op_errno, err);
+        /* Rebalance daemon is allowed to set internal keys */
+        if (!conf->defrag)
+                GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                           op_errno, err);
 
         local = dht_local_init (frame, loc, NULL, GF_FOP_SETXATTR);
         if (!local) {
@@ -3489,6 +3536,22 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_INFO,
                         "fixing the layout of %s", loc->path);
 
+                ret = dict_get_uint32(xattr, "new-commit-hash", &new_hash);
+                if (ret == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "updating commit hash for %s from %u to %u",
+                                uuid_utoa(loc->gfid),
+                                layout->commit_hash, new_hash);
+                        layout->commit_hash = new_hash;
+
+                        ret = dht_update_commit_hash_for_layout (frame);
+                        if (ret) {
+                                op_errno = ENOTCONN;
+                                goto err;
+                        }
+                        return ret;
+                }
+
                 ret = dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
                                                 layout);
                 if (ret) {
@@ -5377,6 +5440,8 @@ dht_mkdir (call_frame_t *frame, xlator_t *this,
                 goto err;
         }
 
+        local->layout->commit_hash = conf->vol_commit_hash;
+
         STACK_WIND (frame, dht_mkdir_hashed_cbk,
                     hashed_subvol,
                     hashed_subvol->fops->mkdir,
@@ -6570,10 +6635,12 @@ dht_log_new_layout_for_dir_selfheal (xlator_t *this, loc_t *loc,
 
                 ret  = snprintf (string, max_string_len,
                                  "[Subvol_name: %s, Err: %d , Start: "
-                                 "%"PRIu32 " , Stop: %"PRIu32 " ], ",
+                                 "%"PRIu32 " , Stop: %"PRIu32 " , Hash: %"
+                                 PRIu32 " ], ",
                                  layout->list[i].xlator->name,
                                  layout->list[i].err, layout->list[i].start,
-                                 layout->list[i].stop);
+                                 layout->list[i].stop,
+                                 layout->list[i].commit_hash);
 
                 if (ret < 0)
                         return;
@@ -6602,10 +6669,12 @@ dht_log_new_layout_for_dir_selfheal (xlator_t *this, loc_t *loc,
 
                 ret  =  snprintf (output_string + off, len - off,
                                   "[Subvol_name: %s, Err: %d , Start: "
-                                  "%"PRIu32 " , Stop: %"PRIu32 " ], ",
+                                  "%"PRIu32 " , Stop: %"PRIu32 " , Hash: %"
+                                  PRIu32  " ], ",
                                   layout->list[i].xlator->name,
                                   layout->list[i].err, layout->list[i].start,
-                                  layout->list[i].stop);
+                                  layout->list[i].stop,
+                                  layout->list[i].commit_hash);
 
                 if (ret < 0)
                         goto err;