From b5de382afa8c5777e455c7a376fc4f1f01d782d1 Mon Sep 17 00:00:00 2001 From: Mohammed Rafi KC Date: Mon, 30 Nov 2015 19:02:54 +0530 Subject: tier:unlink during migration files deleted during promotion were not deleting as the files are moving from hashed to non-hashed. On deleting a file that is undergoing promotion, the unlink call is not sent to the dst file as the hashed subvol == cached subvol. This causes the file to reappear once the migration is complete. This patch also fixes a problem with stale linkfile deleting. Change-Id: I4b02a498218c9d8eeaa4556fa4219e91e7fa71e5 BUG: 1282390 Signed-off-by: Mohammed Rafi KC Reviewed-on: http://review.gluster.org/12829 Tested-by: NetBSD Build System Tested-by: Gluster Build System Reviewed-by: Dan Lambright Tested-by: Dan Lambright --- xlators/cluster/dht/src/dht-common.c | 151 ++++++++--------- xlators/cluster/dht/src/dht-common.h | 3 + xlators/cluster/dht/src/tier-common.c | 296 ++++++++++++++++++++++++++++++++++ xlators/cluster/dht/src/tier-common.h | 5 + xlators/cluster/dht/src/tier.c | 2 +- 5 files changed, 382 insertions(+), 75 deletions(-) (limited to 'xlators/cluster/dht') diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c index ee07a7f4f71..9d661441122 100644 --- a/xlators/cluster/dht/src/dht-common.c +++ b/xlators/cluster/dht/src/dht-common.c @@ -1163,9 +1163,19 @@ dht_lookup_unlink_stale_linkto_cbk (call_frame_t *frame, void *cookie, int dht_fill_dict_to_avoid_unlink_of_migrating_file (dict_t *dict) { - int ret = 0; + int ret = 0; + xlator_t *this = NULL; + char *linktoskip_key = NULL; + + this = THIS; + GF_VALIDATE_OR_GOTO ("dht", this, err); - ret = dict_set_int32 (dict, DHT_SKIP_NON_LINKTO_UNLINK, 1); + if (dht_is_tier_xlator (this)) + linktoskip_key = TIER_SKIP_NON_LINKTO_UNLINK; + else + linktoskip_key = DHT_SKIP_NON_LINKTO_UNLINK; + + ret = dict_set_int32 (dict, linktoskip_key, 1); if (ret) goto err; @@ -2427,103 +2437,104 @@ err: return 0; } - int -dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this, - int op_ret, int op_errno, struct iatt *preparent, - struct iatt *postparent, dict_t *xdata) +dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) { - dht_local_t *local = NULL; - call_frame_t *prev = NULL; + dht_local_t *local = NULL; + call_frame_t *prev = NULL; local = frame->local; prev = cookie; LOCK (&frame->lock); { - if (op_ret == -1) { - local->op_ret = -1; + if ((op_ret == -1) && !((op_errno == ENOENT) || + (op_errno == ENOTCONN))) { local->op_errno = op_errno; gf_msg_debug (this->name, op_errno, - "Unlink: subvolume %s returned -1", - prev->this->name); + "Unlink link: subvolume %s" + " returned -1", + prev->this->name); goto unlock; } local->op_ret = 0; - - local->postparent = *postparent; - local->preparent = *preparent; - - if (local->loc.parent) { - dht_inode_ctx_time_update (local->loc.parent, this, - &local->preparent, 0); - dht_inode_ctx_time_update (local->loc.parent, this, - &local->postparent, 1); - } } unlock: UNLOCK (&frame->lock); + DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno, - &local->preparent, &local->postparent, NULL); + &local->preparent, &local->postparent, xdata); return 0; } - int -dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this, - int op_ret, int op_errno, struct iatt *preparent, - struct iatt *postparent, dict_t *xdata) +dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) { - dht_local_t *local = NULL; - call_frame_t *prev = NULL; - - xlator_t *cached_subvol = NULL; + dht_local_t *local = NULL; + call_frame_t *prev = NULL; + xlator_t *hashed_subvol = NULL; local = frame->local; prev = cookie; LOCK (&frame->lock); { - if ((op_ret == -1) && !((op_errno == ENOENT) || - (op_errno == ENOTCONN))) { - local->op_errno = op_errno; + if (op_ret == -1) { + if (op_errno != ENOENT) { + local->op_ret = -1; + local->op_errno = op_errno; + } else { + local->op_ret = 0; + } gf_msg_debug (this->name, op_errno, - "Unlink link: subvolume %s" - " returned -1", - prev->this->name); + "Unlink: subvolume %s returned -1", + prev->this->name); goto unlock; } local->op_ret = 0; + + local->postparent = *postparent; + local->preparent = *preparent; + + if (local->loc.parent) { + dht_inode_ctx_time_update (local->loc.parent, this, + &local->preparent, 0); + dht_inode_ctx_time_update (local->loc.parent, this, + &local->postparent, 1); + } } unlock: UNLOCK (&frame->lock); - if (local->op_ret == -1) - goto err; - - cached_subvol = dht_subvol_get_cached (this, local->loc.inode); - if (!cached_subvol) { - gf_msg_debug (this->name, 0, - "no cached subvolume for path=%s", - local->loc.path); - local->op_errno = EINVAL; - goto err; + if (!local->op_ret) { + hashed_subvol = dht_subvol_get_hashed (this, &local->loc); + if (hashed_subvol && + hashed_subvol != local->cached_subvol) { + /* + * If hashed and cached are different, then we need + * to unlink linkfile from hashed subvol if data + * file is deleted successfully + */ + STACK_WIND (frame, dht_unlink_linkfile_cbk, + hashed_subvol, + hashed_subvol->fops->unlink, &local->loc, + local->flags, xdata); + return 0; + } } - STACK_WIND (frame, dht_unlink_cbk, - cached_subvol, cached_subvol->fops->unlink, - &local->loc, local->flags, NULL); + DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno, + &local->preparent, &local->postparent, xdata); return 0; - -err: - DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno, - NULL, NULL, NULL); - return 0; } int @@ -5608,7 +5619,6 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag, dict_t *xdata) { xlator_t *cached_subvol = NULL; - xlator_t *hashed_subvol = NULL; int op_errno = -1; dht_local_t *local = NULL; @@ -5623,15 +5633,6 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag, goto err; } - hashed_subvol = dht_subvol_get_hashed (this, loc); - /* Dont fail unlink if hashed_subvol is NULL which can be the result - * of layout anomaly */ - if (!hashed_subvol) { - gf_msg_debug (this->name, 0, - "no subvolume in layout for path=%s", - loc->path); - } - cached_subvol = local->cached_subvol; if (!cached_subvol) { gf_msg_debug (this->name, 0, @@ -5641,15 +5642,9 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag, } local->flags = xflag; - if (hashed_subvol && hashed_subvol != cached_subvol) { - STACK_WIND (frame, dht_unlink_linkfile_cbk, - hashed_subvol, hashed_subvol->fops->unlink, loc, - xflag, xdata); - } else { - STACK_WIND (frame, dht_unlink_cbk, - cached_subvol, cached_subvol->fops->unlink, loc, - xflag, xdata); - } + STACK_WIND (frame, dht_unlink_cbk, + cached_subvol, cached_subvol->fops->unlink, loc, + xflag, xdata); return 0; err: @@ -8121,3 +8116,11 @@ int32_t dht_set_local_rebalance (xlator_t *this, dht_local_t *local, return 0; } +gf_boolean_t +dht_is_tier_xlator (xlator_t *this) +{ + + if (strcmp (this->type, "cluster/tier") == 0) + return _gf_true; + return _gf_false; +} diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h index 4e72557621d..a6e9a408a44 100644 --- a/xlators/cluster/dht/src/dht-common.h +++ b/xlators/cluster/dht/src/dht-common.h @@ -1117,6 +1117,9 @@ dht_layout_missing_dirs (dht_layout_t *layout); int dht_refresh_layout (call_frame_t *frame); +gf_boolean_t +dht_is_tier_xlator (xlator_t *this); + int dht_build_parent_loc (xlator_t *this, loc_t *parent, loc_t *child, int32_t *op_errno); diff --git a/xlators/cluster/dht/src/tier-common.c b/xlators/cluster/dht/src/tier-common.c index 7d05c2973f2..19ed224c58c 100644 --- a/xlators/cluster/dht/src/tier-common.c +++ b/xlators/cluster/dht/src/tier-common.c @@ -16,6 +16,302 @@ #include "tier-common.h" #include "tier.h" +int +tier_unlink_nonhashed_linkfile_cbk (call_frame_t *frame, void *cookie, + xlator_t *this, int op_ret, int op_errno, + struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) +{ + dht_local_t *local = NULL; + call_frame_t *prev = NULL; + + local = frame->local; + prev = cookie; + + LOCK (&frame->lock); + { + if ((op_ret == -1) && (op_errno != ENOENT)) { + local->op_errno = op_errno; + local->op_ret = op_ret; + gf_msg_debug (this->name, op_errno, + "Unlink link: subvolume %s" + " returned -1", + prev->this->name); + goto unlock; + } + + local->op_ret = 0; + } +unlock: + UNLOCK (&frame->lock); + + if (local->op_ret == -1) + goto err; + DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno, + &local->preparent, &local->postparent, NULL); + + + return 0; + +err: + DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno, + NULL, NULL, NULL); + return 0; +} + +int +tier_unlink_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, inode_t *inode, + struct iatt *preparent, dict_t *xdata, + struct iatt *postparent) +{ + dht_local_t *local = NULL; + call_frame_t *prev = NULL; + dht_conf_t *conf = NULL; + xlator_t *hot_subvol = NULL; + + local = frame->local; + prev = cookie; + conf = this->private; + hot_subvol = TIER_UNHASHED_SUBVOL; + + if (!op_ret) { + /* + * linkfile present on hot tier. unlinking the linkfile + */ + STACK_WIND (frame, tier_unlink_nonhashed_linkfile_cbk, + hot_subvol, hot_subvol->fops->unlink, + &local->loc, local->flags, NULL); + return 0; + } + + LOCK (&frame->lock); + { + if (op_errno == ENOENT) { + local->op_ret = 0; + local->op_errno = op_errno; + } else { + local->op_ret = op_ret; + local->op_errno = op_errno; + } + gf_msg_debug (this->name, op_errno, + "Lookup : subvolume %s returned -1", + prev->this->name); + } + + UNLOCK (&frame->lock); + + DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno, + &local->preparent, &local->postparent, xdata); + + return 0; +} + +int +tier_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) +{ + dht_local_t *local = NULL; + call_frame_t *prev = NULL; + + local = frame->local; + prev = cookie; + + LOCK (&frame->lock); + { + /* Ignore EINVAL for tier to ignore error when the file + does not exist on the other tier */ + if ((op_ret == -1) && !((op_errno == ENOENT) || + (op_errno == EINVAL))) { + local->op_errno = op_errno; + local->op_ret = op_ret; + gf_msg_debug (this->name, op_errno, + "Unlink link: subvolume %s" + " returned -1", + prev->this->name); + goto unlock; + } + + local->op_ret = 0; + } +unlock: + UNLOCK (&frame->lock); + + if (local->op_ret == -1) + goto err; + + DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno, + &local->preparent, &local->postparent, xdata); + + return 0; + +err: + DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno, + NULL, NULL, NULL); + return 0; +} + +int32_t +tier_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) +{ + dht_local_t *local = NULL; + call_frame_t *prev = NULL; + struct iatt *stbuf = NULL; + dht_conf_t *conf = NULL; + int ret = -1; + xlator_t *hot_tier = NULL; + xlator_t *cold_tier = NULL; + + local = frame->local; + prev = cookie; + conf = this->private; + + cold_tier = TIER_HASHED_SUBVOL; + hot_tier = TIER_UNHASHED_SUBVOL; + + LOCK (&frame->lock); + { + if (op_ret == -1) { + if (op_errno == ENOENT) { + local->op_ret = 0; + } else { + local->op_ret = -1; + local->op_errno = op_errno; + } + gf_msg_debug (this->name, op_errno, + "Unlink: subvolume %s returned -1" + " with errno = %d", + prev->this->name, op_errno); + goto unlock; + } + + local->op_ret = 0; + + local->postparent = *postparent; + local->preparent = *preparent; + + if (local->loc.parent) { + dht_inode_ctx_time_update (local->loc.parent, this, + &local->preparent, 0); + dht_inode_ctx_time_update (local->loc.parent, this, + &local->postparent, 1); + } + } +unlock: + UNLOCK (&frame->lock); + + if (local->op_ret) + goto out; + + if (cold_tier != local->cached_subvol) { + /* + * File is present in hot tier, so there will be + * a link file on cold tier, deleting the linkfile + * from cold tier + */ + STACK_WIND (frame, tier_unlink_linkfile_cbk, + cold_tier, + cold_tier->fops->unlink, &local->loc, + local->flags, xdata); + return 0; + } + + ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf); + if (!ret && stbuf && ((IS_DHT_MIGRATION_PHASE2 (stbuf)) || + IS_DHT_MIGRATION_PHASE1 (stbuf))) { + /* + * File is migrating from cold to hot tier. + * Delete the destination linkfile. + */ + STACK_WIND (frame, tier_unlink_lookup_cbk, + hot_tier, + hot_tier->fops->lookup, + &local->loc, NULL); + return 0; + + } + +out: + DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno, + &local->preparent, &local->postparent, xdata); + + return 0; +} + +int +tier_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag, + dict_t *xdata) +{ + xlator_t *cached_subvol = NULL; + xlator_t *hashed_subvol = NULL; + dht_conf_t *conf = NULL; + int op_errno = -1; + dht_local_t *local = NULL; + int ret = -1; + + VALIDATE_OR_GOTO (frame, err); + VALIDATE_OR_GOTO (this, err); + VALIDATE_OR_GOTO (loc, err); + + conf = this->private; + + local = dht_local_init (frame, loc, NULL, GF_FOP_UNLINK); + if (!local) { + op_errno = ENOMEM; + + goto err; + } + + hashed_subvol = TIER_HASHED_SUBVOL; + + cached_subvol = local->cached_subvol; + if (!cached_subvol) { + gf_msg_debug (this->name, 0, + "no cached subvolume for path=%s", loc->path); + op_errno = EINVAL; + goto err; + } + + local->flags = xflag; + if (hashed_subvol == cached_subvol) { + /* + * File resides in cold tier. We need to stat + * the file to see if it is being promoted. + * If yes we need to delete the destination + * file as well. + */ + xdata = xdata ? dict_ref (xdata) : dict_new (); + if (xdata) { + ret = dict_set_dynstr_with_alloc (xdata, + DHT_IATT_IN_XDATA_KEY, "yes"); + if (ret) { + gf_msg_debug (this->name, 0, + "Failed to set dictionary key %s", + DHT_IATT_IN_XDATA_KEY); + } + } + } + + /* + * File is on hot tier, delete the data file first, then + * linkfile from cold. + */ + STACK_WIND (frame, tier_unlink_cbk, + cached_subvol, cached_subvol->fops->unlink, loc, + xflag, xdata); + if (xdata) + dict_unref (xdata); + return 0; +err: + op_errno = (op_errno == -1) ? errno : op_errno; + DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL); + + return 0; +} + int tier_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, int op_errno, gf_dirent_t *orig_entries, diff --git a/xlators/cluster/dht/src/tier-common.h b/xlators/cluster/dht/src/tier-common.h index 93a71f75252..81a0c96411d 100644 --- a/xlators/cluster/dht/src/tier-common.h +++ b/xlators/cluster/dht/src/tier-common.h @@ -10,6 +10,11 @@ #ifndef _TIER_COMMON_H_ #define _TIER_COMMON_H_ +/* Function definitions */ + +int32_t +tier_unlink (call_frame_t *frame, xlator_t *this, + loc_t *loc, int xflag, dict_t *xdata); int32_t tier_readdirp (call_frame_t *frame, diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c index d9bc4e9a0b2..51c18de094b 100644 --- a/xlators/cluster/dht/src/tier.c +++ b/xlators/cluster/dht/src/tier.c @@ -1986,7 +1986,7 @@ struct xlator_fops fops = { .readdirp = tier_readdirp, .fsyncdir = dht_fsyncdir, .symlink = dht_symlink, - .unlink = dht_unlink, + .unlink = tier_unlink, .link = dht_link, .mkdir = dht_mkdir, .rmdir = dht_rmdir, -- cgit