From 470869a954c17f32a3ba43ccda7442f82c0da6b2 Mon Sep 17 00:00:00 2001 From: N Balachandran Date: Tue, 1 Sep 2015 15:31:02 +0530 Subject: cluster/tier: Handle FOPs on files being migrated Determine which DHT level is responsible for handling fops on a file undergoing migration based on the name of the the linkto xattr set on the file being migrated and process accordingly. Change-Id: I82772e39314d4fe7f2ba0dcf22de0c6a374ee139 BUG: 1254428 Signed-off-by: N Balachandran Signed-off-by: Nithya Balachandran Reviewed-on: http://review.gluster.org/12090 Tested-by: NetBSD Build System Tested-by: Gluster Build System Reviewed-by: Raghavendra G --- xlators/cluster/dht/src/dht-common.c | 126 +++++++++++++++++++---- xlators/cluster/dht/src/dht-common.h | 13 ++- xlators/cluster/dht/src/dht-helper.c | 105 ++++++++++++++----- xlators/cluster/dht/src/dht-inode-read.c | 139 ++++++++++++++++++++----- xlators/cluster/dht/src/dht-inode-write.c | 162 ++++++++++++++++++++++++++---- xlators/cluster/dht/src/dht-messages.h | 17 +++- 6 files changed, 475 insertions(+), 87 deletions(-) (limited to 'xlators/cluster') diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c index ef1776beb30..e2c9f5547ac 100644 --- a/xlators/cluster/dht/src/dht-common.c +++ b/xlators/cluster/dht/src/dht-common.c @@ -26,13 +26,16 @@ int run_defrag = 0; -int dht_link2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame); +int dht_link2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame, + int ret); int -dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame); +dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, + int ret); int -dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame); +dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, + int ret); int dht_aggregate_quota_xattr (dict_t *dst, char *key, data_t *value) @@ -3379,9 +3382,10 @@ dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, goto out; } - local->op_ret = 0; - + local->op_ret = op_ret; local->rebalance.target_op_fn = dht_setxattr2; + if (xdata) + local->rebalance.xdata = dict_ref (xdata); /* Phase 2 of migration */ if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) { @@ -3398,7 +3402,7 @@ dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, &subvol1, &subvol2); if (!dht_mig_info_is_invalid (local->cached_subvol, subvol1, subvol2)) { - dht_setxattr2 (this, subvol2, frame); + dht_setxattr2 (this, subvol2, frame, 0); return 0; } @@ -3408,8 +3412,6 @@ dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, } out: - if (local->rebalance.xdata) - dict_unref (local->rebalance.xdata); if (local->fop == GF_FOP_SETXATTR) { DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL); @@ -3482,7 +3484,7 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this, } else { local->call_cnt = 1; - local->rebalance.xdata = dict_ref (xattr); + local->rebalance.xattr = dict_ref (xattr); local->rebalance.flags = flags; xdata = xdata ? dict_ref (xdata) : dict_new (); @@ -3565,27 +3567,42 @@ out: int -dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int op_errno = EINVAL; - if (!frame || !frame->local || !subvol) + if (!frame || !frame->local) goto err; local = frame->local; + if (we_are_not_migrating (ret)) { + /* This dht xlator is not migrating the file. Unwind and + * pass on the original mode bits so the higher DHT layer + * can handle this. + */ + DHT_STACK_UNWIND (setxattr, frame, local->op_ret, + local->op_errno, local->rebalance.xdata); + return 0; + } + + if (subvol == NULL) + goto err; + + op_errno = local->op_errno; + local->call_cnt = 2; /* This is the second attempt */ if (local->fop == GF_FOP_SETXATTR) { STACK_WIND (frame, dht_file_setxattr_cbk, subvol, subvol->fops->setxattr, &local->loc, - local->rebalance.xdata, local->rebalance.flags, + local->rebalance.xattr, local->rebalance.flags, NULL); } else { STACK_WIND (frame, dht_file_setxattr_cbk, subvol, subvol->fops->fsetxattr, local->fd, - local->rebalance.xdata, local->rebalance.flags, + local->rebalance.xattr, local->rebalance.flags, NULL); } @@ -3832,7 +3849,7 @@ dht_setxattr (call_frame_t *frame, xlator_t *this, } else { - local->rebalance.xdata = dict_ref (xattr); + local->rebalance.xattr = dict_ref (xattr); local->rebalance.flags = flags; local->call_cnt = 1; @@ -3896,6 +3913,8 @@ dht_file_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, local->op_ret = 0; local->rebalance.target_op_fn = dht_removexattr2; + if (xdata) + local->rebalance.xdata = dict_ref (xdata); /* Phase 2 of migration */ if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) { @@ -3912,7 +3931,7 @@ dht_file_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, &subvol1, &subvol2); if (!dht_mig_info_is_invalid (local->cached_subvol, subvol1, subvol2)) { - dht_removexattr2 (this, subvol2, frame); + dht_removexattr2 (this, subvol2, frame, 0); return 0; } @@ -3932,7 +3951,8 @@ out: } int -dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, + int ret) { dht_local_t *local = NULL; int op_errno = EINVAL; @@ -3944,6 +3964,17 @@ dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) local->call_cnt = 2; /* This is the second attempt */ + if (we_are_not_migrating (ret)) { + + /* This dht xlator is not migrating the file. Unwind and + * pass on the original mode bits so the higher DHT layer + * can handle this. + */ + DHT_STACK_UNWIND (removexattr, frame, local->op_ret, + local->op_errno, local->rebalance.xdata); + return 0; + } + if (local->fop == GF_FOP_REMOVEXATTR) { STACK_WIND (frame, dht_file_removexattr_cbk, subvol, subvol->fops->removexattr, &local->loc, @@ -5602,8 +5633,12 @@ dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this, local->inode = inode_ref (inode); } + local->op_ret = op_ret; local->op_errno = op_errno; local->rebalance.target_op_fn = dht_link2; + dht_set_local_rebalance (this, local, stbuf, preparent, + postparent, xdata); + /* Check if the rebalance phase2 is true */ if (IS_DHT_MIGRATION_PHASE2 (stbuf)) { ret = dht_inode_ctx_get_mig_info (this, local->loc.inode, NULL, @@ -5614,7 +5649,7 @@ dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this, if (!ret) return 0; } else { - dht_link2 (this, subvol, frame); + dht_link2 (this, subvol, frame, 0); return 0; } } @@ -5624,7 +5659,7 @@ dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this, ret = dht_inode_ctx_get_mig_info (this, local->loc.inode, NULL, &subvol); if (subvol) { - dht_link2 (this, subvol, frame); + dht_link2 (this, subvol, frame, 0); return 0; } ret = dht_rebalance_in_progress_check (this, frame); @@ -5642,7 +5677,7 @@ out: int -dht_link2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_link2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int op_errno = EINVAL; @@ -5652,6 +5687,19 @@ dht_link2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) goto err; op_errno = local->op_errno; + + if (we_are_not_migrating (ret)) { + /* This dht xlator is not migrating the file. Unwind and + * pass on the original mode bits so the higher DHT layer + * can handle this. + */ + DHT_STACK_UNWIND (link, frame, local->op_ret, op_errno, + local->inode, + &local->stbuf, &local->preparent, + &local->postparent, NULL); + return 0; + } + if (subvol == NULL) { op_errno = EINVAL; goto err; @@ -7855,3 +7903,43 @@ int32_t dht_migration_needed(xlator_t *this) out: return ret; } + + + +/* +This function should not be called more then once during a FOP +handling path. It is valid only for for ops on files +*/ +int32_t dht_set_local_rebalance (xlator_t *this, dht_local_t *local, + struct iatt *stbuf, + struct iatt *prebuf, struct iatt *postbuf, + dict_t *xdata) +{ + + if (!local) + return -1; + + if (local->rebalance.set) { + gf_msg (this->name, GF_LOG_WARNING, 0, + DHT_MSG_REBAL_STRUCT_SET, + "local->rebalance already set"); + } + + + if (stbuf) + memcpy (&local->rebalance.stbuf, stbuf, sizeof (struct iatt)); + + if (prebuf) + memcpy (&local->rebalance.prebuf, prebuf, sizeof (struct iatt)); + + if (postbuf) + memcpy (&local->rebalance.postbuf, postbuf, + sizeof (struct iatt)); + + if (xdata) + local->rebalance.xdata = dict_ref (xdata); + + local->rebalance.set = 1; + + return 0; +} diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h index 6bf8ba1c406..c48bf5800b9 100644 --- a/xlators/cluster/dht/src/dht-common.h +++ b/xlators/cluster/dht/src/dht-common.h @@ -37,7 +37,7 @@ typedef int (*dht_selfheal_dir_cbk_t) (call_frame_t *frame, void *cookie, int32_t op_ret, int32_t op_errno, dict_t *xdata); typedef int (*dht_defrag_cbk_fn_t) (xlator_t *this, xlator_t *dst_node, - call_frame_t *frame); + call_frame_t *frame, int ret); typedef int (*dht_refresh_layout_unlock) (call_frame_t *frame, xlator_t *this, int op_ret); @@ -116,8 +116,12 @@ struct dht_rebalance_ { struct iobref *iobref; struct iovec *vector; struct iatt stbuf; + struct iatt prebuf; + struct iatt postbuf; dht_defrag_cbk_fn_t target_op_fn; dict_t *xdata; + dict_t *xattr; + int32_t set; }; /** @@ -548,6 +552,8 @@ typedef struct dht_migrate_info { #define layout_is_sane(layout) ((layout) && (layout->cnt > 0)) +#define we_are_not_migrating(x) ((x) == 1) + #define DHT_STACK_UNWIND(fop, frame, params ...) do { \ dht_local_t *__local = NULL; \ xlator_t *__xl = NULL; \ @@ -1078,4 +1084,9 @@ int dht_build_parent_loc (xlator_t *this, loc_t *parent, loc_t *child, int32_t *op_errno); +int32_t dht_set_local_rebalance (xlator_t *this, dht_local_t *local, + struct iatt *stbuf, + struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata); + #endif/* _DHT_H */ diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c index 88bf7a605da..3a4217029b9 100644 --- a/xlators/cluster/dht/src/dht-helper.c +++ b/xlators/cluster/dht/src/dht-helper.c @@ -132,17 +132,6 @@ dht_frame_return (call_frame_t *frame) return this_call_cnt; } -/* - * A slightly "updated" version of the algorithm described in the commit log - * is used here. - * - * The only enhancement is that: - * - * - The number of bits used by the backend filesystem for HUGE d_off which - * is described as 63, and - * - The number of bits used by the d_off presented by the transformation - * upwards which is described as 64, are both made "configurable." - */ int dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc, @@ -455,6 +444,12 @@ dht_local_wipe (xlator_t *this, dht_local_t *local) GF_FREE (local->key); + if (local->rebalance.xdata) + dict_unref (local->rebalance.xdata); + + if (local->rebalance.xattr) + dict_unref (local->rebalance.xattr); + GF_FREE (local->rebalance.vector); if (local->rebalance.iobref) @@ -886,7 +881,12 @@ dht_init_subvolumes (xlator_t *this, dht_conf_t *conf) } - +/* + op_ret values : + 0 : Success. + -1 : Failure. + 1 : File is being migrated but not by this DHT layer. +*/ static int dht_migration_complete_check_done (int op_ret, call_frame_t *frame, void *data) @@ -896,7 +896,7 @@ dht_migration_complete_check_done (int op_ret, call_frame_t *frame, void *data) local = frame->local; - if (op_ret == -1) + if (op_ret != 0) goto out; if (local->cached_subvol == NULL) { @@ -907,7 +907,7 @@ dht_migration_complete_check_done (int op_ret, call_frame_t *frame, void *data) subvol = local->cached_subvol; out: - local->rebalance.target_op_fn (THIS, subvol, frame); + local->rebalance.target_op_fn (THIS, subvol, frame, op_ret); return 0; } @@ -960,15 +960,35 @@ dht_migration_complete_check_task (void *data) SYNCTASK_SETID (frame->root->uid, frame->root->gid); } + /* - * temporary check related to tier promoting/demoting the file; - * the lower level DHT detects the migration (due to sticky - * bits) when it is the responsibility of the tier translator - * to complete the rebalance transaction. It will be corrected - * when rebalance and tier migration are fixed to work together. + * Each DHT xlator layer has its own name for the linkto xattr. + * If the file mode bits indicate the the file is being migrated but + * this layer's linkto xattr is not set, it means that another + * DHT layer is migrating the file. In this case, return 1 so + * the mode bits can be passed on to the higher layer for appropriate + * action. */ - if (strcmp(this->parents->xlator->type, "cluster/tier") == 0) { - ret = 0; + if (-ret == ENODATA) { + /* This DHT translator is not migrating this file */ + + ret = inode_ctx_reset1 (inode, this, &tmp_miginfo); + if (tmp_miginfo) { + + /* This can be a problem if the file was + * migrated by two different layers. Raise + * a warning here. + */ + gf_msg (this->name, GF_LOG_WARNING, 0, + DHT_MSG_HAS_MIGINFO, + "%s: Found miginfo in the inode ctx", + tmp_loc.path ? tmp_loc.path : + uuid_utoa (tmp_loc.gfid)); + + miginfo = (void *)tmp_miginfo; + GF_REF_PUT (miginfo); + } + ret = 1; goto out; } @@ -1094,7 +1114,14 @@ dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame) return ret; } + /* During 'in-progress' state, both nodes should have the file */ +/* + op_ret values : + 0 : Success + -1 : Failure. + 1 : File is being migrated but not by this DHT layer. +*/ static int dht_inprogress_check_done (int op_ret, call_frame_t *frame, void *data) { @@ -1104,7 +1131,7 @@ dht_inprogress_check_done (int op_ret, call_frame_t *frame, void *data) local = frame->local; - if (op_ret == -1) + if (op_ret != 0) goto out; inode = local->loc.inode ? local->loc.inode : local->fd->inode; @@ -1120,7 +1147,7 @@ dht_inprogress_check_done (int op_ret, call_frame_t *frame, void *data) } out: - local->rebalance.target_op_fn (THIS, dst_subvol, frame); + local->rebalance.target_op_fn (THIS, dst_subvol, frame, op_ret); return 0; } @@ -1142,6 +1169,9 @@ dht_rebalance_inprogress_task (void *data) inode_t *inode = NULL; fd_t *iter_fd = NULL; int open_failed = 0; + uint64_t tmp_miginfo = 0; + dht_migrate_info_t *miginfo = NULL; + this = THIS; frame = data; @@ -1167,6 +1197,35 @@ dht_rebalance_inprogress_task (void *data) conf->link_xattr_name, NULL, NULL); } + /* + * Each DHT xlator layer has its own name for the linkto xattr. + * If the file mode bits indicate the the file is being migrated but + * this layer's linkto xattr is not present, it means that another + * DHT layer is migrating the file. In this case, return 1 so + * the mode bits can be passed on to the higher layer for appropriate + * action. + */ + + if (-ret == ENODATA) { + /* This DHT layer is not migrating this file */ + ret = inode_ctx_reset1 (inode, this, &tmp_miginfo); + if (tmp_miginfo) { + /* This can be a problem if the file was + * migrated by two different layers. Raise + * a warning here. + */ + gf_msg (this->name, GF_LOG_WARNING, 0, + DHT_MSG_HAS_MIGINFO, + "%s: Found miginfo in the inode ctx", + tmp_loc.path ? tmp_loc.path : + uuid_utoa (tmp_loc.gfid)); + miginfo = (void *)tmp_miginfo; + GF_REF_PUT (miginfo); + } + ret = 1; + goto out; + } + if (ret < 0) { gf_msg (this->name, GF_LOG_ERROR, -ret, DHT_MSG_GET_XATTR_FAILED, diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c index a5eda4e1270..02265fd2e85 100644 --- a/xlators/cluster/dht/src/dht-inode-read.c +++ b/xlators/cluster/dht/src/dht-inode-read.c @@ -10,13 +10,22 @@ #include "dht-common.h" -int dht_access2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame); -int dht_readv2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame); -int dht_attr2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame); -int dht_open2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame); -int dht_flush2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame); -int dht_lk2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame); -int dht_fsync2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame); +int dht_access2 (xlator_t *this, xlator_t *dst_node, + call_frame_t *frame, int ret); +int dht_readv2 (xlator_t *this, xlator_t *dst_node, + call_frame_t *frame, int ret); +int dht_attr2 (xlator_t *this, xlator_t *dst_node, + call_frame_t *frame, int ret); +int dht_open2 (xlator_t *this, xlator_t *dst_node, + call_frame_t *frame, int ret); +int dht_flush2 (xlator_t *this, xlator_t *dst_node, + call_frame_t *frame, int ret); +int dht_lk2 (xlator_t *this, xlator_t *dst_node, + call_frame_t *frame, int ret); +int dht_fsync2 (xlator_t *this, xlator_t *dst_node, + call_frame_t *frame, int ret); + + int dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, @@ -53,7 +62,7 @@ out: } int -dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int op_errno = EINVAL; @@ -64,6 +73,14 @@ dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) local = frame->local; op_errno = ENOENT; + if (we_are_not_migrating (ret)) { + /* This DHT layer is not migrating the file */ + DHT_STACK_UNWIND (open, frame, -1, local->op_errno, + NULL, NULL); + return 0; + + } + if (subvol == NULL) goto out; @@ -75,10 +92,11 @@ dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) return 0; out: - DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL); + DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL); return 0; } + int dht_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd, dict_t *xdata) @@ -150,6 +168,8 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, goto out; local->op_errno = op_errno; + local->op_ret = op_ret; + /* Check if the rebalance phase2 is true */ if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) { inode = (local->fd) ? local->fd->inode : local->loc.inode; @@ -157,13 +177,15 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, if (!subvol) { /* Phase 2 of migration */ local->rebalance.target_op_fn = dht_attr2; + dht_set_local_rebalance (this, local, NULL, NULL, + stbuf, xdata); ret = dht_rebalance_complete_check (this, frame); if (!ret) return 0; } else { /* value is already set in fd_ctx, that means no need to check for whether its complete or not. */ - dht_attr2 (this, subvol, frame); + dht_attr2 (this, subvol, frame, 0); return 0; } } @@ -176,7 +198,7 @@ err: } int -dht_attr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_attr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int op_errno = EINVAL; @@ -186,6 +208,19 @@ dht_attr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) goto out; op_errno = local->op_errno; + + if (we_are_not_migrating (ret)) { + /* This dht xlator is not migrating the file. Unwind and + * pass on the original mode bits so the higher DHT layer + * can handle this. + */ + DHT_STACK_UNWIND (stat, frame, local->op_ret, op_errno, + &local->rebalance.postbuf, + local->rebalance.xdata); + return 0; + } + + if (subvol == NULL) goto out; @@ -377,7 +412,8 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this, { dht_local_t *local = NULL; int ret = 0; - xlator_t *subvol = 0; + xlator_t *src_subvol = 0; + xlator_t *dst_subvol = 0; local = frame->local; if (!local) { @@ -396,23 +432,31 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this, local->op_errno = op_errno; if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) { /* File would be migrated to other node */ - ret = dht_inode_ctx_get_mig_info (this, local->fd->inode, NULL, - &subvol); - if (!subvol) { + ret = dht_inode_ctx_get_mig_info (this, local->fd->inode, + &src_subvol, + &dst_subvol); + + if (dht_mig_info_is_invalid (local->cached_subvol, + src_subvol, dst_subvol)) { + local->op_ret = op_ret; local->rebalance.target_op_fn = dht_readv2; + dht_set_local_rebalance (this, local, NULL, NULL, + stbuf, xdata); + ret = dht_rebalance_complete_check (this, frame); if (!ret) return 0; } else { /* value is already set in fd_ctx, that means no need to check for whether its complete or not. */ - dht_readv2 (this, subvol, frame); + dht_readv2 (this, dst_subvol, frame, 0); return 0; } } out: DHT_STRIP_PHASE1_FLAGS (stbuf); + DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf, iobref, xdata); @@ -420,7 +464,7 @@ out: } int -dht_readv2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_readv2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int op_errno = EINVAL; @@ -430,6 +474,18 @@ dht_readv2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) goto out; op_errno = local->op_errno; + + if (we_are_not_migrating (ret)) { + /* This dht xlator is not migrating the file. Unwind and + * pass on the original mode bits so the higher DHT layer + * can handle this. + */ + DHT_STACK_UNWIND (readv, frame, local->op_ret, op_errno, + NULL, 0, &local->rebalance.postbuf, + NULL, local->rebalance.xdata); + return 0; + } + if (subvol == NULL) goto out; @@ -446,6 +502,7 @@ out: return 0; } + int dht_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata) @@ -538,7 +595,7 @@ out: } int -dht_access2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_access2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int op_errno = EINVAL; @@ -548,6 +605,17 @@ dht_access2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) goto out; op_errno = local->op_errno; + + if (we_are_not_migrating (ret)) { + /* This dht xlator is not migrating the file. Unwind and + * pass on the original mode bits so the higher DHT layer + * can handle this. + */ + + DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL); + return 0; + } + if (subvol == NULL) goto out; @@ -624,7 +692,7 @@ dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this, /* If context is set, then send flush() it to the destination */ dht_inode_ctx_get_mig_info (this, local->fd->inode, NULL, &subvol); if (subvol) { - dht_flush2 (this, subvol, frame); + dht_flush2 (this, subvol, frame, 0); return 0; } @@ -635,7 +703,7 @@ out: } int -dht_flush2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_flush2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int32_t op_errno = EINVAL; @@ -647,7 +715,6 @@ dht_flush2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) op_errno = local->op_errno; - dht_inode_ctx_get_mig_info (this, local->fd->inode, NULL, &subvol); if (subvol == NULL) goto out; @@ -713,7 +780,8 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, call_frame_t *prev = NULL; int ret = -1; inode_t *inode = NULL; - xlator_t *subvol = 0; + xlator_t *src_subvol = 0; + xlator_t *dst_subvol = 0; local = frame->local; prev = cookie; @@ -736,9 +804,15 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, local->op_errno = op_errno; inode = local->fd->inode; - dht_inode_ctx_get_mig_info (this, inode, NULL, &subvol); - if (!subvol) { + + dht_inode_ctx_get_mig_info (this, inode, &src_subvol, &dst_subvol); + + if (dht_mig_info_is_invalid (local->cached_subvol, + src_subvol, dst_subvol)) { + local->rebalance.target_op_fn = dht_fsync2; + dht_set_local_rebalance (this, local, NULL, prebuf, + postbuf, xdata); /* Check if the rebalance phase1 is true */ if (IS_DHT_MIGRATION_PHASE1 (postbuf)) { @@ -755,13 +829,14 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, if (!ret) return 0; } else { - dht_fsync2 (this, subvol, frame); + dht_fsync2 (this, dst_subvol, frame, 0); return 0; } out: DHT_STRIP_PHASE1_FLAGS (postbuf); DHT_STRIP_PHASE1_FLAGS (prebuf); + DHT_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata); @@ -769,7 +844,7 @@ out: } int -dht_fsync2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_fsync2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int32_t op_errno = EINVAL; @@ -780,6 +855,18 @@ dht_fsync2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) local = frame->local; op_errno = local->op_errno; + if (we_are_not_migrating (ret)) { + /* This dht xlator is not migrating the file. Unwind and + * pass on the original mode bits so the higher DHT layer + * can handle this. + */ + DHT_STACK_UNWIND (fsync, frame, local->op_ret, + op_errno, &local->rebalance.prebuf, + &local->rebalance.postbuf, + local->rebalance.xdata); + return 0; + } + if (subvol == NULL) goto out; diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c index ce5006a80df..a1bf2aa5e8d 100644 --- a/xlators/cluster/dht/src/dht-inode-write.c +++ b/xlators/cluster/dht/src/dht-inode-write.c @@ -11,12 +11,18 @@ #include "dht-common.h" -int dht_writev2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame); -int dht_truncate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame); -int dht_setattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame); -int dht_fallocate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame); -int dht_discard2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame); -int dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame); +int dht_writev2 (xlator_t *this, xlator_t *subvol, + call_frame_t *frame, int ret); +int dht_truncate2 (xlator_t *this, xlator_t *subvol, + call_frame_t *frame, int ret); +int dht_setattr2 (xlator_t *this, xlator_t *subvol, + call_frame_t *frame, int ret); +int dht_fallocate2 (xlator_t *this, xlator_t *subvol, + call_frame_t *frame, int ret); +int dht_discard2 (xlator_t *this, xlator_t *subvol, + call_frame_t *frame, int ret); +int dht_zerofill2 (xlator_t *this, xlator_t *subvol, + call_frame_t *frame, int ret); int dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this, @@ -58,7 +64,15 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this, local->rebalance.target_op_fn = dht_writev2; + local->op_ret = op_ret; local->op_errno = op_errno; + + /* We might need to pass the stbuf information to the higher DHT + * layer for appropriate handling. + */ + + dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata); + /* Phase 2 of migration */ if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) { ret = dht_rebalance_complete_check (this, frame); @@ -75,7 +89,7 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this, &subvol1, &subvol2); if (!dht_mig_info_is_invalid (local->cached_subvol, subvol1, subvol2)) { - dht_writev2 (this, subvol2, frame); + dht_writev2 (this, subvol2, frame, 0); return 0; } ret = dht_rebalance_in_progress_check (this, frame); @@ -94,7 +108,7 @@ out: } int -dht_writev2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_writev2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int32_t op_errno = EINVAL; @@ -105,6 +119,19 @@ dht_writev2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) local = frame->local; op_errno = local->op_errno; + if (we_are_not_migrating (ret)) { + /* This dht xlator is not migrating the file. Unwind and + * pass on the original mode bits so the higher DHT layer + * can handle this. + */ + DHT_STACK_UNWIND (writev, frame, local->op_ret, + local->op_errno, &local->rebalance.prebuf, + &local->rebalance.postbuf, + local->rebalance.xdata); + return 0; + } + + if (subvol == NULL) goto out; @@ -215,7 +242,15 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this, local->rebalance.target_op_fn = dht_truncate2; + local->op_ret = op_ret; local->op_errno = op_errno; + + /* We might need to pass the stbuf information to the higher DHT + * layer for appropriate handling. + */ + + dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata); + /* Phase 2 of migration */ if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) { ret = dht_rebalance_complete_check (this, frame); @@ -233,7 +268,7 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this, &dst_subvol); if (!dht_mig_info_is_invalid (local->cached_subvol, src_subvol, dst_subvol)) { - dht_truncate2 (this, dst_subvol, frame); + dht_truncate2 (this, dst_subvol, frame, 0); return 0; } ret = dht_rebalance_in_progress_check (this, frame); @@ -244,6 +279,7 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this, out: DHT_STRIP_PHASE1_FLAGS (postbuf); DHT_STRIP_PHASE1_FLAGS (prebuf); + DHT_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf, postbuf, xdata); err: @@ -252,7 +288,7 @@ err: int -dht_truncate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_truncate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int32_t op_errno = EINVAL; @@ -263,6 +299,16 @@ dht_truncate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) local = frame->local; op_errno = local->op_errno; + /* This dht xlator is not migrating the file */ + if (we_are_not_migrating (ret)) { + + DHT_STACK_UNWIND (truncate, frame, local->op_ret, + local->op_errno, &local->rebalance.prebuf, + &local->rebalance.postbuf, + local->rebalance.xdata); + return 0; + } + if (subvol == NULL) goto out; @@ -406,8 +452,13 @@ dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, } goto out; } + + local->op_ret = op_ret; + local->op_errno = op_errno; local->rebalance.target_op_fn = dht_fallocate2; + dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata); + /* Phase 2 of migration */ if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) { ret = dht_rebalance_complete_check (this, frame); @@ -424,7 +475,7 @@ dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, &dst_subvol); if (!dht_mig_info_is_invalid (local->cached_subvol, src_subvol, dst_subvol)) { - dht_fallocate2 (this, dst_subvol, frame); + dht_fallocate2 (this, dst_subvol, frame, 0); return 0; } ret = dht_rebalance_in_progress_check (this, frame); @@ -435,6 +486,7 @@ dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, out: DHT_STRIP_PHASE1_FLAGS (postbuf); DHT_STRIP_PHASE1_FLAGS (prebuf); + DHT_STACK_UNWIND (fallocate, frame, op_ret, op_errno, prebuf, postbuf, xdata); err: @@ -442,7 +494,7 @@ err: } int -dht_fallocate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_fallocate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int32_t op_errno = EINVAL; @@ -453,6 +505,19 @@ dht_fallocate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) local = frame->local; op_errno = local->op_errno; + if (we_are_not_migrating (ret)) { + /* This dht xlator is not migrating the file. Unwind and + * pass on the original mode bits so the higher DHT layer + * can handle this. + */ + DHT_STACK_UNWIND (fallocate, frame, local->op_ret, + local->op_errno, + &local->rebalance.prebuf, + &local->rebalance.postbuf, + local->rebalance.xdata); + return 0; + } + if (subvol == NULL) goto out; @@ -550,7 +615,12 @@ dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, } goto out; } + local->rebalance.target_op_fn = dht_discard2; + local->op_ret = op_ret; + local->op_errno = op_errno; + + dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata); /* Phase 2 of migration */ if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) { @@ -568,7 +638,7 @@ dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, &dst_subvol); if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol, dst_subvol)) { - dht_discard2 (this, dst_subvol, frame); + dht_discard2 (this, dst_subvol, frame, 0); return 0; } ret = dht_rebalance_in_progress_check (this, frame); @@ -579,6 +649,7 @@ dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, out: DHT_STRIP_PHASE1_FLAGS (postbuf); DHT_STRIP_PHASE1_FLAGS (prebuf); + DHT_STACK_UNWIND (discard, frame, op_ret, op_errno, prebuf, postbuf, xdata); err: @@ -586,7 +657,7 @@ err: } int -dht_discard2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_discard2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int32_t op_errno = EINVAL; @@ -597,6 +668,19 @@ dht_discard2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) local = frame->local; op_errno = local->op_errno; + if (we_are_not_migrating (ret)) { + /* This dht xlator is not migrating the file. Unwind and + * pass on the original mode bits so the higher DHT layer + * can handle this. + */ + DHT_STACK_UNWIND (discard, frame, local->op_ret, + local->op_errno, + &local->rebalance.prebuf, + &local->rebalance.postbuf, + local->rebalance.xdata); + return 0; + } + if (subvol == NULL) goto out; @@ -689,7 +773,13 @@ dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, } goto out; } + local->rebalance.target_op_fn = dht_zerofill2; + local->op_ret = op_ret; + local->op_errno = op_errno; + + dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata); + /* Phase 2 of migration */ if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) { ret = dht_rebalance_complete_check (this, frame); @@ -706,7 +796,7 @@ dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, &subvol1, &subvol2); if (!dht_mig_info_is_invalid (local->cached_subvol, subvol1, subvol2)) { - dht_zerofill2 (this, subvol2, frame); + dht_zerofill2 (this, subvol2, frame, 0); return 0; } @@ -718,6 +808,7 @@ dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, out: DHT_STRIP_PHASE1_FLAGS (postbuf); DHT_STRIP_PHASE1_FLAGS (prebuf); + DHT_STACK_UNWIND (zerofill, frame, op_ret, op_errno, prebuf, postbuf, xdata); err: @@ -725,7 +816,7 @@ err: } int -dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int32_t op_errno = EINVAL; @@ -737,6 +828,20 @@ dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) op_errno = local->op_errno; + if (we_are_not_migrating (ret)) { + /* This dht xlator is not migrating the file. Unwind and + * pass on the original mode bits so the higher DHT layer + * can handle this. + */ + DHT_STACK_UNWIND (zerofill, frame, local->op_ret, + local->op_errno, + &local->rebalance.prebuf, + &local->rebalance.postbuf, + local->rebalance.xdata); + + return 0; + } + if (subvol == NULL) goto out; @@ -749,6 +854,7 @@ dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) return 0; out: + DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL); return 0; } @@ -821,10 +927,18 @@ dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, if (local->call_cnt != 1) goto out; + local->op_ret = op_ret; + local->op_errno = op_errno; + local->rebalance.target_op_fn = dht_setattr2; + /* Phase 2 of migration */ if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) { + + dht_set_local_rebalance (this, local, NULL, prebuf, + postbuf, xdata); + ret = dht_rebalance_complete_check (this, frame); if (!ret) return 0; @@ -837,6 +951,7 @@ dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, out: DHT_STRIP_PHASE1_FLAGS (postbuf); DHT_STRIP_PHASE1_FLAGS (prebuf); + DHT_STACK_UNWIND (setattr, frame, op_ret, op_errno, prebuf, postbuf, xdata); @@ -844,7 +959,7 @@ out: } int -dht_setattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +dht_setattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) { dht_local_t *local = NULL; int32_t op_errno = EINVAL; @@ -855,6 +970,19 @@ dht_setattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) local = frame->local; op_errno = local->op_errno; + if (we_are_not_migrating (ret)) { + /* This dht xlator is not migrating the file. Unwind and + * pass on the original mode bits so the higher DHT layer + * can handle this. + */ + DHT_STACK_UNWIND (setattr, frame, local->op_ret, + local->op_errno, + &local->rebalance.prebuf, + &local->rebalance.postbuf, + local->rebalance.xdata); + return 0; + } + if (subvol == NULL) goto out; diff --git a/xlators/cluster/dht/src/dht-messages.h b/xlators/cluster/dht/src/dht-messages.h index 60030832d9f..80b35557408 100644 --- a/xlators/cluster/dht/src/dht-messages.h +++ b/xlators/cluster/dht/src/dht-messages.h @@ -40,7 +40,7 @@ */ #define GLFS_DHT_BASE GLFS_MSGID_COMP_DHT -#define GLFS_DHT_NUM_MESSAGES 104 +#define GLFS_DHT_NUM_MESSAGES 106 #define GLFS_MSGID_END (GLFS_DHT_BASE + GLFS_DHT_NUM_MESSAGES + 1) /* Messages with message IDs */ @@ -978,5 +978,20 @@ #define DHT_MSG_COMMIT_HASH_INFO (GLFS_DHT_BASE + 104) +/* + * @messageid 109105 + * @diagnosis + * @recommendedaction None + */ + +#define DHT_MSG_REBAL_STRUCT_SET (GLFS_DHT_BASE + 105) + +/* + * @messageid 109106 + * @diagnosis + * @recommendedaction None + */ + +#define DHT_MSG_HAS_MIGINFO (GLFS_DHT_BASE + 106) #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages" #endif /* _DHT_MESSAGES_H_ */ -- cgit