From 4124fc8cb1b5026d8b9bb31b5e24d97aa1d94f86 Mon Sep 17 00:00:00 2001 From: Nithya Balachandran Date: Mon, 13 Apr 2015 14:24:44 +0530 Subject: cluster/dht: Fix dht_setxattr to follow files under migration If a file is under migration, then any xattrs created on it are lost post migration of the file. This is because the xattrs are set only on the cached subvol of the source and as the source is under migration, it becomes a linkto file post migration. Change-Id: Ib8e233b519cf954e7723c6e26b38fa8f9b8c85c0 BUG: 1225839 Signed-off-by: Nithya Balachandran Reviewed-on: http://review.gluster.org/10968 Tested-by: Gluster Build System Reviewed-by: Raghavendra G Tested-by: Raghavendra G --- xlators/cluster/dht/src/dht-common.c | 363 +++++++++++++++++++++++++++++++++-- xlators/cluster/ec/src/ec-combine.c | 1 + xlators/storage/posix/src/posix.c | 123 ++++++++++-- 3 files changed, 458 insertions(+), 29 deletions(-) (limited to 'xlators') diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c index d41dbffba5d..3a461cbf3a8 100644 --- a/xlators/cluster/dht/src/dht-common.c +++ b/xlators/cluster/dht/src/dht-common.c @@ -31,6 +31,12 @@ int dht_link2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame); +int +dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame); + +int +dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame); + int dht_aggregate_quota_xattr (dict_t *dst, char *key, data_t *value) { @@ -96,6 +102,8 @@ out: return ret; } + + int dht_aggregate (dict_t *this, char *key, data_t *value, void *data) { @@ -3264,6 +3272,81 @@ err: return 0; } +int +dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, dict_t *xdata) +{ + int ret = -1; + dht_local_t *local = NULL; + call_frame_t *prev = NULL; + struct iatt *stbuf = NULL; + inode_t *inode = NULL; + xlator_t *subvol1 = NULL, *subvol2 = NULL; + + local = frame->local; + prev = cookie; + + local->op_errno = op_errno; + + if ((op_ret == -1) && !dht_inode_missing (op_errno)) { + gf_msg_debug (this->name, op_errno, + "subvolume %s returned -1.", + prev->this->name); + goto out; + } + + if (local->call_cnt != 1) + goto out; + + ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf); + + if ((!op_ret) && !stbuf) { + goto out; + } + + local->op_ret = 0; + + local->rebalance.target_op_fn = dht_setxattr2; + + /* Phase 2 of migration */ + if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) { + ret = dht_rebalance_complete_check (this, frame); + if (!ret) + return 0; + } + + /* Phase 1 of migration */ + if (IS_DHT_MIGRATION_PHASE1 (stbuf)) { + inode = (local->fd) ? local->fd->inode : local->loc.inode; + + ret = dht_inode_ctx_get_mig_info (this, inode, + &subvol1, &subvol2); + if (!dht_mig_info_is_invalid (local->cached_subvol, + subvol1, subvol2)) { + dht_setxattr2 (this, subvol2, frame); + return 0; + } + + ret = dht_rebalance_in_progress_check (this, frame); + if (!ret) + return 0; + } + +out: + if (local->rebalance.xdata) + dict_unref (local->rebalance.xdata); + + if (local->fop == GF_FOP_SETXATTR) { + DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL); + } else { + DHT_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL); + } + + return 0; +} + + + int dht_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr, int flags, dict_t *xdata) @@ -3272,6 +3355,10 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this, dht_local_t *local = NULL; int op_errno = EINVAL; dht_conf_t *conf = NULL; + dht_layout_t *layout = NULL; + int ret = -1; + int call_cnt = 0; + int i = 0; VALIDATE_OR_GOTO (frame, err); VALIDATE_OR_GOTO (this, err); @@ -3299,11 +3386,47 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this, goto err; } - local->call_cnt = 1; + layout = local->layout; + if (!layout) { + gf_msg_debug (this->name, 0, + "no layout for fd=%p", fd); + op_errno = EINVAL; + goto err; + } + + local->call_cnt = call_cnt = layout->cnt; + + if (IA_ISDIR (fd->inode->ia_type)) { + for (i = 0; i < call_cnt; i++) { + STACK_WIND (frame, dht_err_cbk, + layout->list[i].xlator, + layout->list[i].xlator->fops->fsetxattr, + fd, xattr, flags, NULL); + } + + } else { - STACK_WIND (frame, dht_err_cbk, subvol, subvol->fops->fsetxattr, - fd, xattr, flags, NULL); + local->call_cnt = 1; + local->rebalance.xdata = dict_ref (xattr); + local->rebalance.flags = flags; + xdata = xdata ? dict_ref (xdata) : dict_new (); + if (xdata) + ret = dict_set_dynstr_with_alloc (xdata, + DHT_IATT_IN_XDATA_KEY, "yes"); + if (ret) { + gf_msg_debug (this->name, 0, + "Failed to set dictionary key %s for fd=%p", + DHT_IATT_IN_XDATA_KEY, fd); + } + + STACK_WIND (frame, dht_file_setxattr_cbk, subvol, + subvol->fops->fsetxattr, fd, xattr, flags, xdata); + + if (xdata) + dict_unref (xdata); + + } return 0; err: @@ -3324,6 +3447,7 @@ dht_common_setxattr_cbk (call_frame_t *frame, void *cookie, return 0; } + int dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, int op_errno, dict_t *xattr, @@ -3365,6 +3489,40 @@ out: } + +int +dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +{ + dht_local_t *local = NULL; + int op_errno = EINVAL; + + if (!frame || !frame->local || !subvol) + goto err; + + local = frame->local; + + local->call_cnt = 2; /* This is the second attempt */ + + if (local->fop == GF_FOP_SETXATTR) { + STACK_WIND (frame, dht_file_setxattr_cbk, subvol, + subvol->fops->setxattr, &local->loc, + local->rebalance.xdata, local->rebalance.flags, + NULL); + } else { + STACK_WIND (frame, dht_file_setxattr_cbk, subvol, + subvol->fops->fsetxattr, local->fd, + local->rebalance.xdata, local->rebalance.flags, + NULL); + } + + return 0; + +err: + DHT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL); + return 0; +} + + int dht_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr, int flags, dict_t *xdata) @@ -3587,11 +3745,32 @@ dht_setxattr (call_frame_t *frame, xlator_t *this, goto err; } - for (i = 0; i < call_cnt; i++) { - STACK_WIND (frame, dht_err_cbk, - layout->list[i].xlator, - layout->list[i].xlator->fops->setxattr, + if (IA_ISDIR (loc->inode->ia_type)) { + + for (i = 0; i < call_cnt; i++) { + STACK_WIND (frame, dht_err_cbk, + layout->list[i].xlator, + layout->list[i].xlator->fops->setxattr, + loc, xattr, flags, xdata); + } + + } else { + + local->rebalance.xdata = dict_ref (xattr); + local->rebalance.flags = flags; + local->call_cnt = 1; + + xdata = xdata ? dict_ref (xdata) : dict_new (); + if (xdata) + ret = dict_set_dynstr_with_alloc (xdata, + DHT_IATT_IN_XDATA_KEY, "yes"); + + STACK_WIND (frame, dht_file_setxattr_cbk, + subvol, subvol->fops->setxattr, loc, xattr, flags, xdata); + + if (xdata) + dict_unref (xdata); } return 0; @@ -3604,6 +3783,108 @@ err: } + + +int +dht_file_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, dict_t *xdata) +{ + int ret = -1; + dht_local_t *local = NULL; + call_frame_t *prev = NULL; + struct iatt *stbuf = NULL; + inode_t *inode = NULL; + xlator_t *subvol1 = NULL, *subvol2 = NULL; + + local = frame->local; + prev = cookie; + + local->op_errno = op_errno; + + if ((op_ret == -1) && !dht_inode_missing (op_errno)) { + gf_msg_debug (this->name, op_errno, + "subvolume %s returned -1", + prev->this->name); + goto out; + } + + if (local->call_cnt != 1) + goto out; + + ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf); + + if ((!op_ret) && !stbuf) { + goto out; + } + + local->op_ret = 0; + + local->rebalance.target_op_fn = dht_removexattr2; + + /* Phase 2 of migration */ + if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) { + ret = dht_rebalance_complete_check (this, frame); + if (!ret) + return 0; + } + + /* Phase 1 of migration */ + if (IS_DHT_MIGRATION_PHASE1 (stbuf)) { + inode = (local->fd) ? local->fd->inode : local->loc.inode; + ret = dht_inode_ctx_get_mig_info (this, inode, + &subvol1, &subvol2); + if (!dht_mig_info_is_invalid (local->cached_subvol, + subvol1, subvol2)) { + dht_removexattr2 (this, subvol2, frame); + return 0; + } + + ret = dht_rebalance_in_progress_check (this, frame); + if (!ret) + return 0; + } + +out: + if (local->fop == GF_FOP_REMOVEXATTR) { + DHT_STACK_UNWIND (removexattr, frame, op_ret, op_errno, NULL); + } else { + DHT_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL); + } + return 0; + +} + +int +dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame) +{ + dht_local_t *local = NULL; + int op_errno = EINVAL; + + if (!frame || !frame->local || !subvol) + goto err; + + local = frame->local; + + local->call_cnt = 2; /* This is the second attempt */ + + if (local->fop == GF_FOP_REMOVEXATTR) { + STACK_WIND (frame, dht_file_removexattr_cbk, subvol, + subvol->fops->removexattr, &local->loc, + local->key, NULL); + } else { + STACK_WIND (frame, dht_file_removexattr_cbk, subvol, + subvol->fops->fremovexattr, local->fd, + local->key, NULL); + } + + return 0; + +err: + DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL); + return 0; +} + + int dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, int op_errno, dict_t *xdata) @@ -3630,6 +3911,8 @@ dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, unlock: UNLOCK (&frame->lock); + + this_call_cnt = dht_frame_return (frame); if (is_last_call (this_call_cnt)) { DHT_STACK_UNWIND (removexattr, frame, local->op_ret, @@ -3651,6 +3934,7 @@ dht_removexattr (call_frame_t *frame, xlator_t *this, int call_cnt = 0; dht_conf_t *conf = NULL; int i; + int ret = 0; VALIDATE_OR_GOTO (this, err); VALIDATE_OR_GOTO (this->private, err); @@ -3688,11 +3972,33 @@ dht_removexattr (call_frame_t *frame, xlator_t *this, local->call_cnt = call_cnt = layout->cnt; local->key = gf_strdup (key); - for (i = 0; i < call_cnt; i++) { - STACK_WIND (frame, dht_removexattr_cbk, - layout->list[i].xlator, - layout->list[i].xlator->fops->removexattr, - loc, key, NULL); + if (IA_ISDIR (loc->inode->ia_type)) { + for (i = 0; i < call_cnt; i++) { + STACK_WIND (frame, dht_removexattr_cbk, + layout->list[i].xlator, + layout->list[i].xlator->fops->removexattr, + loc, key, NULL); + } + + } else { + + local->call_cnt = 1; + xdata = xdata ? dict_ref (xdata) : dict_new (); + if (xdata) + ret = dict_set_dynstr_with_alloc (xdata, + DHT_IATT_IN_XDATA_KEY, "yes"); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, "Failed to " + "set dictionary key %s for %s", + DHT_IATT_IN_XDATA_KEY, loc->path); + } + + STACK_WIND (frame, dht_file_removexattr_cbk, + subvol, subvol->fops->removexattr, + loc, key, xdata); + + if (xdata) + dict_unref (xdata); } return 0; @@ -3714,6 +4020,7 @@ dht_fremovexattr (call_frame_t *frame, xlator_t *this, dht_layout_t *layout = NULL; int call_cnt = 0; dht_conf_t *conf = 0; + int ret = 0; int i; @@ -3753,11 +4060,33 @@ dht_fremovexattr (call_frame_t *frame, xlator_t *this, local->call_cnt = call_cnt = layout->cnt; local->key = gf_strdup (key); - for (i = 0; i < call_cnt; i++) { - STACK_WIND (frame, dht_removexattr_cbk, - layout->list[i].xlator, - layout->list[i].xlator->fops->fremovexattr, - fd, key, NULL); + if (IA_ISDIR (fd->inode->ia_type)) { + for (i = 0; i < call_cnt; i++) { + STACK_WIND (frame, dht_removexattr_cbk, + layout->list[i].xlator, + layout->list[i].xlator->fops->fremovexattr, + fd, key, NULL); + } + + } else { + + local->call_cnt = 1; + xdata = xdata ? dict_ref (xdata) : dict_new (); + if (xdata) + ret = dict_set_dynstr_with_alloc (xdata, + DHT_IATT_IN_XDATA_KEY, "yes"); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, "Failed to " + "set dictionary key %s for fd=%p", + DHT_IATT_IN_XDATA_KEY, fd); + } + + STACK_WIND (frame, dht_file_removexattr_cbk, + subvol, subvol->fops->fremovexattr, + fd, key, xdata); + + if (xdata) + dict_unref (xdata); } return 0; diff --git a/xlators/cluster/ec/src/ec-combine.c b/xlators/cluster/ec/src/ec-combine.c index b787d9be303..3fc8ab5c015 100644 --- a/xlators/cluster/ec/src/ec-combine.c +++ b/xlators/cluster/ec/src/ec-combine.c @@ -260,6 +260,7 @@ ec_value_ignore (char *key) (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) || (strncmp(key, GF_XATTR_CLRLK_CMD, strlen (GF_XATTR_CLRLK_CMD)) == 0) || + (strcmp(key, DHT_IATT_IN_XDATA_KEY) == 0) || (strncmp(key, EC_QUOTA_PREFIX, strlen(EC_QUOTA_PREFIX)) == 0) || (fnmatch(MARKER_XATTR_PREFIX ".*." XTIME, key, 0) == 0) || (fnmatch(GF_XATTR_MARKER_KEY ".*", key, 0) == 0) || diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c index fbbeacd28b1..71b83c2e0ac 100644 --- a/xlators/storage/posix/src/posix.c +++ b/xlators/storage/posix/src/posix.c @@ -3239,6 +3239,31 @@ map_xattr_flags(int flags) } #endif +static +int32_t posix_set_iatt_in_dict (dict_t *dict, struct iatt *in_stbuf) +{ + int ret = -1; + struct iatt *stbuf = NULL; + int32_t len = sizeof(struct iatt); + + if (!dict || !in_stbuf) + return ret; + + stbuf = GF_CALLOC (1, len, gf_common_mt_char); + if (!stbuf) + return ret; + + memcpy (stbuf, in_stbuf, len); + + ret = dict_set_bin (dict, DHT_IATT_IN_XDATA_KEY, stbuf, len); + if (ret) + GF_FREE (stbuf); + + return ret; +} + + + int32_t posix_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict, int flags, dict_t *xdata) @@ -3246,7 +3271,9 @@ posix_setxattr (call_frame_t *frame, xlator_t *this, int32_t op_ret = -1; int32_t op_errno = 0; char * real_path = NULL; - + struct iatt stbuf = {0}; + int32_t ret = 0; + dict_t *xattr = NULL; posix_xattr_filler_t filler = {0,}; DECLARE_OLD_FS_ID_VAR; @@ -3265,6 +3292,7 @@ posix_setxattr (call_frame_t *frame, xlator_t *this, } op_ret = -1; + dict_del (dict, GFID_XATTR_KEY); dict_del (dict, GF_XATTR_VOL_ID_KEY); @@ -3280,12 +3308,31 @@ posix_setxattr (call_frame_t *frame, xlator_t *this, if (op_ret < 0) { op_errno = -op_ret; op_ret = -1; + goto out; } +/* + * FIXFIX: Send the stbuf info in the xdata for now + * This is used by DHT to redirect FOPs if the file is being migrated + * Ignore errors for now + */ + if (dict_get (xdata, DHT_IATT_IN_XDATA_KEY)) { + ret = posix_pstat(this, loc->gfid, real_path, &stbuf); + if (ret) + goto out; + + xattr = dict_new(); + if (!xattr) + goto out; + ret = posix_set_iatt_in_dict (xattr, &stbuf); + } out: SET_TO_OLD_FS_ID (); - STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL); + STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xattr); + + if (xattr) + dict_unref(xattr); return 0; } @@ -4313,13 +4360,14 @@ int32_t posix_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict, int flags, dict_t *xdata) { - int32_t op_ret = -1; - int32_t op_errno = 0; - struct posix_fd * pfd = NULL; - int _fd = -1; - int ret = -1; - - posix_xattr_filler_t filler = {0,}; + int32_t op_ret = -1; + int32_t op_errno = 0; + struct posix_fd *pfd = NULL; + int _fd = -1; + int ret = -1; + struct iatt stbuf = {0,}; + dict_t *xattr = NULL; + posix_xattr_filler_t filler = {0,}; DECLARE_OLD_FS_ID_VAR; SET_FS_ID (frame->root->uid, frame->root->gid); @@ -4366,10 +4414,28 @@ posix_fsetxattr (call_frame_t *frame, xlator_t *this, } } + if (dict_get (xdata, DHT_IATT_IN_XDATA_KEY)) { + ret = posix_fdstat (this, pfd->fd, &stbuf); + if (ret == -1) { + gf_log (this->name, GF_LOG_ERROR, + "fsetxattr (fstat) failed on fd=%p: %s", + fd, strerror (op_errno)); + goto out; + } + + xattr = dict_new (); + if (!xattr) + goto out; + ret = posix_set_iatt_in_dict (xattr, &stbuf); + } + out: SET_TO_OLD_FS_ID (); - STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL); + STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xattr); + + if (xattr) + dict_unref (xattr); return 0; } @@ -4429,7 +4495,10 @@ posix_removexattr (call_frame_t *frame, xlator_t *this, { int32_t op_ret = -1; int32_t op_errno = 0; + int32_t ret = -1; char * real_path = NULL; + struct iatt stbuf = {0}; + dict_t *xattr = NULL; posix_xattr_filler_t filler = {0,}; DECLARE_OLD_FS_ID_VAR; @@ -4485,12 +4554,26 @@ posix_removexattr (call_frame_t *frame, xlator_t *this, goto out; } + if (dict_get (xdata, DHT_IATT_IN_XDATA_KEY)) { + ret = posix_pstat(this, loc->gfid, real_path, &stbuf); + if (ret) + goto out; + xattr = dict_new(); + if (!xattr) + goto out; + + ret = posix_set_iatt_in_dict (xattr, &stbuf); + } op_ret = 0; out: SET_TO_OLD_FS_ID (); - STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, NULL); + STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xattr); + + if (xattr) + dict_unref (xattr); + return 0; } @@ -4501,6 +4584,8 @@ posix_fremovexattr (call_frame_t *frame, xlator_t *this, int32_t op_ret = -1; int32_t op_errno = 0; struct posix_fd * pfd = NULL; + struct iatt stbuf = {0,}; + dict_t *xattr = NULL; int _fd = -1; int ret = -1; @@ -4541,12 +4626,26 @@ posix_fremovexattr (call_frame_t *frame, xlator_t *this, goto out; } + if (dict_get (xdata, DHT_IATT_IN_XDATA_KEY)) { + ret = posix_fdstat (this, pfd->fd, &stbuf); + if (ret) + goto out; + xattr = dict_new(); + if (!xattr) + goto out; + + ret = posix_set_iatt_in_dict (xattr, &stbuf); + } op_ret = 0; out: SET_TO_OLD_FS_ID (); - STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, NULL); + STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xattr); + + if (xattr) + dict_unref (xattr); + return 0; } -- cgit