From bc91dd4de39ffd481a52b837f322f6782c14e9f1 Mon Sep 17 00:00:00 2001 From: Xavier Hernandez Date: Fri, 7 Nov 2014 12:12:19 +0100 Subject: ec: Fix self-healing issues. Three problems have been detected: 1. Self healing is executed in background, allowing the fop that detected the problem to continue without blocks nor delays. While this is quite interesting to avoid unnecessary delays, it can cause spurious failures of self-heal because it may try to recover a file inside a directory that a previous self-heal has not recovered yet, causing the file self-heal to fail. 2. When a partial self-heal is being executed on a directory, if a full self-heal is attempted, it won't be executed because another self-heal is already in process, so the directory won't be fully repaired. 3. Information contained in loc's of some fop's is not enough to do a complete self-heal. To solve these problems, I've made some changes: * Improved ec_loc_from_loc() to add all available information to a loc. * Before healing an entry, it's parent is checked and partially healed if necessary to avoid failures. * All heal requests received for the same inode while another self-heal is being processed are queued. When the first heal completes, all pending requests are answered using the results of the first heal (without full execution), unless the first heal was a partial heal. In this case all partial heals are answered, and the first full heal is processed normally. * An special virtual xattr (not physically stored on bricks) named 'trusted.ec.heal' has been created to allow synchronous self-heal of files. Now, the recommended way to heal an entire volume is this: find -d -exec getfattr -h -n trusted.ec.heal {} \; Some minor changes: * ec_loc_prepare() has been renamed to ec_loc_update(). * All loc management functions return 0 on success and -1 on error. * Do not delay fop unlocks if heal is needed. * Added basic ec xattrs initially on create, mkdir and mknod fops. * Some coding style changes Change-Id: I2a5fd9c57349a153710880d6ac4b1fa0c1475985 BUG: 1161588 Signed-off-by: Xavier Hernandez Reviewed-on: http://review.gluster.org/9072 Tested-by: Gluster Build System Reviewed-by: Dan Lambright --- xlators/cluster/ec/src/ec-dir-write.c | 150 ++++++++++++++++++++++------------ 1 file changed, 96 insertions(+), 54 deletions(-) (limited to 'xlators/cluster/ec/src/ec-dir-write.c') diff --git a/xlators/cluster/ec/src/ec-dir-write.c b/xlators/cluster/ec/src/ec-dir-write.c index 3d8055c40a7..2b1064a98f0 100644 --- a/xlators/cluster/ec/src/ec-dir-write.c +++ b/xlators/cluster/ec/src/ec-dir-write.c @@ -149,9 +149,8 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state) LOCK(&fop->fd->lock); ctx = __ec_fd_get(fop->fd, fop->xl); - if ((ctx == NULL) || !ec_loc_from_loc(fop->xl, &ctx->loc, - &fop->loc[0])) - { + if ((ctx == NULL) || + (ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0])) != 0) { UNLOCK(&fop->fd->lock); fop->error = EIO; @@ -194,6 +193,18 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state) return EC_STATE_REPORT; } + if (ec_dict_set_number(fop->xdata, EC_XATTR_VERSION, 0) != 0) { + fop->error = EIO; + + return EC_STATE_REPORT; + } + + if (ec_dict_set_number(fop->xdata, EC_XATTR_SIZE, 0) != 0) { + fop->error = EIO; + + return EC_STATE_REPORT; + } + fop->int32 &= ~O_ACCMODE; fop->int32 |= O_RDWR; @@ -222,27 +233,27 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state) cbk->op_errno = EIO; } } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { + if (cbk->op_ret >= 0) { ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); - ec_loc_prepare(fop->xl, &fop->loc[0], cbk->inode, - &cbk->iatt[0]); + if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]) != 0) { + cbk->op_ret = -1; + cbk->op_errno = EIO; + } else { + LOCK(&fop->fd->lock); - LOCK(&fop->fd->lock); + ctx = __ec_fd_get(fop->fd, fop->xl); + if (ctx != NULL) { + ctx->open |= cbk->mask; + } - ctx = __ec_fd_get(fop->fd, fop->xl); - if (ctx != NULL) - { - ctx->open |= cbk->mask; + UNLOCK(&fop->fd->lock); } - - UNLOCK(&fop->fd->lock); + } + if (cbk->op_ret < 0) { + ec_fop_set_error(fop, cbk->op_errno); } } else @@ -511,22 +522,21 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state) cbk->op_errno = EIO; } } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { + if (cbk->op_ret >= 0) { ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); - - ec_loc_prepare(fop->xl, &fop->loc[0], cbk->inode, - &cbk->iatt[0]); - - if (cbk->iatt[0].ia_type == IA_IFREG) - { + if (cbk->iatt[0].ia_type == IA_IFREG) { cbk->iatt[0].ia_size = fop->pre_size; } + + if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]) != 0) { + cbk->op_ret = -1; + cbk->op_errno = EIO; + } + } + if (cbk->op_ret < 0) { + ec_fop_set_error(fop, cbk->op_errno); } } else @@ -754,6 +764,23 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state) switch (state) { case EC_STATE_INIT: + if (fop->xdata == NULL) { + fop->xdata = dict_new(); + if (fop->xdata == NULL) { + fop->error = EIO; + + return EC_STATE_REPORT; + } + } + + if (ec_dict_set_number(fop->xdata, EC_XATTR_VERSION, 0) != 0) { + fop->error = EIO; + + return EC_STATE_REPORT; + } + + /* Fall through */ + case EC_STATE_LOCK: ec_lock_prepare_entry(fop, &fop->loc[0], 1); ec_lock(fop); @@ -777,17 +804,18 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state) cbk->op_errno = EIO; } } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { + if (cbk->op_ret >= 0) { ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); - ec_loc_prepare(fop->xl, &fop->loc[0], cbk->inode, - &cbk->iatt[0]); + if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]) != 0) { + cbk->op_ret = -1; + cbk->op_errno = EIO; + } + } + if (cbk->op_ret < 0) { + ec_fop_set_error(fop, cbk->op_errno); } } else @@ -1037,6 +1065,18 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state) return EC_STATE_REPORT; } + + if (ec_dict_set_number(fop->xdata, EC_XATTR_VERSION, 0) != 0) { + fop->error = EIO; + + return EC_STATE_REPORT; + } + + if (ec_dict_set_number(fop->xdata, EC_XATTR_SIZE, 0) != 0) { + fop->error = EIO; + + return EC_STATE_REPORT; + } } /* Fall through */ @@ -1064,17 +1104,18 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state) cbk->op_errno = EIO; } } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { + if (cbk->op_ret >= 0) { ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); - ec_loc_prepare(fop->xl, &fop->loc[0], cbk->inode, - &cbk->iatt[0]); + if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]) != 0) { + cbk->op_ret = -1; + cbk->op_errno = EIO; + } + } + if (cbk->op_ret < 0) { + ec_fop_set_error(fop, cbk->op_errno); } } else @@ -1822,17 +1863,18 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state) cbk->op_errno = EIO; } } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { + if (cbk->op_ret >= 0) { ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); - ec_loc_prepare(fop->xl, &fop->loc[0], cbk->inode, - &cbk->iatt[0]); + if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]) != 0) { + cbk->op_ret = -1; + cbk->op_errno = EIO; + } + } + if (cbk->op_ret < 0) { + ec_fop_set_error(fop, cbk->op_errno); } } else -- cgit