diff options
| -rw-r--r-- | tests/afr.rc | 68 | ||||
| -rw-r--r-- | tests/bugs/bug-1130892.t | 60 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-common.c | 35 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-dir-write.c | 30 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-self-heal-common.c | 37 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-self-heal-entry.c | 73 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-self-heal-name.c | 39 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-self-heal.h | 8 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr.h | 16 |
9 files changed, 276 insertions, 90 deletions
diff --git a/tests/afr.rc b/tests/afr.rc index 60ea1b7f123..938f9c07ab2 100644 --- a/tests/afr.rc +++ b/tests/afr.rc @@ -1,5 +1,73 @@ #!/bin/bash +function diff_dirs { + diff <(ls $1 | sort) <(ls $2 | sort) +} + +function heal_status { + local f1_path="${1}/${3}" + local f2_path="${2}/${3}" + local zero_xattr="000000000000000000000000" + local insync="" + diff_dirs $f1_path $f2_path + if [ $? -eq 0 ]; + then + insync="Y" + else + insync="N" + fi + local xattr11=$(get_hex_xattr trusted.afr.$V0-client-0 $f1_path) + local xattr12=$(get_hex_xattr trusted.afr.$V0-client-1 $f1_path) + local xattr21=$(get_hex_xattr trusted.afr.$V0-client-0 $f2_path) + local xattr22=$(get_hex_xattr trusted.afr.$V0-client-1 $f2_path) + local dirty1=$(get_hex_xattr trusted.afr.dirty $f1_path) + local dirty2=$(get_hex_xattr trusted.afr.dirty $f2_path) + if [ -z $xattr11 ]; then xattr11="000000000000000000000000"; fi + if [ -z $xattr12 ]; then xattr12="000000000000000000000000"; fi + if [ -z $xattr21 ]; then xattr21="000000000000000000000000"; fi + if [ -z $xattr22 ]; then xattr22="000000000000000000000000"; fi + if [ -z $dirty1 ]; then dirty1="000000000000000000000000"; fi + if [ -z $dirty2 ]; then dirty2="000000000000000000000000"; fi + echo ${insync}${xattr11}${xattr12}${xattr21}${xattr22}${dirty1}${dirty2} +} +# Check if given dir's self-heal is done +function is_dir_heal_done { + local zero_xattr="000000000000000000000000" + if [ "$(heal_status $@)" == "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" ]; + then + echo "Y" + else + echo "N" + fi +} +# Check if the given file's self-heal is done +function is_file_heal_done { + local f1_path="${1}/${3}" + local f2_path="${2}/${3}" + local zxattr="000000000000000000000000" + local size1=$(stat -c "%s" $f1_path) + local size2=$(stat -c "%s" $f2_path) + local diff=$((size1-size2)) + local x11=$(get_hex_xattr trusted.afr.$V0-client-0 $f1_path) + local x12=$(get_hex_xattr trusted.afr.$V0-client-1 $f1_path) + local x21=$(get_hex_xattr trusted.afr.$V0-client-0 $f2_path) + local x22=$(get_hex_xattr trusted.afr.$V0-client-1 $f2_path) + local dirty1=$(get_hex_xattr trusted.afr.dirty $f1_path) + local dirty2=$(get_hex_xattr trusted.afr.dirty $f2_path) + if [ -z $x11 ]; then x11="000000000000000000000000"; fi + if [ -z $x12 ]; then x12="000000000000000000000000"; fi + if [ -z $x21 ]; then x21="000000000000000000000000"; fi + if [ -z $x22 ]; then x22="000000000000000000000000"; fi + if [ -z $dirty1 ]; then dirty1="000000000000000000000000"; fi + if [ -z $dirty2 ]; then dirty2="000000000000000000000000"; fi + if [ "${diff}${x11}${x12}${x21}${x22}${dirty1}${dirty2}" == "0${zxattr}${zxattr}${zxattr}${zxattr}${zxattr}${zxattr}" ]; + then + echo "Y" + else + echo "N" + fi +} + #count the number of entries marked for self-heal #in brick $1's index diff --git a/tests/bugs/bug-1130892.t b/tests/bugs/bug-1130892.t new file mode 100644 index 00000000000..438e795d571 --- /dev/null +++ b/tests/bugs/bug-1130892.t @@ -0,0 +1,60 @@ +#!/bin/bash +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc +. $(dirname $0)/../afr.rc + +cleanup; + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info; + +# Create a 1X2 replica +TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}-{0,1} +EXPECT 'Created' volinfo_field $V0 'Status'; + +# Disable self-heal daemon +TEST gluster volume set $V0 self-heal-daemon off + +# Disable all perf-xlators +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 performance.stat-prefetch off +TEST $CLI volume set $V0 performance.read-ahead off + +# Volume start +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; + +# FUSE Mount +TEST glusterfs -s $H0 --volfile-id $V0 $M0 + +# Create files and dirs +TEST mkdir -p $M0/one/two/ +TEST `echo "Carpe diem" > $M0/one/two/three` + +# Simulate disk-replacement +TEST kill_brick $V0 $H0 $B0/${V0}-1 +TEST rm -rf $B0/${V0}-1/one +TEST rm -rf $B0/${V0}-1/.glusterfs + +# Start force +TEST $CLI volume start $V0 force + +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 1 + +TEST stat $M0/one + +# Check pending xattrs +EXPECT "00000000" afr_get_specific_changelog_xattr $B0/${V0}-0/one trusted.afr.$V0-client-1 data +EXPECT_NOT "00000000" afr_get_specific_changelog_xattr $B0/${V0}-0/one trusted.afr.$V0-client-1 entry +EXPECT_NOT "00000000" afr_get_specific_changelog_xattr $B0/${V0}-0/one trusted.afr.$V0-client-1 metadata + +TEST gluster volume set $V0 self-heal-daemon on +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status +EXPECT_WITHIN $HEAL_TIMEOUT "Y" is_dir_heal_done $B0/${V0}-0 $B0/${V0}-1 one +EXPECT_WITHIN $HEAL_TIMEOUT "Y" is_dir_heal_done $B0/${V0}-0 $B0/${V0}-1 one/two +EXPECT_WITHIN $HEAL_TIMEOUT "Y" is_file_heal_done $B0/${V0}-0 $B0/${V0}-1 one/two/three + +cleanup; diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c index 2a6b0c957fb..e9a05de2546 100644 --- a/xlators/cluster/afr/src/afr-common.c +++ b/xlators/cluster/afr/src/afr-common.c @@ -3684,3 +3684,38 @@ afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this) fd_ctx->open_fd_count = local->open_fd_count; } + +int** +afr_mark_pending_changelog (afr_private_t *priv, unsigned char *pending, + dict_t *xattr, ia_type_t iat) +{ + int i = 0; + int **changelog = NULL; + int idx = -1; + int m_idx = 0; + int ret = 0; + + m_idx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION); + + idx = afr_index_from_ia_type (iat); + + changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS); + if (!changelog) + goto out; + + for (i = 0; i < priv->child_count; i++) { + if (!pending[i]) + continue; + + changelog[i][m_idx] = hton32(1); + if (idx != -1) + changelog[i][idx] = hton32(1); + } + ret = afr_set_pending_dict (priv, xattr, changelog); + if (ret < 0) { + afr_matrix_cleanup (changelog, priv->child_count); + return NULL; + } +out: + return changelog; +} diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c index a9f272126ae..f5c385c34a4 100644 --- a/xlators/cluster/afr/src/afr-dir-write.c +++ b/xlators/cluster/afr/src/afr-dir-write.c @@ -278,8 +278,6 @@ afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this) dict_t *xattr = NULL; int32_t **changelog = NULL; int i = 0; - int idx = -1; - int m_idx = 0; int op_errno = ENOMEM; unsigned char *pending = NULL; int call_count = 0; @@ -295,22 +293,10 @@ afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this) if (!new_local) goto out; - changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS); - if (!changelog) - goto out; - - new_local->pending = changelog; xattr = dict_new (); if (!xattr) goto out; - if (IA_ISREG (local->cont.dir_fop.buf.ia_type)) { - idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION); - } else if (IA_ISDIR (local->cont.dir_fop.buf.ia_type)) { - idx = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION); - } - m_idx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION); - pending = alloca0 (priv->child_count); for (i = 0; i < priv->child_count; i++) { @@ -319,19 +305,19 @@ afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this) call_count ++; continue; } - - changelog[i][m_idx] = hton32(1); - if (idx != -1) - changelog[i][idx] = hton32(1); pending[i] = 1; } + changelog = afr_mark_pending_changelog (priv, pending, xattr, + local->cont.dir_fop.buf.ia_type); + if (!changelog) + goto out; + + new_local->pending = changelog; + changelog = NULL; uuid_copy (new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid); new_local->loc.inode = inode_ref (local->inode); - - afr_set_pending_dict (priv, xattr, changelog); - new_local->call_count = call_count; for (i = 0; i < priv->child_count; i++) { @@ -349,6 +335,8 @@ afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this) new_frame = NULL; out: + if (changelog) + afr_matrix_cleanup (changelog, priv->child_count); if (new_frame) AFR_STACK_DESTROY (new_frame); if (xattr) diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c index 9a88a7d9e5c..ab1552a0224 100644 --- a/xlators/cluster/afr/src/afr-self-heal-common.c +++ b/xlators/cluster/afr/src/afr-self-heal-common.c @@ -975,6 +975,43 @@ afr_frame_create (xlator_t *this) return frame; } +int +afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode, + int source, struct afr_reply *replies, + unsigned char *sources, unsigned char *newentry) +{ + int ret = 0; + int i = 0; + afr_private_t *priv = NULL; + dict_t *xattr = NULL; + int **changelog = NULL; + + priv = this->private; + + uuid_copy (inode->gfid, replies[source].poststat.ia_gfid); + + xattr = dict_new(); + if (!xattr) + return -ENOMEM; + + changelog = afr_mark_pending_changelog (priv, newentry, xattr, + replies[source].poststat.ia_type); + + if (!changelog) + goto out; + + for (i = 0; i < priv->child_count; i++) { + if (!sources[i]) + continue; + afr_selfheal_post_op (frame, this, inode, i, xattr); + } +out: + if (changelog) + afr_matrix_cleanup (changelog, priv->child_count); + if (xattr) + dict_unref (xattr); + return ret; +} /* * This is the entry point for healing a given GFID diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c index cb682a7dccc..0cf65009c5f 100644 --- a/xlators/cluster/afr/src/afr-self-heal-entry.c +++ b/xlators/cluster/afr/src/afr-self-heal-entry.c @@ -69,7 +69,8 @@ afr_selfheal_entry_delete (xlator_t *this, inode_t *dir, const char *name, int afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir, const char *name, inode_t *inode, - struct afr_reply *replies) + struct afr_reply *replies, + unsigned char *newentry) { int ret = 0; loc_t loc = {0,}; @@ -80,7 +81,6 @@ afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir, char *linkname = NULL; mode_t mode = 0; struct iatt newent = {0,}; - priv = this->private; xdata = dict_new(); @@ -111,6 +111,8 @@ afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir, switch (iatt->ia_type) { case IA_IFDIR: ret = syncop_mkdir (priv->children[dst], &loc, mode, xdata, 0); + if (ret == 0) + newentry[dst] = 1; break; case IA_IFLNK: ret = syncop_lookup (priv->children[dst], &srcloc, 0, 0, 0, 0); @@ -123,7 +125,9 @@ afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir, goto out; ret = syncop_symlink (priv->children[dst], &loc, linkname, xdata, NULL); - } + if (ret == 0) + newentry[dst] = 1; + } break; default: ret = dict_set_int32 (xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1); @@ -131,9 +135,9 @@ afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir, goto out; ret = syncop_mknod (priv->children[dst], &loc, mode, iatt->ia_rdev, xdata, &newent); - if (ret == 0 && iatt->ia_size && !newent.ia_size) { + if (ret == 0 && newent.ia_nlink == 1) { /* New entry created. Mark @dst pending on all sources */ - ret = 1; + newentry[dst] = 1; } break; } @@ -148,49 +152,6 @@ out: static int -afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode, - int source, struct afr_reply *replies, - unsigned char *sources, unsigned char *newentry) -{ - int ret = 0; - int i = 0; - afr_private_t *priv = NULL; - dict_t *xattr = NULL; - int **changelog = NULL; - int idx = 0; - - priv = this->private; - - idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION); - - uuid_copy (inode->gfid, replies[source].poststat.ia_gfid); - - changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS); - - xattr = dict_new(); - if (!xattr) - return -ENOMEM; - - for (i = 0; i < priv->child_count; i++) { - if (!newentry[i]) - continue; - changelog[i][idx] = hton32(1); - } - - afr_set_pending_dict (priv, xattr, changelog); - - for (i = 0; i < priv->child_count; i++) { - if (!sources[i]) - continue; - afr_selfheal_post_op (frame, this, inode, i, xattr); - } - - dict_unref (xattr); - return ret; -} - - -static int __afr_selfheal_heal_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd, char *name, inode_t *inode, int source, unsigned char *sources, unsigned char *healed_sinks, @@ -202,6 +163,7 @@ __afr_selfheal_heal_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd, unsigned char *newentry = NULL; priv = this->private; + newentry = alloca0 (priv->child_count); if (!replies[source].valid) @@ -221,11 +183,7 @@ __afr_selfheal_heal_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd, ret = afr_selfheal_recreate_entry (this, i, source, fd->inode, name, inode, - replies); - if (ret > 0) { - newentry[i] = 1; - ret = 0; - } + replies, newentry); } if (ret < 0) break; @@ -248,9 +206,12 @@ __afr_selfheal_merge_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd, afr_private_t *priv = NULL; int i = 0; int source = -1; + unsigned char *newentry = NULL; priv = this->private; + newentry = alloca0 (priv->child_count); + for (i = 0; i < priv->child_count; i++) { if (replies[i].valid && replies[i].op_ret == 0) { source = i; @@ -271,9 +232,13 @@ __afr_selfheal_merge_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd, continue; ret = afr_selfheal_recreate_entry (this, i, source, fd->inode, - name, inode, replies); + name, inode, replies, + newentry); } + if (AFR_COUNT (newentry, priv->child_count)) + afr_selfheal_newentry_mark (frame, this, inode, source, replies, + sources, newentry); return ret; } diff --git a/xlators/cluster/afr/src/afr-self-heal-name.c b/xlators/cluster/afr/src/afr-self-heal-name.c index f8b887e9a24..a3020f4e1a7 100644 --- a/xlators/cluster/afr/src/afr-self-heal-name.c +++ b/xlators/cluster/afr/src/afr-self-heal-name.c @@ -17,7 +17,6 @@ #include "afr.h" #include "afr-self-heal.h" - int __afr_selfheal_assign_gfid (xlator_t *this, inode_t *parent, uuid_t pargfid, const char *bname, inode_t *inode, @@ -105,17 +104,22 @@ out: return ret; } - int -__afr_selfheal_name_impunge (xlator_t *this, inode_t *parent, uuid_t pargfid, +__afr_selfheal_name_impunge (call_frame_t *frame, xlator_t *this, + inode_t *parent, uuid_t pargfid, const char *bname, inode_t *inode, struct afr_reply *replies, int gfid_idx) { int i = 0; afr_private_t *priv = NULL; - int ret = 0; + int ret = 0; + unsigned char *newentry = NULL; + unsigned char *sources = NULL; - priv = this->private; + priv = this->private; + + newentry = alloca0 (priv->child_count); + sources = alloca0 (priv->child_count); uuid_copy (parent->gfid, pargfid); @@ -124,13 +128,19 @@ __afr_selfheal_name_impunge (xlator_t *this, inode_t *parent, uuid_t pargfid, continue; if (uuid_compare (replies[i].poststat.ia_gfid, - replies[gfid_idx].poststat.ia_gfid) == 0) + replies[gfid_idx].poststat.ia_gfid) == 0) { + sources[i] = 1; continue; + } ret |= afr_selfheal_recreate_entry (this, i, gfid_idx, parent, - bname, inode, replies); + bname, inode, replies, + newentry); } + if (AFR_COUNT (newentry, priv->child_count)) + afr_selfheal_newentry_mark (frame, this, inode, gfid_idx, replies, + sources, newentry); return ret; } @@ -381,8 +391,8 @@ out: } int -__afr_selfheal_name_do (xlator_t *this, inode_t *parent, uuid_t pargfid, - const char *bname, inode_t *inode, +__afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent, + uuid_t pargfid, const char *bname, inode_t *inode, unsigned char *sources, unsigned char *sinks, unsigned char *healed_sinks, int source, unsigned char *locked_on, struct afr_reply *replies, @@ -438,7 +448,8 @@ __afr_selfheal_name_do (xlator_t *this, inode_t *parent, uuid_t pargfid, return -1; } - return __afr_selfheal_name_impunge (this, parent, pargfid, bname, inode, + return __afr_selfheal_name_impunge (frame, this, parent, pargfid, + bname, inode, replies, gfid_idx); } @@ -579,10 +590,10 @@ afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent, goto unlock; } - ret = __afr_selfheal_name_do (this, parent, pargfid, bname, - inode, sources, sinks, healed_sinks, - source, locked_on, replies, - gfid_req); + ret = __afr_selfheal_name_do (frame, this, parent, pargfid, + bname, inode, sources, sinks, + healed_sinks, source, locked_on, + replies, gfid_req); } unlock: afr_selfheal_unentrylk (frame, this, parent, this->name, bname, diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h index 84da0d8000f..2713ffa09bf 100644 --- a/xlators/cluster/afr/src/afr-self-heal.h +++ b/xlators/cluster/afr/src/afr-self-heal.h @@ -152,7 +152,8 @@ afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode, int afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir, const char *name, inode_t *inode, - struct afr_reply *replies); + struct afr_reply *replies, + unsigned char *newentry); int afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode, @@ -173,4 +174,9 @@ afr_selfheal_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this, void afr_replies_copy (struct afr_reply *dst, struct afr_reply *src, int count); +int +afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode, + int source, struct afr_reply *replies, + unsigned char *sources, unsigned char *newentry); + #endif /* !_AFR_SELFHEAL_H */ diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h index 8e7bea79712..4adbd5d5c83 100644 --- a/xlators/cluster/afr/src/afr.h +++ b/xlators/cluster/afr/src/afr.h @@ -187,6 +187,18 @@ afr_index_for_transaction_type (afr_transaction_type type) return -1; /* make gcc happy */ } +static inline int +afr_index_from_ia_type (ia_type_t type) +{ + switch (type) { + case IA_IFDIR: + return afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION); + case IA_IFREG: + return afr_index_for_transaction_type (AFR_DATA_TRANSACTION); + default: return -1; + } +} + typedef struct { loc_t loc; char *basename; @@ -935,6 +947,10 @@ afr_matrix_cleanup (int32_t **pending, unsigned int m); int32_t** afr_matrix_create (unsigned int m, unsigned int n); +int** +afr_mark_pending_changelog (afr_private_t *priv, unsigned char *pending, + dict_t *xattr, ia_type_t iat); + void afr_filter_xattrs (dict_t *xattr); |