From 6c707f442a74002356187bbd1c719b0f80dfebad Mon Sep 17 00:00:00 2001 From: Ravishankar N Date: Mon, 30 Jan 2017 09:54:16 +0530 Subject: afr: all children of AFR must be up to resolve s-brain Problem: The various split-brain resolution policies (favorite-child-policy based, CLI based and mount (get/setfattr) based) attempt to resolve split-brain even when not all bricks of replica are up. This can be a problem when say in a replica 3, the only good copy is down and the other 2 bricks are up and blame each other (i.e. split-brain). We end up healing the file in such a case and allow I/O on it. Fix: A decision on whether the file is in split-brain or not must be taken only if we are able to examine the afr xattrs of *all* bricks of a given replica. > Reviewed-on: https://review.gluster.org/16476 > Smoke: Gluster Build System > NetBSD-regression: NetBSD Build System > CentOS-regression: Gluster Build System > Reviewed-by: Pranith Kumar Karampuri (cherry picked from commit 0e03336a9362e5717e561f76b0c543e5a197b31b) Change-Id: Icddb1268b380005799990f5379ef957d84639ef9 BUG: 1420984 Signed-off-by: Ravishankar N Reviewed-on: https://review.gluster.org/16589 NetBSD-regression: NetBSD Build System CentOS-regression: Gluster Build System Smoke: Gluster Build System Reviewed-by: Pranith Kumar Karampuri --- xlators/cluster/afr/src/afr-common.c | 32 +++++++++++++++------- xlators/cluster/afr/src/afr-self-heal-common.c | 38 +++++++++++++++++++++++--- xlators/cluster/afr/src/afr-self-heal.h | 6 +++- 3 files changed, 61 insertions(+), 15 deletions(-) (limited to 'xlators') diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c index 4d9327b8a11..c324c02c008 100644 --- a/xlators/cluster/afr/src/afr-common.c +++ b/xlators/cluster/afr/src/afr-common.c @@ -661,14 +661,17 @@ afr_set_split_brain_choice (int ret, call_frame_t *frame, void *opaque) gf_boolean_t timer_reset = _gf_false; int old_spb_choice = -1; - if (ret) - goto out; - frame = data->frame; loc = data->loc; this = frame->this; priv = this->private; + if (ret) { + op_errno = -ret; + ret = -1; + goto out; + } + delta.tv_sec = priv->spb_choice_timeout; delta.tv_nsec = 0; @@ -5264,6 +5267,12 @@ afr_is_split_brain (call_frame_t *frame, xlator_t *this, inode_t *inode, if (ret) goto out; + if (!afr_can_decide_split_brain_source_sinks (replies, + priv->child_count)) { + ret = -EAGAIN; + goto out; + } + ret = _afr_is_split_brain (frame, this, replies, AFR_DATA_TRANSACTION, d_spb); if (ret) @@ -5316,6 +5325,13 @@ afr_get_split_brain_status (void *opaque) if (!inode) goto out; + dict = dict_new (); + if (!dict) { + op_errno = ENOMEM; + ret = -1; + goto out; + } + /* Calculation for string length : * (child_count X length of child-name) + strlen (" Choices :") * child-name consists of : @@ -5329,13 +5345,9 @@ afr_get_split_brain_status (void *opaque) &m_spb); if (ret) { op_errno = -ret; - ret = -1; - goto out; - } - - dict = dict_new (); - if (!dict) { - op_errno = ENOMEM; + if (ret == -EAGAIN) + ret = dict_set_str (dict, GF_AFR_SBRAIN_STATUS, + SBRAIN_HEAL_NO_GO_MSG); ret = -1; goto out; } diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c index 40cfd01f4e9..c11ca11fdd9 100644 --- a/xlators/cluster/afr/src/afr-self-heal-common.c +++ b/xlators/cluster/afr/src/afr-self-heal-common.c @@ -472,6 +472,19 @@ afr_dict_contains_heal_op (call_frame_t *frame) return _gf_true; } +gf_boolean_t +afr_can_decide_split_brain_source_sinks (struct afr_reply *replies, + int child_count) +{ + int i = 0; + + for (i = 0; i < child_count; i++) + if (replies[i].valid != 1 || replies[i].op_ret != 0) + return _gf_false; + + return _gf_true; +} + int afr_mark_split_brain_source_sinks_by_heal_op (call_frame_t *frame, xlator_t *this, unsigned char *sources, @@ -510,6 +523,14 @@ afr_mark_split_brain_source_sinks_by_heal_op (call_frame_t *frame, } xdata_rsp = local->xdata_rsp; + if (!afr_can_decide_split_brain_source_sinks (replies, + priv->child_count)) { + ret = dict_set_str (xdata_rsp, "sh-fail-msg", + SBRAIN_HEAL_NO_GO_MSG); + ret = -1; + goto out; + } + for (i = 0 ; i < priv->child_count; i++) if (locked_on[i]) sources[i] = 1; @@ -748,26 +769,35 @@ afr_sh_get_fav_by_policy (xlator_t *this, struct afr_reply *replies, int fav_child = -1; priv = this->private; + if (!afr_can_decide_split_brain_source_sinks (replies, + priv->child_count)) { + return -1; + } + switch (priv->fav_child_policy) { case AFR_FAV_CHILD_BY_SIZE: fav_child = afr_sh_fav_by_size (this, replies, inode); - if (policy_str && fav_child >= 0) + if (policy_str && fav_child >= 0) { *policy_str = "SIZE"; + } break; case AFR_FAV_CHILD_BY_CTIME: fav_child = afr_sh_fav_by_ctime (this, replies, inode); - if (policy_str && fav_child >= 0) + if (policy_str && fav_child >= 0) { *policy_str = "CTIME"; + } break; case AFR_FAV_CHILD_BY_MTIME: fav_child = afr_sh_fav_by_mtime (this, replies, inode); - if (policy_str && fav_child >= 0) + if (policy_str && fav_child >= 0) { *policy_str = "MTIME"; + } break; case AFR_FAV_CHILD_BY_MAJORITY: fav_child = afr_sh_fav_by_majority (this, replies, inode); - if (policy_str && fav_child >= 0) + if (policy_str && fav_child >= 0) { *policy_str = "MAJORITY"; + } break; case AFR_FAV_CHILD_NONE: default: diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h index 500227abe24..a33905033cc 100644 --- a/xlators/cluster/afr/src/afr-self-heal.h +++ b/xlators/cluster/afr/src/afr-self-heal.h @@ -81,7 +81,8 @@ #define IA_EQUAL(f,s,field) (memcmp (&(f.ia_##field), &(s.ia_##field), sizeof (s.ia_##field)) == 0) - +#define SBRAIN_HEAL_NO_GO_MSG "Failed to obtain replies from all bricks of "\ + "the replica (are they up?). Cannot resolve split-brain." int afr_selfheal (xlator_t *this, uuid_t gfid); @@ -220,6 +221,9 @@ afr_mark_active_sinks (xlator_t *this, unsigned char *sources, gf_boolean_t afr_dict_contains_heal_op (call_frame_t *frame); +gf_boolean_t +afr_can_decide_split_brain_source_sinks (struct afr_reply *replies, + int child_count); int afr_mark_split_brain_source_sinks (call_frame_t *frame, xlator_t *this, inode_t *inode, -- cgit