diff options
| -rw-r--r-- | tests/bugs/bug-1368312.t | 84 | ||||
| -rw-r--r-- | xlators/cluster/dht/src/dht-common.c | 212 | ||||
| -rw-r--r-- | xlators/cluster/dht/src/dht-common.h | 10 |
3 files changed, 294 insertions, 12 deletions
diff --git a/tests/bugs/bug-1368312.t b/tests/bugs/bug-1368312.t new file mode 100644 index 00000000000..135048f448e --- /dev/null +++ b/tests/bugs/bug-1368312.t @@ -0,0 +1,84 @@ +#!/bin/bash +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc +cleanup; + +function compare_get_split_brain_status { + local path=$1 + local choice=$2 + echo `getfattr -n replica.split-brain-status $path` | cut -f2 -d"=" | sed -e 's/^"//' -e 's/"$//' | grep $choice + if [ $? -ne 0 ] + then + echo 1 + else + echo 0 + fi + +} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1,2,3,4,5} +TEST $CLI volume start $V0 + +#Disable self-heal-daemon +TEST $CLI volume set $V0 cluster.self-heal-daemon off + +TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0; + +TEST mkdir $M0/tmp1 + +#Create metadata split-brain +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST chmod 666 $M0/tmp1 +TEST $CLI volume start $V0 force +TEST kill_brick $V0 $H0 $B0/${V0}1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 + +TEST chmod 757 $M0/tmp1 + +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 + +EXPECT 2 get_pending_heal_count $V0 + + +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST chmod 755 $M0/tmp1 +TEST $CLI volume start $V0 force +TEST kill_brick $V0 $H0 $B0/${V0}3 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2 + +TEST chmod 766 $M0/tmp1 + +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 3 + +EXPECT 4 get_pending_heal_count $V0 + +TEST kill_brick $V0 $H0 $B0/${V0}4 +TEST chmod 765 $M0/tmp1 +TEST $CLI volume start $V0 force +TEST kill_brick $V0 $H0 $B0/${V0}5 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 4 + +TEST chmod 756 $M0/tmp1 + +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 4 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 5 + +EXPECT 6 get_pending_heal_count $V0 + +cd $M0 +EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-0 +EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-1 +EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-2 +EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-3 +EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-4 +EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-5 + +cd - +cleanup diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c index c4326fd075f..65f4f3d2ef7 100644 --- a/xlators/cluster/dht/src/dht-common.c +++ b/xlators/cluster/dht/src/dht-common.c @@ -127,6 +127,190 @@ out: } +int add_opt(char **optsp, const char *opt) +{ + char *newopts = NULL; + unsigned oldsize = 0; + unsigned newsize = 0; + + if (*optsp == NULL) + newopts = gf_strdup (opt); + else { + oldsize = strlen (*optsp); + newsize = oldsize + 1 + strlen (opt) + 1; + newopts = GF_REALLOC (*optsp, newsize); + if (newopts) + sprintf (newopts + oldsize, ",%s", opt); + } + if (newopts == NULL) { + gf_msg ("dht", GF_LOG_WARNING, 0, + DHT_MSG_NO_MEMORY, + "Error to add choices in buffer in add_opt"); + return -1; + } + *optsp = newopts; + return 0; +} + +/* Return Choice list from Split brain status */ +char * +getChoices (const char *value) +{ + int i = 0; + char *ptr = NULL; + char *tok = NULL; + char *result = NULL; + char *newval = NULL; + + ptr = strstr (value, "Choices:"); + if (!ptr) { + result = ptr; + goto out; + } + + newval = gf_strdup (ptr); + if (!newval) { + result = newval; + goto out; + } + + tok = strtok (newval, ":"); + if (!tok) { + result = tok; + goto out; + } + + while (tok) { + i++; + if (i == 2) + break; + tok = strtok (NULL, ":"); + } + + result = gf_strdup (tok); + +out: + if (newval) + GF_FREE (newval); + + return result; +} + +/* This function prepare a list of choices for key + (replica.split-brain-status) in case of metadata split brain + only on the basis of key-value passed to this function. + After prepare the list of choices it update the same key in dict + with this value to reflect the same in + replica.split-brain-status attr for file. + +*/ + +int +dht_aggregate_split_brain_xattr (dict_t *dst, char *key, data_t *value) +{ + + int ret = 0; + char *oldvalue = NULL; + char *old_choice = NULL; + char *new_choice = NULL; + char *full_choice = NULL; + char *status = NULL; + + if (value == NULL) { + gf_msg ("dht", GF_LOG_WARNING, 0, + DHT_MSG_DATA_NULL, + "GF_AFR_SBRAIN_STATUS value is NULL"); + ret = -1; + goto out; + } + + ret = dict_get_str (dst, key, &oldvalue); + if (ret) + goto out; + + if (oldvalue && (strstr (oldvalue, "not"))) { + gf_msg_debug ("dht", 0, + "Need to update split-brain status in dict"); + ret = -1; + goto out; + } + if (oldvalue && (strstr (oldvalue, "metadata-split-brain:yes")) + && (strstr (oldvalue, "data-split-brain:no"))) { + if (strstr (value->data, "not")) { + gf_msg_debug ("dht", 0, + "No need to update split-brain status"); + ret = 0; + goto out; + } + if (strstr (value->data, "yes") && + (strncmp (oldvalue, value->data, strlen(oldvalue)))) { + old_choice = getChoices (oldvalue); + if (!old_choice) { + gf_msg ("dht", GF_LOG_WARNING, 0, + DHT_MSG_NO_MEMORY, + "Error to get choices"); + ret = -1; + goto out; + } + + ret = add_opt (&full_choice, old_choice); + if (ret) { + gf_msg ("dht", GF_LOG_WARNING, 0, + DHT_MSG_NO_MEMORY, + "Error to add choices"); + ret = -1; + goto out; + } + + new_choice = getChoices (value->data); + if (!new_choice) { + gf_msg ("dht", GF_LOG_WARNING, 0, + DHT_MSG_NO_MEMORY, + "Error to get choices"); + ret = -1; + goto out; + } + + ret = add_opt (&full_choice, new_choice); + if (ret) { + gf_msg ("dht", GF_LOG_WARNING, 0, + DHT_MSG_NO_MEMORY, + "Error to add choices "); + ret = -1; + goto out; + } + ret = gf_asprintf (&status, + "data-split-brain:%s " + "metadata-split-brain:%s Choices:%s", + "no", "yes", full_choice); + + if (-1 == ret) { + gf_msg ("dht", GF_LOG_WARNING, 0, + DHT_MSG_NO_MEMORY, + "Error to prepare status "); + goto out; + } + ret = dict_set_dynstr (dst, key, status); + if (ret) { + gf_msg ("dht", GF_LOG_WARNING, 0, + DHT_MSG_DICT_SET_FAILED, + "Failed to set full choice"); + } + } + } + +out: + if (old_choice) + GF_FREE (old_choice); + if (new_choice) + GF_FREE (new_choice); + if (full_choice) + GF_FREE (full_choice); + + return ret; +} + + int dht_aggregate (dict_t *this, char *key, data_t *value, void *data) @@ -137,18 +321,22 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data) dst = data; - if (strcmp (key, QUOTA_SIZE_KEY) == 0) { + /* compare split brain xattr only */ + if (strcmp (key, GF_AFR_SBRAIN_STATUS) == 0) { + ret = dht_aggregate_split_brain_xattr(dst, key, value); + if (!ret) + goto out; + } else if (strcmp (key, QUOTA_SIZE_KEY) == 0) { ret = dht_aggregate_quota_xattr (dst, key, value); if (ret) { gf_msg ("dht", GF_LOG_WARNING, 0, DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED, "Failed to aggregate quota xattr"); - goto out; } + goto out; } else if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) { ret = gf_get_min_stime (THIS, dst, key, value); - if (ret < 0) - goto out; + goto out; } else { /* compare user xattrs only */ if (!strncmp (key, "user.", strlen ("user."))) { @@ -161,16 +349,16 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data) key); } } - ret = dict_set (dst, key, value); - if (ret) { - gf_msg ("dht", GF_LOG_WARNING, 0, - DHT_MSG_DICT_SET_FAILED, - "Failed to set dictionary value: key = %s", - key); - } } - ret = 0; + ret = dict_set (dst, key, value); + if (ret) { + gf_msg ("dht", GF_LOG_WARNING, 0, + DHT_MSG_DICT_SET_FAILED, + "Failed to set dictionary value: key = %s", + key); + } + out: return ret; } diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h index 9230681541f..3a66e3477df 100644 --- a/xlators/cluster/dht/src/dht-common.h +++ b/xlators/cluster/dht/src/dht-common.h @@ -1269,4 +1269,14 @@ void dht_normalize_stats (struct statvfs *buf, unsigned long bsize, unsigned long frsize); +int +add_opt(char **optsp, const char *opt); + +char * +getChoices (const char *value); + +int +dht_aggregate_split_brain_xattr (dict_t *dst, char *key, data_t *value); + + #endif/* _DHT_H */ |