From 898f0b7ce31ddf8ec02e572c5d22eff2e4205b4c Mon Sep 17 00:00:00 2001 From: Atin Mukherjee Date: Thu, 21 Sep 2017 14:05:35 +0530 Subject: glusterd: retrieve uuid under mutex lock In a multi node cluster, if one of the glusterd instance goes down and comes back, then there might be a race situation where glusterd needs to retrieve its uuid (glusterd_retrieve_uuid) and at the same time as part of receiving a friend handshake from other peer, glusterd iterates over the volume information recieved from remote node and checks for if a brick is local or not by calling MY_UUID which in turn calls glusterd_retrieve_uuid. And the same applies for glusterd_store_global_info () function too. This could end up in a situation where for the same node glusterd ends up generating two UUID files in /var/lib/glusterd. Following is the log snippet which confirms the above: [2017-09-01 03:09:24.458030] I [glusterd.c:146:glusterd_uuid_init] 0-management: retrieved UUID: fd46a495-7e33-468f-88f6-63c815fac640 // thread 1 retrieve uuid from glusterd.info [2017-09-01 03:09:24.458034] E [glusterd-store.c:2109:glusterd_retrieve_uuid] 0-: No previous uuid is present //thread 2 can not retrieve uuid, because in thread1 the file pointer has already become eof. [2017-09-01 03:09:24.458041] E [glusterd-store.c:2117:glusterd_retrieve_uuid] 0-: Returning -1 [2017-09-01 03:09:24.458076] I [glusterd.c:176:glusterd_uuid_generate_save] 0-management: generated UUID: 190bb292-a296-4125-96da-42b247511cc4 [2017-09-01 03:09:24.458129] E [store.c:367:gf_store_save_value] 0-: Able to store key: UUID,value: 190bb292-a296-4125-96da-42b247511cc4 Fix is to retrieve the uuid under mutex lock. Credits : cynthia.zhou@nokia-sbell.com Change-Id: Ib9a5e159c3febf2aef13aa5e38f0a51fe409dadb BUG: 1493967 Signed-off-by: Atin Mukherjee --- xlators/mgmt/glusterd/src/glusterd-store.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'xlators/mgmt/glusterd') diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c index de4b5a60281..862cba4f9cc 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.c +++ b/xlators/mgmt/glusterd/src/glusterd-store.c @@ -2059,9 +2059,13 @@ glusterd_store_global_info (xlator_t *this) ret = -1; goto out; } - - ret = gf_store_save_value (handle->fd, GLUSTERD_STORE_UUID_KEY, - uuid_str); + pthread_mutex_lock (&conf->mutex); + { + ret = gf_store_save_value (handle->fd, + GLUSTERD_STORE_UUID_KEY, + uuid_str); + } + pthread_mutex_unlock (&conf->mutex); if (ret) { gf_msg (this->name, GF_LOG_CRITICAL, 0, GD_MSG_UUID_SET_FAIL, @@ -2296,15 +2300,19 @@ glusterd_retrieve_uuid () priv->handle = handle; } - - ret = gf_store_retrieve_value (priv->handle, GLUSTERD_STORE_UUID_KEY, - &uuid_str); - + pthread_mutex_lock (&priv->mutex); + { + ret = gf_store_retrieve_value (priv->handle, + GLUSTERD_STORE_UUID_KEY, + &uuid_str); + } + pthread_mutex_unlock (&priv->mutex); if (ret) { gf_msg_debug (this->name, 0, "No previous uuid is present"); goto out; } + gf_uuid_parse (uuid_str, priv->uuid); out: -- cgit