diff options
Diffstat (limited to 'xlators/cluster/nsr-server/src/leader.c')
-rw-r--r-- | xlators/cluster/nsr-server/src/leader.c | 283 |
1 files changed, 283 insertions, 0 deletions
diff --git a/xlators/cluster/nsr-server/src/leader.c b/xlators/cluster/nsr-server/src/leader.c new file mode 100644 index 000000000..645f68bf6 --- /dev/null +++ b/xlators/cluster/nsr-server/src/leader.c @@ -0,0 +1,283 @@ +/* + Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <regex.h> +//#include <stdlib.h> +#include <string.h> + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include "call-stub.h" +#include "defaults.h" +#include "xlator.h" +#include "api/src/glfs.h" +#include "api/src/glfs-internal.h" + +#ifndef NSR_SIM_ETCD +#include "etcd-api.h" +#endif +#include "nsr-internal.h" +#include "../../nsr-recon/src/recon_driver.h" +#include "../../nsr-recon/src/recon_xlator.h" + + +/* Vote format: UUID,vote_status,fitness,term_number */ +#define VOTE_ELEMS 4 /* Whole match plus four actual pieces. */ +#define DEFAULT_FITNESS 42 +#define DEFAULT_KEY "nsr" +#define LEADER_TTL 5 /* TBD: make this tunable */ + +typedef enum { LS_SUCCESS, LS_FAILURE, LS_ERROR } leader_retval_t; +enum { NO_LEADER, TENTATIVE, CONFIRMED }; + +regex_t vote_re; + +// Simulation of etcd routines +#ifndef NSR_SIM_ETCD +#endif + +long +nsr_get_fitness (xlator_t *this) +{ + /* TBD: calculate based on presence/absence from terms */ + return 42; +} + +static void +nsr_set_leader (xlator_t *this, etcd_session etcd) +{ + long term = 0; + etcd_result res; + nsr_private_t *priv = this->private; + char *term_key = priv->term_uuid; + char n_t[sizeof(long)+1]; + char *text = NULL; + + gf_log (this->name, GF_LOG_INFO, "Just became leader"); + + text = etcd_get(etcd, priv->term_uuid); + if(text == NULL) { + term = 0; + } else { + term = strtol(text, NULL, 10); + } + sprintf(n_t,"%ld",term+1); + res = etcd_set(etcd, term_key,n_t,text,0); + if(res != ETCD_OK) { + gf_log (this->name, GF_LOG_ERROR, "failed to set term"); + return; + } + priv->leader = _gf_true; + + priv->current_term = term + 1; + + if (priv->nsr_recon_start == _gf_false) { + atomic_fetch_and(&(priv->fence_io), 0); + return; + } + + // Move this inside recon notify??? + atomic_fetch_or(&(priv->fence_io), 1); + + nsr_recon_notify_event_set_leader(priv); + + return; +} + + +leader_retval_t +nsr_get_leader (xlator_t *this, etcd_session etcd, char *key) +{ + char *text = NULL; + regmatch_t matches[VOTE_ELEMS]; + char *nominee; + long state; + long fitness; + char *vote = NULL; + int retval = LS_ERROR; + nsr_private_t *priv = this->private; + + for (;;sleep(1)) { + + if (text) { + free(text); + } + + text = etcd_get(etcd,key); + if (text) { + if (regexec(&vote_re,text,VOTE_ELEMS,matches,0) != 0) { + gf_log (this->name, GF_LOG_ERROR, + "got malformed vote %s\n", text); + continue; + } + /* We can be destructive here, so convert commas. */ + text[matches[1].rm_eo] = '\0'; + text[matches[2].rm_eo] = '\0'; + nominee = text + matches[1].rm_so; + state = strtol(text+matches[2].rm_so,NULL,10); + fitness = strtol(text+matches[3].rm_so,NULL,10); + } + else { + nominee = NULL; + state = NO_LEADER; + fitness = 0; + } + + if (state == CONFIRMED) { + gf_log (this->name, GF_LOG_TRACE, + "leader is %s\n",nominee); + if (strcmp(nominee,priv->brick_uuid) == 0) { + nsr_set_leader(this, etcd); + retval = LS_SUCCESS; + } + else { + priv->leader = _gf_false; + retval = LS_FAILURE; + } + break; + } + + /* TBD: override based on fitness */ + if ((state >= TENTATIVE) && (strcmp(nominee, + priv->brick_uuid) != 0)) { + continue; + } + + if (vote) { + free(vote); + } + + fitness = nsr_get_fitness(this); + if (asprintf(&vote,"%s,%ld,%ld",priv->brick_uuid, + state+1,fitness) < 0) { + gf_log (this->name, GF_LOG_ERROR, + "failed to construct vote\n"); + break; + } + + if (text) { + text[matches[1].rm_eo] = ','; + text[matches[2].rm_eo] = ','; + } + if (etcd_set(etcd,key,vote,text,LEADER_TTL) != ETCD_OK) { + gf_log (this->name, GF_LOG_ERROR, + "failed to cast vote\n"); + continue; + } + + } + + if (text) { + free(text); + } + if (vote) { + free(vote); + } + return retval; +} + +leader_retval_t +nsr_confirm (xlator_t *this, etcd_session etcd, char *key) +{ + char *vote; + long fitness; + nsr_private_t *priv = this->private; + + fitness = nsr_get_fitness(this); + if (asprintf(&vote,"%s,%ld,%ld",priv->brick_uuid,(long)CONFIRMED, + fitness) < 0) { + fprintf (stderr, "%s: failed to construct confirmation\n", + __func__); + return LS_ERROR; + } + + if (etcd_set(etcd,key,vote,vote,LEADER_TTL) != ETCD_OK) { + fprintf (stderr, "%s: failed to confirm\n", __func__); + free(vote); + return LS_FAILURE; + } + + free(vote); + return LS_SUCCESS; +} + +gf_boolean_t +nsr_init_re (xlator_t *this) +{ + static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + static int was_inited = 0; + static char *vote_re_str = "([^,]+),([^,]+),([^,]+)"; + gf_boolean_t retval = _gf_false; + + pthread_mutex_lock(&mutex); + if (!was_inited) { + if (regcomp(&vote_re,vote_re_str,REG_EXTENDED) == 0) { + retval = _gf_true; + } + else { + gf_log (this->name, GF_LOG_ERROR, + "failed to set up vote regex\n"); + } + } + pthread_mutex_unlock(&mutex); + + return retval; +} + + +void * +nsr_leader_thread (void *arg) +{ + xlator_t *this = (xlator_t *) arg; + leader_retval_t retval; + nsr_private_t *priv = this->private; + + if (!nsr_init_re(this)) { + gf_log (this->name, GF_LOG_ERROR, "could not init regex"); + return NULL; + } + + gf_log (this->name, GF_LOG_INFO, + "calling glfs_opens_str on servers %s", priv->etcd_servers); + + priv->etcd = etcd_open_str(priv->etcd_servers); + if (!(priv->etcd)) { + gf_log (this->name, GF_LOG_ERROR, + "failed to open etcd session\n"); + return NULL; + } + + priv->leader_inited = 1; + + for (;;) { + if (nsr_get_leader(this,priv->etcd,priv->vol_uuid) == LS_ERROR) { + break; + } + if (priv->leader) { + do { + sleep(1); + retval = nsr_confirm(this,priv->etcd,priv->vol_uuid); + } while (retval == LS_SUCCESS); + if (retval == LS_ERROR) { + break; + } + } + else { + sleep(1); + } + } + + etcd_close_str(priv->etcd); + return NULL; +} + |