From ced2d41bfdf6ee3ee820fa6090e4bf3147ab0d84 Mon Sep 17 00:00:00 2001 From: Csaba Henk Date: Mon, 18 Apr 2011 17:25:30 +0000 Subject: glusterd / geo-rep: do all gsyncd pre-configuration in glusterd init This is made possible by gsyncd config templating, by which session specific settings can be expressed in a generic form. Benefits: - in glusterd we use only generic (rx pattern based) config settings so we don't violate users' freedom to make settigs for their sessions - don't have to invoke gsyncd in excess to prepare a session - we can pre-configure slave side too (relying on the templates and gsyncd service auto-discovery) - much leaner, much more expressive code Signed-off-by: Csaba Henk Signed-off-by: Anand Avati BUG: 2785 (gsyncd logs on slave side go to /dev/null) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=2785 --- xlators/mgmt/glusterd/src/glusterd-op-sm.c | 1 + xlators/mgmt/glusterd/src/glusterd-utils.c | 145 ++++++----------------------- xlators/mgmt/glusterd/src/glusterd.c | 131 ++++++++++++++++++++++---- 3 files changed, 140 insertions(+), 137 deletions(-) (limited to 'xlators/mgmt') diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c index c51853d9016..0bae57847b9 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c @@ -100,6 +100,7 @@ static char *gsync_reserved_opts[] = { "pid-file", "log-file", "state-file", + "session-owner", NULL }; diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index 17ac245e262..c80b3ae95c7 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -3245,19 +3245,32 @@ glusterd_delete_all_bricks (glusterd_volinfo_t* volinfo) return ret; } +int +mkdir_if_missing (char *path) +{ + struct stat st = {0,}; + int ret = 0; + + ret = mkdir (path, 0777); + if (!ret || errno == EEXIST) + ret = stat (path, &st); + if (ret == -1 || !S_ISDIR (st.st_mode)) + gf_log ("", GF_LOG_WARNING, "Failed to create the" + " directory %s", path); + + return ret; +} + int glusterd_start_gsync (char *master, char *slave, char *uuid_str, char **op_errstr) { int32_t ret = 0; int32_t status = 0; - char cmd[PATH_MAX] = {0,}; - char prmfile[PATH_MAX] = {0,}; + char buf[PATH_MAX] = {0,}; char local_uuid_str [64] = {0}; - char *tslash = NULL; xlator_t *this = NULL; glusterd_conf_t *priv = NULL; - char msg[3*PATH_MAX] = {0}; int errcode = 0; this = THIS; @@ -3272,136 +3285,34 @@ glusterd_start_gsync (char *master, char *slave, char *uuid_str, ret = gsync_status (master, slave, &status); if (status == 0) goto out; - ret = glusterd_gsync_get_param_file (prmfile, "pid", master, - slave, priv->workdir); - if (ret == -1) { - errcode = -1; - goto out; - } - unlink (prmfile); - tslash = strrchr(prmfile, '/'); - if (tslash) { - *tslash = '\0'; - ret = mkdir (prmfile, 0777); - if (ret && (errno != EEXIST)) { - errcode = -1; - gf_log ("", GF_LOG_WARNING, "Failed to create the" - " directory %s", prmfile); - goto out; - } - *tslash = '/'; - } - - memset (cmd, 0, sizeof (cmd)); - ret = snprintf (cmd, PATH_MAX, GSYNCD_PREFIX "/gsyncd -c %s/%s :%s %s" - " --config-set pid-file %s", priv->workdir, - GSYNC_CONF, master, slave, prmfile); - if (ret <= 0) { - ret = -1; - errcode = -1; - gf_log ("", GF_LOG_WARNING, "failed to construct the " - "config set command for %s %s", master, slave); - goto out; - } - ret = gf_system (cmd); + snprintf (buf, PATH_MAX, "%s/"GEOREP"/%s", priv->workdir, master); + ret = mkdir_if_missing (buf); if (ret) { errcode = -1; - gf_log ("", GF_LOG_WARNING, "failed to set the pid " - "option for %s %s", master, slave); - goto out; - } - - ret = glusterd_gsync_get_param_file (prmfile, "status", NULL, NULL, NULL); - if (ret != -1) - ret = snprintf (cmd, PATH_MAX, GSYNCD_PREFIX "/gsyncd -c %s/%s " - ":%s %s --config-set state-file %s", - priv->workdir,GSYNC_CONF, master, slave, - prmfile); - if (ret >= PATH_MAX) - ret = -1; - if (ret != -1) - ret = gf_system (cmd) ? -1 : 0; - if (ret == -1) { - errcode = -1; - gf_log ("", GF_LOG_WARNING, "failed to set status file " - "for %s %s", master, slave); goto out; } - ret = glusterd_gsync_get_param_file (prmfile, "log", master, - slave, DEFAULT_LOG_FILE_DIRECTORY); - if (ret == -1) { - errcode = -1; - goto out; - } - /* XXX "mkdir -p": eventually this should be made into a library routine */ - tslash = strrchr(prmfile, '/'); - if (tslash) { - char *slash = prmfile; - struct stat st = {0,}; - - *tslash = '\0'; - if (*slash == '/') - slash++; - while (slash) { - slash = strchr (slash, '/'); - if (slash) - *slash = '\0'; - ret = mkdir (prmfile, 0777); - if (ret == -1 && errno != EEXIST) { - errcode = -1; - gf_log ("", GF_LOG_WARNING, "Failed to create" - " the directory %s", prmfile); - goto out; - } - if (slash) { - *slash = '/'; - slash++; - } - } - ret = stat (prmfile, &st); - if (ret == -1 || !S_ISDIR (st.st_mode)) { - ret = -1; - errcode = -1; - gf_log ("", GF_LOG_WARNING, "Failed to create the" - "directory %s", prmfile); - goto out; - } - *tslash = '/'; - } - - ret = snprintf (cmd, PATH_MAX, GSYNCD_PREFIX "/gsyncd -c %s/%s :%s %s" - " --config-set log-file %s", priv->workdir, - GSYNC_CONF, master, slave, prmfile); - if (ret >= PATH_MAX) - ret = -1; - if (ret != -1) - ret = gf_system (cmd) ? -1 : 0; - if (ret == -1) { + snprintf (buf, PATH_MAX, DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/%s", + master); + ret = mkdir_if_missing (buf); + if (ret) { errcode = -1; - gf_log ("", GF_LOG_WARNING, "failed to set status file " - "for %s %s", master, slave); goto out; } - memset (cmd, 0, sizeof (cmd)); - ret = snprintf (cmd, PATH_MAX, GSYNCD_PREFIX "/gsyncd --monitor -c" - "%s/%s :%s %s", priv->workdir, GSYNC_CONF, master, - slave); + ret = snprintf (buf, PATH_MAX, GSYNCD_PREFIX "/gsyncd --monitor -c " + "%s/"GSYNC_CONF" :%s %s", priv->workdir, master, slave); if (ret <= 0) { ret = -1; errcode = -1; goto out; } - ret = gf_system (cmd); + ret = gf_system (buf); if (ret == -1) { - ret = snprintf (msg, sizeof (msg), GEOREP" start failed for %s " - "%s", master, slave); - if (ret <=0) - goto out; - *op_errstr = gf_strdup (msg); + gf_asprintf (op_errstr, GEOREP" start failed for %s %s", + master, slave); goto out; } diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c index 4d98fafd7ee..f72f058134a 100644 --- a/xlators/mgmt/glusterd/src/glusterd.c +++ b/xlators/mgmt/glusterd/src/glusterd.c @@ -229,34 +229,67 @@ out: return ret; } +/* defined in usterd-utils.c -- no + * glusterd header where it would be + * appropriate to put to, and too + * accidental routine to place in + * libglusterfs. + * + * (Indeed, XXX: we'd rather need a general + * "mkdir -p" like routine in + * libglusterfs) + */ +extern int mkdir_if_missing (char *path); + static int -configure_syncaemon (xlator_t *this, const char *workdir) +configure_syncaemon (glusterd_conf_t *conf) { int ret = 0; #if SYNCDAEMON_COMPILE - char voldir[PATH_MAX] = {0,}; + char georepdir[PATH_MAX] = {0,}; char cmd[4096] = {0,}; + char volid[64] = {0,}; int blen = 0; setenv ("_GLUSTERD_CALLED_", "1", 1); - snprintf (voldir, PATH_MAX, "%s/"GEOREP, workdir); - ret = mkdir (voldir, 0777); - if ((-1 == ret) && (errno != EEXIST)) { - gf_log (this->name, GF_LOG_CRITICAL, - "Unable to create "GEOREP" directory %s (%s)", - voldir, strerror (errno)); + snprintf (georepdir, PATH_MAX, "%s/"GEOREP, conf->workdir); + ret = mkdir_if_missing (georepdir); + if (-1 == ret) { + gf_log ("glusterd", GF_LOG_CRITICAL, + "Unable to create "GEOREP" directory %s", + georepdir); + return -1; + } + ret = mkdir_if_missing (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP); + if (-1 == ret) { + gf_log ("glusterd", GF_LOG_CRITICAL, + "Unable to create "GEOREP" log directory"); + return -1; + } + ret = mkdir_if_missing (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves"); + if (-1 == ret) { + gf_log ("glusterd", GF_LOG_CRITICAL, + "Unable to create "GEOREP" slave log directory"); return -1; } blen = snprintf (cmd, PATH_MAX, GSYNCD_PREFIX"/gsyncd -c %s/"GSYNC_CONF - " --config-set-rx ", workdir); + " --config-set-rx ", conf->workdir); + + /* Calling out to gsyncd to configure it: + * - use system(3) for options with multi-word values as system + * groks quotes; + * - use gf_system() for options with template expanders so + * that they are not messed up by shell + */ + + /************ + * master pre-configuration + ************/ /* remote-gsyncd */ - strcpy (cmd + blen, - "remote-gsyncd " - "'"GSYNCD_PREFIX"/gsyncd --gluster-command "GFS_PREFIX"/sbin/glusterfs' " - ". ."); + strcpy (cmd + blen, "remote-gsyncd " GSYNCD_PREFIX"/gsyncd . ."); ret = system (cmd); if (ret) goto out; @@ -280,15 +313,73 @@ configure_syncaemon (xlator_t *this, const char *workdir) sprintf (cmd + blen, "ssh-command " "'ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no " - "-i %s/"GEOREP"/secret.pem' . .", workdir); + "-i %s/secret.pem' . .", georepdir); + ret = system (cmd); + if (ret) + goto out; + + /* session-owner */ + uuid_unparse (conf->uuid, volid); + sprintf (cmd + blen, "session-owner %s . .", volid); + ret = system (cmd); + if (ret) + goto out; + + /* pid-file */ + sprintf (cmd + blen, "pid-file %s/${mastervol}/${eSlave}.pid . .", georepdir); + ret = gf_system (cmd); + if (ret) + goto out; + + /* state-file */ + sprintf (cmd + blen, "state-file %s/${mastervol}/${eSlave}.status . .", georepdir); + ret = gf_system (cmd); + if (ret) + goto out; + + /* log-file */ + strcpy (cmd + blen, + "log-file "DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.log . ."); + ret = gf_system (cmd); + if (ret) + goto out; + + /* gluster-log-file */ + strcpy (cmd + blen, "gluster-log-file " + DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.gluster.log . ."); + ret = gf_system (cmd); + if (ret) + goto out; + + /************ + * slave pre-configuration + ************/ + + /* gluster-command */ + strcpy (cmd + blen, + "gluster-command '"GFS_PREFIX"/sbin/glusterfs " + "--xlator-option *-dht.assert-no-child-down=true' ."); ret = system (cmd); if (ret) goto out; + /* log-file */ + strcpy (cmd + blen, + "log-file "DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.log ."); + ret = gf_system (cmd); + if (ret) + goto out; + + /* gluster-log-file */ + strcpy (cmd + blen, "gluster-log-file " + DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.gluster.log ."); + ret = gf_system (cmd); + if (ret) + goto out; + out: #else - (void)this; - (void)workdir; + (void)conf; #endif return ret ? -1 : 0; } @@ -403,10 +494,6 @@ init (xlator_t *this) exit (1); } - ret = configure_syncaemon (this, dirname); - if (ret) - goto out; - ret = glusterd_rpcsvc_options_build (this->options); if (ret) goto out; @@ -496,6 +583,10 @@ init (xlator_t *this) if (ret < 0) goto out; + ret = configure_syncaemon (conf); + if (ret) + goto out; + ret = glusterd_restore (); if (ret < 0) goto out; -- cgit