glusterd / geo-rep: do all gsyncd pre-configuration in glusterd init

This is made possible by gsyncd config templating, by which
session specific settings can be expressed in a generic form.

Benefits:
- in glusterd we use only generic (rx pattern based) config settings
  so we don't violate users' freedom to make settigs for their sessions
- don't have to invoke gsyncd in excess to prepare a session
- we can pre-configure slave side too (relying on the templates and
  gsyncd service auto-discovery)
- much leaner, much more expressive code

Signed-off-by: Csaba Henk <csaba@lowlife.hu>
Signed-off-by: Anand Avati <avati@gluster.com>

BUG: 2785 (gsyncd logs on slave side go to /dev/null)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=2785

3 files changed, 140 insertions, 137 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index c51853d9016..0bae57847b9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -100,6 +100,7 @@ static char *gsync_reserved_opts[] = {
         "pid-file",
         "log-file",
         "state-file",
+        "session-owner",
         NULL
 };
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 17ac245e262..c80b3ae95c7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -3246,18 +3246,31 @@ glusterd_delete_all_bricks (glusterd_volinfo_t* volinfo)
 }
 
 int
+mkdir_if_missing (char *path)
+{
+        struct stat st = {0,};
+        int        ret = 0;
+
+        ret = mkdir (path, 0777);
+        if (!ret || errno == EEXIST)
+                ret = stat (path, &st);
+        if (ret == -1 || !S_ISDIR (st.st_mode))
+                gf_log ("", GF_LOG_WARNING, "Failed to create the"
+                        " directory %s", path);
+
+        return ret;
+}
+
+int
 glusterd_start_gsync (char *master, char *slave, char *uuid_str,
                       char **op_errstr)
 {
         int32_t         ret     = 0;
         int32_t         status  = 0;
-        char            cmd[PATH_MAX] = {0,};
-        char            prmfile[PATH_MAX]   = {0,};
+        char            buf[PATH_MAX]   = {0,};
         char            local_uuid_str [64] = {0};
-        char            *tslash = NULL;
         xlator_t        *this = NULL;
         glusterd_conf_t *priv = NULL;
-        char            msg[3*PATH_MAX] = {0};
         int             errcode = 0;
 
         this = THIS;
@@ -3272,136 +3285,34 @@ glusterd_start_gsync (char *master, char *slave, char *uuid_str,
         ret = gsync_status (master, slave, &status);
         if (status == 0)
                 goto out;
-        ret = glusterd_gsync_get_param_file (prmfile, "pid", master,
-                                             slave, priv->workdir);
-        if (ret == -1) {
-                errcode = -1;
-                goto out;
-        }
-        unlink (prmfile);
-        tslash = strrchr(prmfile, '/');
-        if (tslash) {
-                *tslash = '\0';
-                ret = mkdir (prmfile, 0777);
-                if (ret && (errno != EEXIST)) {
-                        errcode = -1;
-                        gf_log ("", GF_LOG_WARNING, "Failed to create the"
-                                " directory %s", prmfile);
-                        goto out;
-                }
-                *tslash = '/';
-        }
-
-        memset (cmd, 0, sizeof (cmd));
-        ret = snprintf (cmd, PATH_MAX, GSYNCD_PREFIX "/gsyncd -c %s/%s :%s %s"
-                                       " --config-set pid-file %s", priv->workdir,
-                                       GSYNC_CONF, master, slave, prmfile);
-        if (ret <= 0) {
-                ret = -1;
-                errcode = -1;
-                gf_log ("", GF_LOG_WARNING, "failed to construct the  "
-                        "config set command for %s %s", master, slave);
-                goto out;
-        }
 
-        ret = gf_system (cmd);
+        snprintf (buf, PATH_MAX, "%s/"GEOREP"/%s", priv->workdir, master);
+        ret = mkdir_if_missing (buf);
         if (ret) {
                 errcode = -1;
-                gf_log ("", GF_LOG_WARNING, "failed to set the pid "
-                        "option for %s %s", master, slave);
-                goto out;
-        }
-
-        ret = glusterd_gsync_get_param_file (prmfile, "status", NULL, NULL, NULL);
-        if (ret != -1)
-                ret = snprintf (cmd, PATH_MAX, GSYNCD_PREFIX "/gsyncd -c %s/%s "
-                                ":%s %s --config-set state-file %s",
-                                priv->workdir,GSYNC_CONF, master, slave,
-                                prmfile);
-        if (ret >= PATH_MAX)
-                ret = -1;
-        if (ret != -1)
-                ret = gf_system (cmd) ? -1 : 0;
-        if (ret == -1) {
-                errcode = -1;
-                gf_log ("", GF_LOG_WARNING, "failed to set status file "
-                        "for %s %s", master, slave);
                 goto out;
         }
 
-        ret = glusterd_gsync_get_param_file (prmfile, "log", master,
-                                             slave, DEFAULT_LOG_FILE_DIRECTORY);
-        if (ret == -1) {
-                errcode = -1;
-                goto out;
-        }
-        /* XXX "mkdir -p": eventually this should be made into a library routine */
-        tslash = strrchr(prmfile, '/');
-        if (tslash) {
-                char *slash = prmfile;
-                struct stat st = {0,};
-
-                *tslash = '\0';
-                if (*slash == '/')
-                        slash++;
-                while (slash) {
-                        slash = strchr (slash, '/');
-                        if (slash)
-                                *slash = '\0';
-                        ret = mkdir (prmfile, 0777);
-                        if (ret == -1 && errno != EEXIST) {
-                                errcode = -1;
-                                gf_log ("", GF_LOG_WARNING, "Failed to create"
-                                        " the directory %s", prmfile);
-                                goto out;
-                        }
-                        if (slash) {
-                                *slash = '/';
-                                slash++;
-                        }
-                }
-                ret = stat (prmfile, &st);
-                if (ret == -1 || !S_ISDIR (st.st_mode)) {
-                        ret = -1;
-                        errcode  = -1;
-                        gf_log ("", GF_LOG_WARNING, "Failed to create the"
-                                "directory %s", prmfile);
-                        goto out;
-                }
-                *tslash = '/';
-        }
-
-        ret = snprintf (cmd, PATH_MAX, GSYNCD_PREFIX "/gsyncd -c %s/%s :%s %s"
-                        " --config-set log-file %s", priv->workdir,
-                        GSYNC_CONF, master, slave, prmfile);
-        if (ret >= PATH_MAX)
-                ret = -1;
-        if (ret != -1)
-                ret = gf_system (cmd) ? -1 : 0;
-        if (ret == -1) {
+        snprintf (buf, PATH_MAX, DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/%s",
+                  master);
+        ret = mkdir_if_missing (buf);
+        if (ret) {
                 errcode = -1;
-                gf_log ("", GF_LOG_WARNING, "failed to set status file "
-                        "for %s %s", master, slave);
                 goto out;
         }
 
-        memset (cmd, 0, sizeof (cmd));
-        ret = snprintf (cmd, PATH_MAX, GSYNCD_PREFIX "/gsyncd --monitor -c"
-                        "%s/%s :%s %s", priv->workdir, GSYNC_CONF, master,
-                        slave);
+        ret = snprintf (buf, PATH_MAX, GSYNCD_PREFIX "/gsyncd --monitor -c "
+                        "%s/"GSYNC_CONF" :%s %s", priv->workdir, master, slave);
         if (ret <= 0) {
                 ret = -1;
                 errcode = -1;
                 goto out;
         }
 
-        ret = gf_system (cmd);
+        ret = gf_system (buf);
         if (ret == -1) {
-                ret = snprintf (msg, sizeof (msg), GEOREP" start failed for %s "
-                                "%s", master, slave);
-                if (ret <=0)
-                        goto out;
-                *op_errstr = gf_strdup (msg);
+                gf_asprintf (op_errstr, GEOREP" start failed for %s %s",
+                             master, slave);
                 goto out;
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 4d98fafd7ee..f72f058134a 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -229,34 +229,67 @@ out:
         return ret;
 }
 
+/* defined in usterd-utils.c -- no
+ * glusterd header where it would be
+ * appropriate to put to, and too
+ * accidental routine to place in
+ * libglusterfs.
+ *
+ * (Indeed, XXX: we'd rather need a general
+ * "mkdir -p" like routine in
+ * libglusterfs)
+ */
+extern int mkdir_if_missing (char *path);
+
 static int
-configure_syncaemon (xlator_t *this, const char *workdir)
+configure_syncaemon (glusterd_conf_t *conf)
 {
         int ret = 0;
 #if SYNCDAEMON_COMPILE
-        char voldir[PATH_MAX] = {0,};
+        char georepdir[PATH_MAX] = {0,};
         char cmd[4096] = {0,};
+        char volid[64] = {0,};
         int blen = 0;
 
         setenv ("_GLUSTERD_CALLED_", "1", 1);
 
-        snprintf (voldir, PATH_MAX, "%s/"GEOREP, workdir);
-        ret = mkdir (voldir, 0777);
-        if ((-1 == ret) && (errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to create "GEOREP" directory %s (%s)",
-                        voldir, strerror (errno));
+        snprintf (georepdir, PATH_MAX, "%s/"GEOREP, conf->workdir);
+        ret = mkdir_if_missing (georepdir);
+        if (-1 == ret) {
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "Unable to create "GEOREP" directory %s",
+                        georepdir);
+                return -1;
+        }
+        ret = mkdir_if_missing (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP);
+        if (-1 == ret) {
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "Unable to create "GEOREP" log directory");
+                return -1;
+        }
+        ret = mkdir_if_missing (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves");
+        if (-1 == ret) {
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "Unable to create "GEOREP" slave log directory");
                 return -1;
         }
 
         blen = snprintf (cmd, PATH_MAX, GSYNCD_PREFIX"/gsyncd -c %s/"GSYNC_CONF
-                         " --config-set-rx ", workdir);
+                         " --config-set-rx ", conf->workdir);
+
+        /* Calling out to gsyncd to configure it:
+         * - use system(3) for options with multi-word values as system
+         *   groks quotes;
+         * - use gf_system() for options with template expanders so
+         *   that they are not messed up by shell
+         */
+
+        /************
+         * master pre-configuration
+         ************/
 
         /* remote-gsyncd */
-        strcpy (cmd + blen,
-                "remote-gsyncd "
-                "'"GSYNCD_PREFIX"/gsyncd --gluster-command "GFS_PREFIX"/sbin/glusterfs' "
-                ". .");
+        strcpy (cmd + blen, "remote-gsyncd " GSYNCD_PREFIX"/gsyncd . .");
         ret = system (cmd);
         if (ret)
                 goto out;
@@ -280,15 +313,73 @@ configure_syncaemon (xlator_t *this, const char *workdir)
         sprintf (cmd + blen,
                  "ssh-command "
                  "'ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no "
-                 "-i %s/"GEOREP"/secret.pem' . .", workdir);
+                 "-i %s/secret.pem' . .", georepdir);
+        ret = system (cmd);
+        if (ret)
+                goto out;
+
+        /* session-owner */
+        uuid_unparse (conf->uuid, volid);
+        sprintf (cmd + blen, "session-owner %s . .", volid);
+        ret = system (cmd);
+        if (ret)
+                goto out;
+
+        /* pid-file */
+        sprintf (cmd + blen, "pid-file %s/${mastervol}/${eSlave}.pid . .", georepdir);
+        ret = gf_system (cmd);
+        if (ret)
+                goto out;
+
+        /* state-file */
+        sprintf (cmd + blen, "state-file %s/${mastervol}/${eSlave}.status . .", georepdir);
+        ret = gf_system (cmd);
+        if (ret)
+                goto out;
+
+        /* log-file */
+        strcpy (cmd + blen,
+                "log-file "DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.log . .");
+        ret = gf_system (cmd);
+        if (ret)
+                goto out;
+
+        /* gluster-log-file */
+        strcpy (cmd + blen, "gluster-log-file "
+                DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.gluster.log . .");
+        ret = gf_system (cmd);
+        if (ret)
+                goto out;
+
+        /************
+         * slave pre-configuration
+         ************/
+
+        /* gluster-command */
+        strcpy (cmd + blen,
+                "gluster-command '"GFS_PREFIX"/sbin/glusterfs "
+                "--xlator-option *-dht.assert-no-child-down=true' .");
         ret = system (cmd);
         if (ret)
                 goto out;
 
+        /* log-file */
+        strcpy (cmd + blen,
+                "log-file "DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.log .");
+        ret = gf_system (cmd);
+        if (ret)
+                goto out;
+
+        /* gluster-log-file */
+        strcpy (cmd + blen, "gluster-log-file "
+                DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.gluster.log .");
+        ret = gf_system (cmd);
+        if (ret)
+                goto out;
+
  out:
 #else
-        (void)this;
-        (void)workdir;
+        (void)conf;
 #endif
         return ret ? -1 : 0;
 }
@@ -403,10 +494,6 @@ init (xlator_t *this)
                 exit (1);
         }
 
-        ret = configure_syncaemon (this, dirname);
-        if (ret)
-                goto out;
-
         ret = glusterd_rpcsvc_options_build (this->options);
         if (ret)
                 goto out;
@@ -496,6 +583,10 @@ init (xlator_t *this)
         if (ret < 0)
                 goto out;
 
+        ret = configure_syncaemon (conf);
+        if (ret)
+                goto out;
+
         ret = glusterd_restore ();
         if (ret < 0)
                 goto out;