geo-rep: partial support for unprivileged gsyncd via mountbroker

gsyncd:
  - mounting code is split to a direct and a mountbroker based backend
  - option gluster-command gone
  - new options: gluster-params, gluster-cli-options, mountbroker
  - mountbroker mount backend is used if either a mountbroker label
    is given through the mountbroker option, or if gsyncd is
    unprivileged; in this case the username is used as label
  - have gluster cli invocations log to stderr so that we don't
    hit a permission issue with the logfiles
glusterd:
  - do gsyncd pre-config with new options
  - add option geo-replication-log-group, so if that specified
    geo-rep logfile directories are given to that group (and
    thus members of the given group can do logging there)

This is just WIP as geo-rep relies on trusted extended attributes
and those are not accessible for unprivileged users. Even if we
solved this issue, glusterd security settings are too coarse,
so that if we made it possible for an unprivileged gsyncd
to operate, we would open up too far.

Change-Id: Icd520b58cbadccea3fad7c0f437b99de1e22db14
BUG: 2825
Reviewed-on: http://review.gluster.com/399
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

5 files changed, 197 insertions, 46 deletions

diff --git a/xlators/features/marker/utils/gsyncd.in b/xlators/features/marker/utils/gsyncd.in
index a7af8c0b0b4..b517b341f96 100755
--- a/xlators/features/marker/utils/gsyncd.in
+++ b/xlators/features/marker/utils/gsyncd.in
@@ -42,7 +42,7 @@ else
 fi
 
 if [ $config_wanted = 1 ]; then
-    wd="`${gluster} system:: getwd`"
+    wd="`${gluster} --log-file=/dev/stderr system:: getwd`"
     if [ $? -eq 0 ]; then
         config_file="$wd/geo-replication/gsyncd.conf"
     fi
diff --git a/xlators/features/marker/utils/syncdaemon/gsyncd.py b/xlators/features/marker/utils/syncdaemon/gsyncd.py
index c0d39ffd62d..797000970a5 100644
--- a/xlators/features/marker/utils/syncdaemon/gsyncd.py
+++ b/xlators/features/marker/utils/syncdaemon/gsyncd.py
@@ -139,9 +139,12 @@ def main_i():
         return lambda o, oo, vx, p: store_local(o, oo, FreeObject(op=op, **dmake(vx)), p)
 
     op = OptionParser(usage="%prog [options...] <master> <slave>", version="%prog 0.0.1")
-    op.add_option('--gluster-command',     metavar='CMD',   default='glusterfs')
+    op.add_option('--gluster-command-dir', metavar='DIR',   default='')
     op.add_option('--gluster-log-file',    metavar='LOGF',  default=os.devnull, type=str, action='callback', callback=store_abs)
     op.add_option('--gluster-log-level',   metavar='LVL')
+    op.add_option('--gluster-params',      metavar='PRMS',  default='')
+    op.add_option('--gluster-cli-options', metavar='OPTS',  default='--log-file=/dev/stderr')
+    op.add_option('--mountbroker',         metavar='LABEL')
     op.add_option('-p', '--pid-file',      metavar='PIDF',  type=str, action='callback', callback=store_abs)
     op.add_option('-l', '--log-file',      metavar='LOGF',  type=str, action='callback', callback=store_abs)
     op.add_option('--state-file',          metavar='STATF', type=str, action='callback', callback=store_abs)
diff --git a/xlators/features/marker/utils/syncdaemon/resource.py b/xlators/features/marker/utils/syncdaemon/resource.py
index f92e8573409..b851d661a9e 100644
--- a/xlators/features/marker/utils/syncdaemon/resource.py
+++ b/xlators/features/marker/utils/syncdaemon/resource.py
@@ -1,7 +1,6 @@
 import re
 import os
 import sys
-import pwd
 import stat
 import time
 import errno
@@ -579,41 +578,126 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
         """determine our position in the connectibility matrix"""
         return True
 
-    def connect(self):
-        """inhibit the resource beyond
+    class Mounter(object):
+        """Abstract base class for mounter backends"""
 
-        - create temprorary mount point
-        - call glusterfs to mount the volume over there
-        - change to mounted fs root
-        - lazy umount + delete temp. mount point
-        """
-        def umount_l(d):
-            po = Popen(['umount', '-l', d], stderr=subprocess.PIPE)
+        def __init__(self, params):
+            self.params = params
+
+        @classmethod
+        def get_glusterprog(cls):
+            return os.path.join(gconf.gluster_command_dir, cls.glusterprog)
+
+        def umount_l(self, d):
+            """perform lazy umount"""
+            po = Popen(self.make_umount_argv(d), stderr=subprocess.PIPE)
             po.wait()
             return po
-        d = tempfile.mkdtemp(prefix='gsyncd-aux-mount-')
-        mounted = False
-        try:
-            po = Popen(gconf.gluster_command.split() + \
-                       (gconf.gluster_log_level and ['-L', gconf.gluster_log_level] or []) + \
-                       ['-l', gconf.gluster_log_file, '-s', self.host,
-                        '--volfile-id', self.volume, '--client-pid=-1', d],
-                       stderr=subprocess.PIPE)
+
+        @classmethod
+        def make_umount_argv(cls, d):
+            raise NotImplementedError
+
+        def make_mount_argv(self, *a):
+            raise NotImplementedError
+
+        def cleanup_mntpt(self):
+            pass
+
+        def handle_mounter(self, po):
             po.wait()
-            po.terminate_geterr()
-            mounted = True
-            logging.debug('auxiliary glusterfs mount in place')
-            os.chdir(d)
-            umount_l(d).terminate_geterr()
+
+        def inhibit(self, *a):
+            """inhibit a gluster filesystem
+
+            Mount glusterfs over a temporary mountpoint,
+            change into the mount, and lazy unmount the
+            filesystem.
+            """
             mounted = False
-        finally:
             try:
-                if mounted:
-                    umount_l(d).terminate_geterr(fail_on_err = False)
-                os.rmdir(d)
-            except:
-                logging.warn('stale mount possibly left behind on ' + d)
-        logging.debug('auxiliary glusterfs mount prepared')
+                po = Popen(self.make_mount_argv(*a), **self.mountkw)
+                self.handle_mounter(po)
+                po.terminate_geterr()
+                d = self.mntpt
+                mounted = True
+                logging.debug('auxiliary glusterfs mount in place')
+                os.chdir(d)
+                self.umount_l(d).terminate_geterr()
+                mounted = False
+            finally:
+                try:
+                    if mounted:
+                        self.umount_l(d).terminate_geterr(fail_on_err = False)
+                    self.cleanup_mntpt()
+                except:
+                    logging.warn('stale mount possibly left behind on ' + d)
+            logging.debug('auxiliary glusterfs mount prepared')
+
+    class DirectMounter(Mounter):
+        """mounter backend which calls mount(8), umount(8) directly"""
+
+        mountkw = {'stderr': subprocess.PIPE}
+        glusterprog = 'glusterfs'
+
+        @staticmethod
+        def make_umount_argv(d):
+            return ['umount', '-l', d]
+
+        def make_mount_argv(self):
+            self.mntpt = tempfile.mkdtemp(prefix = 'gsyncd-aux-mount-')
+            return [self.get_glusterprog()] + ['--' + p for p in self.params] + [self.mntpt]
+
+        def cleanup_mntpt(self):
+            os.rmdir(self.mntpt)
+
+    class MountbrokerMounter(Mounter):
+        """mounter backend using the mountbroker gluster service"""
+
+        mountkw = {'stderr': subprocess.PIPE, 'stdout': subprocess.PIPE}
+        glusterprog = 'gluster'
+
+        @classmethod
+        def make_cli_argv(cls):
+            return [cls.get_glusterprog()] + gconf.gluster_cli_options.split() + ['system::']
+
+        @classmethod
+        def make_umount_argv(cls, d):
+            return cls.make_cli_argv() + ['umount', d, 'lazy']
+
+        def make_mount_argv(self, label):
+            return self.make_cli_argv() + \
+                   ['mount', label, 'user-map-root=' + syncdutils.getusername()] + self.params
+
+        def handle_mounter(self, po):
+            self.mntpt = po.stdout.readline()[:-1]
+            po.stdout.close()
+            sup(self, po)
+            if po.returncode != 0:
+                # if cli terminated with error due to being
+                # refused by glusterd, what it put
+                # out on stdout is a diagnostic message
+                logging.error('glusterd answered: %s' % self.mntpt)
+
+    def connect(self):
+        """inhibit the resource beyond
+
+        Choose mounting backend (direct or mountbroker),
+        set up glusterfs parameters and perform the mount
+        with given backend
+        """
+
+        label = getattr(gconf, 'mountbroker', None)
+        if not label:
+            uid = os.geteuid()
+            if uid != 0:
+                label = syncdutils.getusername(uid)
+        mounter = label and self.MountbrokerMounter or self.DirectMounter
+        params = gconf.gluster_params.split() + \
+                   (gconf.gluster_log_level and ['log-level=' + gconf.gluster_log_level] or []) + \
+                   ['log-file=' + gconf.gluster_log_file, 'volfile-server=' + self.host,
+                    'volfile-id=' + self.volume, 'client-pid=-1']
+        mounter(params).inhibit(*[l for l in [label] if l])
 
     def connect_remote(self, *a, **kw):
         sup(self, *a, **kw)
@@ -653,7 +737,7 @@ class SSH(AbstractUrl, SlaveRemote):
         if m:
             u, h = m.groups()
         else:
-            u, h = pwd.getpwuid(os.geteuid()).pw_name, self.remote_addr
+            u, h = syncdutils.getusername(), self.remote_addr
         remote_addr = '@'.join([u, gethostbyname(h)])
         return ':'.join([remote_addr, self.inner_rsc.get_url(canonical=True)])
 
diff --git a/xlators/features/marker/utils/syncdaemon/syncdutils.py b/xlators/features/marker/utils/syncdaemon/syncdutils.py
index 6a08fbdaf9a..f82f412a014 100644
--- a/xlators/features/marker/utils/syncdaemon/syncdutils.py
+++ b/xlators/features/marker/utils/syncdaemon/syncdutils.py
@@ -1,5 +1,6 @@
 import os
 import sys
+import pwd
 import time
 import fcntl
 import shutil
@@ -216,3 +217,8 @@ class Thread(baseThread):
 
 class GsyncdError(Exception):
     pass
+
+def getusername(uid = None):
+    if uid == None:
+        uid = os.geteuid()
+    return pwd.getpwuid(uid).pw_name
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 6daf84a06c2..cb7f9769dbe 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -23,6 +23,7 @@
 #include "config.h"
 #endif
 #include <time.h>
+#include <grp.h>
 #include <sys/uio.h>
 #include <sys/resource.h>
 
@@ -299,10 +300,37 @@ glusterd_check_gsync_present ()
 
 }
 
-int
+static int
+group_write_allow (char *path, gid_t gid)
+{
+        struct stat st = {0,};
+        int ret        = 0;
+
+        ret = stat (path, &st);
+        if (ret == -1)
+                goto out;
+        GF_ASSERT (S_ISDIR (st.st_mode));
+
+        ret = chown (path, -1, gid);
+        if (ret == -1)
+                goto out;
+
+        ret = chmod (path, (st.st_mode & ~S_IFMT) | S_IWGRP|S_IXGRP|S_ISVTX);
+
+ out:
+        if (ret == -1)
+                gf_log ("", GF_LOG_CRITICAL,
+                        "failed to set up write access to %s for group %d (%s)",
+                        path, gid, strerror (errno));
+        return ret;
+}
+
+static int
 glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
 {
-        int ret = 0;
+        char *greplg_s   = NULL;
+        struct group *gr = NULL;
+        int ret          = 0;
 
         GF_ASSERT (georepdir);
         GF_ASSERT (conf);
@@ -351,11 +379,29 @@ glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
                         "Unable to create "GEOREP" slave log directory");
                 goto out;
         }
-        ret = 0;
+
+        ret = dict_get_str (THIS->options, GEOREP"-log-group", &greplg_s);
+        if (ret)
+                ret = 0;
+        else {
+                gr = getgrnam (greplg_s);
+                if (!gr) {
+                        gf_log ("glusterd", GF_LOG_CRITICAL,
+                                "group "GEOREP"-log-group %s does not exist", greplg_s);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP,
+                                         gr->gr_gid);
+                if (ret == 0)
+                        ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"
+                                                 GEOREP"-slaves", gr->gr_gid);
+        }
+
  out:
         gf_log("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
-
 }
 #endif
 
@@ -417,12 +463,17 @@ configure_syncdaemon (glusterd_conf_t *conf)
                          "/usr/local/libexec/glusterfs/gsyncd", ".", "^ssh:", NULL);
         RUN_GSYNCD_CMD;
 
-        /* gluster-command */
+        /* gluster-command-dir */
         /* XXX $sbindir should be used (throughout the codebase) */
         runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "gluster-command",
-                         GFS_PREFIX"/sbin/glusterfs "
-                          "--xlator-option *-dht.assert-no-child-down=true",
+        runner_add_args (&runner, "gluster-command-dir", GFS_PREFIX"/sbin/",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-params */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "gluster-params",
+                         "xlator-option=*-dht.assert-no-child-down=true",
                          ".", ".", NULL);
         RUN_GSYNCD_CMD;
 
@@ -470,11 +521,16 @@ configure_syncdaemon (glusterd_conf_t *conf)
          * slave pre-configuration
          ************/
 
-        /* gluster-command */
+        /* gluster-command-dir */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "gluster-command-dir", GFS_PREFIX"/sbin/",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-params */
         runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "gluster-command",
-                         GFS_PREFIX"/sbin/glusterfs "
-                          "--xlator-option *-dht.assert-no-child-down=true",
+        runner_add_args (&runner, "gluster-params",
+                         "xlator-option=*-dht.assert-no-child-down=true",
                          ".", NULL);
         RUN_GSYNCD_CMD;
 
@@ -1001,6 +1057,8 @@ struct volume_options options[] = {
         { .key  = {"mountbroker-"GEOREP".*"},
           .type = GF_OPTION_TYPE_ANY,
         },
-
+        { .key = {GEOREP"-log-group"},
+          .type = GF_OPTION_TYPE_ANY,
+        },
         { .key   = {NULL} },
 };