geo-rep: partial support for unprivileged gsyncd via mountbroker

gsyncd:
  - mounting code is split to a direct and a mountbroker based backend
  - option gluster-command gone
  - new options: gluster-params, gluster-cli-options, mountbroker
  - mountbroker mount backend is used if either a mountbroker label
    is given through the mountbroker option, or if gsyncd is
    unprivileged; in this case the username is used as label
  - have gluster cli invocations log to stderr so that we don't
    hit a permission issue with the logfiles
glusterd:
  - do gsyncd pre-config with new options
  - add option geo-replication-log-group, so if that specified
    geo-rep logfile directories are given to that group (and
    thus members of the given group can do logging there)

This is just WIP as geo-rep relies on trusted extended attributes
and those are not accessible for unprivileged users. Even if we
solved this issue, glusterd security settings are too coarse,
so that if we made it possible for an unprivileged gsyncd
to operate, we would open up too far.

Change-Id: Icd520b58cbadccea3fad7c0f437b99de1e22db14
BUG: 2825
Reviewed-on: http://review.gluster.com/399
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

4 files changed, 126 insertions, 33 deletions

diff --git a/xlators/features/marker/utils/gsyncd.in b/xlators/features/marker/utils/gsyncd.in
index a7af8c0b0..b517b341f 100755
--- a/xlators/features/marker/utils/gsyncd.in
+++ b/xlators/features/marker/utils/gsyncd.in
@@ -42,7 +42,7 @@ else
 fi
 
 if [ $config_wanted = 1 ]; then
-    wd="`${gluster} system:: getwd`"
+    wd="`${gluster} --log-file=/dev/stderr system:: getwd`"
     if [ $? -eq 0 ]; then
         config_file="$wd/geo-replication/gsyncd.conf"
     fi
diff --git a/xlators/features/marker/utils/syncdaemon/gsyncd.py b/xlators/features/marker/utils/syncdaemon/gsyncd.py
index c0d39ffd6..797000970 100644
--- a/xlators/features/marker/utils/syncdaemon/gsyncd.py
+++ b/xlators/features/marker/utils/syncdaemon/gsyncd.py
@@ -139,9 +139,12 @@ def main_i():
         return lambda o, oo, vx, p: store_local(o, oo, FreeObject(op=op, **dmake(vx)), p)
 
     op = OptionParser(usage="%prog [options...] <master> <slave>", version="%prog 0.0.1")
-    op.add_option('--gluster-command',     metavar='CMD',   default='glusterfs')
+    op.add_option('--gluster-command-dir', metavar='DIR',   default='')
     op.add_option('--gluster-log-file',    metavar='LOGF',  default=os.devnull, type=str, action='callback', callback=store_abs)
     op.add_option('--gluster-log-level',   metavar='LVL')
+    op.add_option('--gluster-params',      metavar='PRMS',  default='')
+    op.add_option('--gluster-cli-options', metavar='OPTS',  default='--log-file=/dev/stderr')
+    op.add_option('--mountbroker',         metavar='LABEL')
     op.add_option('-p', '--pid-file',      metavar='PIDF',  type=str, action='callback', callback=store_abs)
     op.add_option('-l', '--log-file',      metavar='LOGF',  type=str, action='callback', callback=store_abs)
     op.add_option('--state-file',          metavar='STATF', type=str, action='callback', callback=store_abs)
diff --git a/xlators/features/marker/utils/syncdaemon/resource.py b/xlators/features/marker/utils/syncdaemon/resource.py
index f92e85734..b851d661a 100644
--- a/xlators/features/marker/utils/syncdaemon/resource.py
+++ b/xlators/features/marker/utils/syncdaemon/resource.py
@@ -1,7 +1,6 @@
 import re
 import os
 import sys
-import pwd
 import stat
 import time
 import errno
@@ -579,41 +578,126 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
         """determine our position in the connectibility matrix"""
         return True
 
-    def connect(self):
-        """inhibit the resource beyond
+    class Mounter(object):
+        """Abstract base class for mounter backends"""
 
-        - create temprorary mount point
-        - call glusterfs to mount the volume over there
-        - change to mounted fs root
-        - lazy umount + delete temp. mount point
-        """
-        def umount_l(d):
-            po = Popen(['umount', '-l', d], stderr=subprocess.PIPE)
+        def __init__(self, params):
+            self.params = params
+
+        @classmethod
+        def get_glusterprog(cls):
+            return os.path.join(gconf.gluster_command_dir, cls.glusterprog)
+
+        def umount_l(self, d):
+            """perform lazy umount"""
+            po = Popen(self.make_umount_argv(d), stderr=subprocess.PIPE)
             po.wait()
             return po
-        d = tempfile.mkdtemp(prefix='gsyncd-aux-mount-')
-        mounted = False
-        try:
-            po = Popen(gconf.gluster_command.split() + \
-                       (gconf.gluster_log_level and ['-L', gconf.gluster_log_level] or []) + \
-                       ['-l', gconf.gluster_log_file, '-s', self.host,
-                        '--volfile-id', self.volume, '--client-pid=-1', d],
-                       stderr=subprocess.PIPE)
+
+        @classmethod
+        def make_umount_argv(cls, d):
+            raise NotImplementedError
+
+        def make_mount_argv(self, *a):
+            raise NotImplementedError
+
+        def cleanup_mntpt(self):
+            pass
+
+        def handle_mounter(self, po):
             po.wait()
-            po.terminate_geterr()
-            mounted = True
-            logging.debug('auxiliary glusterfs mount in place')
-            os.chdir(d)
-            umount_l(d).terminate_geterr()
+
+        def inhibit(self, *a):
+            """inhibit a gluster filesystem
+
+            Mount glusterfs over a temporary mountpoint,
+            change into the mount, and lazy unmount the
+            filesystem.
+            """
             mounted = False
-        finally:
             try:
-                if mounted:
-                    umount_l(d).terminate_geterr(fail_on_err = False)
-                os.rmdir(d)
-            except:
-                logging.warn('stale mount possibly left behind on ' + d)
-        logging.debug('auxiliary glusterfs mount prepared')
+                po = Popen(self.make_mount_argv(*a), **self.mountkw)
+                self.handle_mounter(po)
+                po.terminate_geterr()
+                d = self.mntpt
+                mounted = True
+                logging.debug('auxiliary glusterfs mount in place')
+                os.chdir(d)
+                self.umount_l(d).terminate_geterr()
+                mounted = False
+            finally:
+                try:
+                    if mounted:
+                        self.umount_l(d).terminate_geterr(fail_on_err = False)
+                    self.cleanup_mntpt()
+                except:
+                    logging.warn('stale mount possibly left behind on ' + d)
+            logging.debug('auxiliary glusterfs mount prepared')
+
+    class DirectMounter(Mounter):
+        """mounter backend which calls mount(8), umount(8) directly"""
+
+        mountkw = {'stderr': subprocess.PIPE}
+        glusterprog = 'glusterfs'
+
+        @staticmethod
+        def make_umount_argv(d):
+            return ['umount', '-l', d]
+
+        def make_mount_argv(self):
+            self.mntpt = tempfile.mkdtemp(prefix = 'gsyncd-aux-mount-')
+            return [self.get_glusterprog()] + ['--' + p for p in self.params] + [self.mntpt]
+
+        def cleanup_mntpt(self):
+            os.rmdir(self.mntpt)
+
+    class MountbrokerMounter(Mounter):
+        """mounter backend using the mountbroker gluster service"""
+
+        mountkw = {'stderr': subprocess.PIPE, 'stdout': subprocess.PIPE}
+        glusterprog = 'gluster'
+
+        @classmethod
+        def make_cli_argv(cls):
+            return [cls.get_glusterprog()] + gconf.gluster_cli_options.split() + ['system::']
+
+        @classmethod
+        def make_umount_argv(cls, d):
+            return cls.make_cli_argv() + ['umount', d, 'lazy']
+
+        def make_mount_argv(self, label):
+            return self.make_cli_argv() + \
+                   ['mount', label, 'user-map-root=' + syncdutils.getusername()] + self.params
+
+        def handle_mounter(self, po):
+            self.mntpt = po.stdout.readline()[:-1]
+            po.stdout.close()
+            sup(self, po)
+            if po.returncode != 0:
+                # if cli terminated with error due to being
+                # refused by glusterd, what it put
+                # out on stdout is a diagnostic message
+                logging.error('glusterd answered: %s' % self.mntpt)
+
+    def connect(self):
+        """inhibit the resource beyond
+
+        Choose mounting backend (direct or mountbroker),
+        set up glusterfs parameters and perform the mount
+        with given backend
+        """
+
+        label = getattr(gconf, 'mountbroker', None)
+        if not label:
+            uid = os.geteuid()
+            if uid != 0:
+                label = syncdutils.getusername(uid)
+        mounter = label and self.MountbrokerMounter or self.DirectMounter
+        params = gconf.gluster_params.split() + \
+                   (gconf.gluster_log_level and ['log-level=' + gconf.gluster_log_level] or []) + \
+                   ['log-file=' + gconf.gluster_log_file, 'volfile-server=' + self.host,
+                    'volfile-id=' + self.volume, 'client-pid=-1']
+        mounter(params).inhibit(*[l for l in [label] if l])
 
     def connect_remote(self, *a, **kw):
         sup(self, *a, **kw)
@@ -653,7 +737,7 @@ class SSH(AbstractUrl, SlaveRemote):
         if m:
             u, h = m.groups()
         else:
-            u, h = pwd.getpwuid(os.geteuid()).pw_name, self.remote_addr
+            u, h = syncdutils.getusername(), self.remote_addr
         remote_addr = '@'.join([u, gethostbyname(h)])
         return ':'.join([remote_addr, self.inner_rsc.get_url(canonical=True)])
 
diff --git a/xlators/features/marker/utils/syncdaemon/syncdutils.py b/xlators/features/marker/utils/syncdaemon/syncdutils.py
index 6a08fbdaf..f82f412a0 100644
--- a/xlators/features/marker/utils/syncdaemon/syncdutils.py
+++ b/xlators/features/marker/utils/syncdaemon/syncdutils.py
@@ -1,5 +1,6 @@
 import os
 import sys
+import pwd
 import time
 import fcntl
 import shutil
@@ -216,3 +217,8 @@ class Thread(baseThread):
 
 class GsyncdError(Exception):
     pass
+
+def getusername(uid = None):
+    if uid == None:
+        uid = os.geteuid()
+    return pwd.getpwuid(uid).pw_name