diff options
| -rwxr-xr-x | xlators/features/marker/utils/gsyncd.in | 2 | ||||
| -rw-r--r-- | xlators/features/marker/utils/syncdaemon/gsyncd.py | 5 | ||||
| -rw-r--r-- | xlators/features/marker/utils/syncdaemon/resource.py | 146 | ||||
| -rw-r--r-- | xlators/features/marker/utils/syncdaemon/syncdutils.py | 6 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd.c | 84 |
5 files changed, 197 insertions, 46 deletions
diff --git a/xlators/features/marker/utils/gsyncd.in b/xlators/features/marker/utils/gsyncd.in index a7af8c0b0b4..b517b341f96 100755 --- a/xlators/features/marker/utils/gsyncd.in +++ b/xlators/features/marker/utils/gsyncd.in @@ -42,7 +42,7 @@ else fi if [ $config_wanted = 1 ]; then - wd="`${gluster} system:: getwd`" + wd="`${gluster} --log-file=/dev/stderr system:: getwd`" if [ $? -eq 0 ]; then config_file="$wd/geo-replication/gsyncd.conf" fi diff --git a/xlators/features/marker/utils/syncdaemon/gsyncd.py b/xlators/features/marker/utils/syncdaemon/gsyncd.py index c0d39ffd62d..797000970a5 100644 --- a/xlators/features/marker/utils/syncdaemon/gsyncd.py +++ b/xlators/features/marker/utils/syncdaemon/gsyncd.py @@ -139,9 +139,12 @@ def main_i(): return lambda o, oo, vx, p: store_local(o, oo, FreeObject(op=op, **dmake(vx)), p) op = OptionParser(usage="%prog [options...] <master> <slave>", version="%prog 0.0.1") - op.add_option('--gluster-command', metavar='CMD', default='glusterfs') + op.add_option('--gluster-command-dir', metavar='DIR', default='') op.add_option('--gluster-log-file', metavar='LOGF', default=os.devnull, type=str, action='callback', callback=store_abs) op.add_option('--gluster-log-level', metavar='LVL') + op.add_option('--gluster-params', metavar='PRMS', default='') + op.add_option('--gluster-cli-options', metavar='OPTS', default='--log-file=/dev/stderr') + op.add_option('--mountbroker', metavar='LABEL') op.add_option('-p', '--pid-file', metavar='PIDF', type=str, action='callback', callback=store_abs) op.add_option('-l', '--log-file', metavar='LOGF', type=str, action='callback', callback=store_abs) op.add_option('--state-file', metavar='STATF', type=str, action='callback', callback=store_abs) diff --git a/xlators/features/marker/utils/syncdaemon/resource.py b/xlators/features/marker/utils/syncdaemon/resource.py index f92e8573409..b851d661a9e 100644 --- a/xlators/features/marker/utils/syncdaemon/resource.py +++ b/xlators/features/marker/utils/syncdaemon/resource.py @@ -1,7 +1,6 @@ import re import os import sys -import pwd import stat import time import errno @@ -579,41 +578,126 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote): """determine our position in the connectibility matrix""" return True - def connect(self): - """inhibit the resource beyond + class Mounter(object): + """Abstract base class for mounter backends""" - - create temprorary mount point - - call glusterfs to mount the volume over there - - change to mounted fs root - - lazy umount + delete temp. mount point - """ - def umount_l(d): - po = Popen(['umount', '-l', d], stderr=subprocess.PIPE) + def __init__(self, params): + self.params = params + + @classmethod + def get_glusterprog(cls): + return os.path.join(gconf.gluster_command_dir, cls.glusterprog) + + def umount_l(self, d): + """perform lazy umount""" + po = Popen(self.make_umount_argv(d), stderr=subprocess.PIPE) po.wait() return po - d = tempfile.mkdtemp(prefix='gsyncd-aux-mount-') - mounted = False - try: - po = Popen(gconf.gluster_command.split() + \ - (gconf.gluster_log_level and ['-L', gconf.gluster_log_level] or []) + \ - ['-l', gconf.gluster_log_file, '-s', self.host, - '--volfile-id', self.volume, '--client-pid=-1', d], - stderr=subprocess.PIPE) + + @classmethod + def make_umount_argv(cls, d): + raise NotImplementedError + + def make_mount_argv(self, *a): + raise NotImplementedError + + def cleanup_mntpt(self): + pass + + def handle_mounter(self, po): po.wait() - po.terminate_geterr() - mounted = True - logging.debug('auxiliary glusterfs mount in place') - os.chdir(d) - umount_l(d).terminate_geterr() + + def inhibit(self, *a): + """inhibit a gluster filesystem + + Mount glusterfs over a temporary mountpoint, + change into the mount, and lazy unmount the + filesystem. + """ mounted = False - finally: try: - if mounted: - umount_l(d).terminate_geterr(fail_on_err = False) - os.rmdir(d) - except: - logging.warn('stale mount possibly left behind on ' + d) - logging.debug('auxiliary glusterfs mount prepared') + po = Popen(self.make_mount_argv(*a), **self.mountkw) + self.handle_mounter(po) + po.terminate_geterr() + d = self.mntpt + mounted = True + logging.debug('auxiliary glusterfs mount in place') + os.chdir(d) + self.umount_l(d).terminate_geterr() + mounted = False + finally: + try: + if mounted: + self.umount_l(d).terminate_geterr(fail_on_err = False) + self.cleanup_mntpt() + except: + logging.warn('stale mount possibly left behind on ' + d) + logging.debug('auxiliary glusterfs mount prepared') + + class DirectMounter(Mounter): + """mounter backend which calls mount(8), umount(8) directly""" + + mountkw = {'stderr': subprocess.PIPE} + glusterprog = 'glusterfs' + + @staticmethod + def make_umount_argv(d): + return ['umount', '-l', d] + + def make_mount_argv(self): + self.mntpt = tempfile.mkdtemp(prefix = 'gsyncd-aux-mount-') + return [self.get_glusterprog()] + ['--' + p for p in self.params] + [self.mntpt] + + def cleanup_mntpt(self): + os.rmdir(self.mntpt) + + class MountbrokerMounter(Mounter): + """mounter backend using the mountbroker gluster service""" + + mountkw = {'stderr': subprocess.PIPE, 'stdout': subprocess.PIPE} + glusterprog = 'gluster' + + @classmethod + def make_cli_argv(cls): + return [cls.get_glusterprog()] + gconf.gluster_cli_options.split() + ['system::'] + + @classmethod + def make_umount_argv(cls, d): + return cls.make_cli_argv() + ['umount', d, 'lazy'] + + def make_mount_argv(self, label): + return self.make_cli_argv() + \ + ['mount', label, 'user-map-root=' + syncdutils.getusername()] + self.params + + def handle_mounter(self, po): + self.mntpt = po.stdout.readline()[:-1] + po.stdout.close() + sup(self, po) + if po.returncode != 0: + # if cli terminated with error due to being + # refused by glusterd, what it put + # out on stdout is a diagnostic message + logging.error('glusterd answered: %s' % self.mntpt) + + def connect(self): + """inhibit the resource beyond + + Choose mounting backend (direct or mountbroker), + set up glusterfs parameters and perform the mount + with given backend + """ + + label = getattr(gconf, 'mountbroker', None) + if not label: + uid = os.geteuid() + if uid != 0: + label = syncdutils.getusername(uid) + mounter = label and self.MountbrokerMounter or self.DirectMounter + params = gconf.gluster_params.split() + \ + (gconf.gluster_log_level and ['log-level=' + gconf.gluster_log_level] or []) + \ + ['log-file=' + gconf.gluster_log_file, 'volfile-server=' + self.host, + 'volfile-id=' + self.volume, 'client-pid=-1'] + mounter(params).inhibit(*[l for l in [label] if l]) def connect_remote(self, *a, **kw): sup(self, *a, **kw) @@ -653,7 +737,7 @@ class SSH(AbstractUrl, SlaveRemote): if m: u, h = m.groups() else: - u, h = pwd.getpwuid(os.geteuid()).pw_name, self.remote_addr + u, h = syncdutils.getusername(), self.remote_addr remote_addr = '@'.join([u, gethostbyname(h)]) return ':'.join([remote_addr, self.inner_rsc.get_url(canonical=True)]) diff --git a/xlators/features/marker/utils/syncdaemon/syncdutils.py b/xlators/features/marker/utils/syncdaemon/syncdutils.py index 6a08fbdaf9a..f82f412a014 100644 --- a/xlators/features/marker/utils/syncdaemon/syncdutils.py +++ b/xlators/features/marker/utils/syncdaemon/syncdutils.py @@ -1,5 +1,6 @@ import os import sys +import pwd import time import fcntl import shutil @@ -216,3 +217,8 @@ class Thread(baseThread): class GsyncdError(Exception): pass + +def getusername(uid = None): + if uid == None: + uid = os.geteuid() + return pwd.getpwuid(uid).pw_name diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c index 6daf84a06c2..cb7f9769dbe 100644 --- a/xlators/mgmt/glusterd/src/glusterd.c +++ b/xlators/mgmt/glusterd/src/glusterd.c @@ -23,6 +23,7 @@ #include "config.h" #endif #include <time.h> +#include <grp.h> #include <sys/uio.h> #include <sys/resource.h> @@ -299,10 +300,37 @@ glusterd_check_gsync_present () } -int +static int +group_write_allow (char *path, gid_t gid) +{ + struct stat st = {0,}; + int ret = 0; + + ret = stat (path, &st); + if (ret == -1) + goto out; + GF_ASSERT (S_ISDIR (st.st_mode)); + + ret = chown (path, -1, gid); + if (ret == -1) + goto out; + + ret = chmod (path, (st.st_mode & ~S_IFMT) | S_IWGRP|S_IXGRP|S_ISVTX); + + out: + if (ret == -1) + gf_log ("", GF_LOG_CRITICAL, + "failed to set up write access to %s for group %d (%s)", + path, gid, strerror (errno)); + return ret; +} + +static int glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf) { - int ret = 0; + char *greplg_s = NULL; + struct group *gr = NULL; + int ret = 0; GF_ASSERT (georepdir); GF_ASSERT (conf); @@ -351,11 +379,29 @@ glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf) "Unable to create "GEOREP" slave log directory"); goto out; } - ret = 0; + + ret = dict_get_str (THIS->options, GEOREP"-log-group", &greplg_s); + if (ret) + ret = 0; + else { + gr = getgrnam (greplg_s); + if (!gr) { + gf_log ("glusterd", GF_LOG_CRITICAL, + "group "GEOREP"-log-group %s does not exist", greplg_s); + ret = -1; + goto out; + } + + ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP, + gr->gr_gid); + if (ret == 0) + ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/" + GEOREP"-slaves", gr->gr_gid); + } + out: gf_log("", GF_LOG_DEBUG, "Returning %d", ret); return ret; - } #endif @@ -417,12 +463,17 @@ configure_syncdaemon (glusterd_conf_t *conf) "/usr/local/libexec/glusterfs/gsyncd", ".", "^ssh:", NULL); RUN_GSYNCD_CMD; - /* gluster-command */ + /* gluster-command-dir */ /* XXX $sbindir should be used (throughout the codebase) */ runinit_gsyncd_setrx (&runner, conf); - runner_add_args (&runner, "gluster-command", - GFS_PREFIX"/sbin/glusterfs " - "--xlator-option *-dht.assert-no-child-down=true", + runner_add_args (&runner, "gluster-command-dir", GFS_PREFIX"/sbin/", + ".", ".", NULL); + RUN_GSYNCD_CMD; + + /* gluster-params */ + runinit_gsyncd_setrx (&runner, conf); + runner_add_args (&runner, "gluster-params", + "xlator-option=*-dht.assert-no-child-down=true", ".", ".", NULL); RUN_GSYNCD_CMD; @@ -470,11 +521,16 @@ configure_syncdaemon (glusterd_conf_t *conf) * slave pre-configuration ************/ - /* gluster-command */ + /* gluster-command-dir */ + runinit_gsyncd_setrx (&runner, conf); + runner_add_args (&runner, "gluster-command-dir", GFS_PREFIX"/sbin/", + ".", NULL); + RUN_GSYNCD_CMD; + + /* gluster-params */ runinit_gsyncd_setrx (&runner, conf); - runner_add_args (&runner, "gluster-command", - GFS_PREFIX"/sbin/glusterfs " - "--xlator-option *-dht.assert-no-child-down=true", + runner_add_args (&runner, "gluster-params", + "xlator-option=*-dht.assert-no-child-down=true", ".", NULL); RUN_GSYNCD_CMD; @@ -1001,6 +1057,8 @@ struct volume_options options[] = { { .key = {"mountbroker-"GEOREP".*"}, .type = GF_OPTION_TYPE_ANY, }, - + { .key = {GEOREP"-log-group"}, + .type = GF_OPTION_TYPE_ANY, + }, { .key = {NULL} }, }; |