diff options
Diffstat (limited to 'extras')
193 files changed, 19113 insertions, 2609 deletions
diff --git a/extras/FreeBSD/Makefile b/extras/FreeBSD/Makefile index e366d6f16a8..26fe85c885f 100644 --- a/extras/FreeBSD/Makefile +++ b/extras/FreeBSD/Makefile @@ -20,7 +20,7 @@ BUILD_DEPENDS= fusefs-libs>2.6.3:${PORTSDIR}/sysutils/fusefs-libs LIB_DEPENDS= fuse.2:${PORTSDIR}/sysutils/fusefs-libs RUN_DEPENDS= ${LOCALBASE}/modules/fuse.ko:${PORTSDIR}/sysutils/fusefs-kmod -MAN8= glusterfs.8 +MAN8= glusterfs.8 gluster.8 glusterd.8 glusterfs-volgen.8 GNU_CONFIGURE= yes USE_LDCONFIG= yes USE_AUTOTOOLS= libtool:15 diff --git a/extras/LinuxRPM/Makefile.am b/extras/LinuxRPM/Makefile.am new file mode 100644 index 00000000000..f02853798c0 --- /dev/null +++ b/extras/LinuxRPM/Makefile.am @@ -0,0 +1,55 @@ + +GFS_TAR = ../../glusterfs-$(VERSION).tar.gz + +.PHONY: all + +all: + @echo "To build RPMS run 'make glusterrpms'" + +.PHONY: glusterrpms glusterrpms_without_autogen +.PHONY: autogen prep srcrpm testsrpm clean + +glusterrpms: autogen glusterrpms_without_autogen + +glusterrpms_without_autogen: prep srcrpm rpms + -rm -rf rpmbuild + +autogen: + cd ../.. && \ + rm -rf autom4te.cache && \ + ./autogen.sh && \ + ./configure --enable-gnfs --with-previous-options + +prep: + $(MAKE) -C ../.. dist; + -mkdir -p rpmbuild/BUILD + -mkdir -p rpmbuild/SPECS + -mkdir -p rpmbuild/RPMS + -mkdir -p rpmbuild/SRPMS + -mkdir -p rpmbuild/SOURCES + -rm -rf rpmbuild/SOURCES/* + cp ../../*.tar.gz ./rpmbuild/SOURCES + cp ../../glusterfs.spec ./rpmbuild/SPECS + +srcrpm: + rpmbuild --define '_topdir $(shell pwd)/rpmbuild' -bs rpmbuild/SPECS/glusterfs.spec + mv rpmbuild/SRPMS/* . + +rpms: + rpmbuild --define '_topdir $(shell pwd)/rpmbuild' --with gnfs -bb rpmbuild/SPECS/glusterfs.spec + mv rpmbuild/RPMS/*/* . + +# EPEL-5 does not like new versions of rpmbuild and requires some +# _source_* defines + +testsrpm: prep + rpmbuild --define '_topdir $(shell pwd)/rpmbuild' \ + --define '_source_payload w9.gzdio' \ + --define '_source_filedigest_algorithm 1' \ + -bs rpmbuild/SPECS/glusterfs.spec + mv rpmbuild/SRPMS/* ../.. + -rm -rf rpmbuild + +clean: + -rm -rf rpmbuild + -rm -f *.rpm diff --git a/extras/LinuxRPM/make_glusterrpms b/extras/LinuxRPM/make_glusterrpms new file mode 100755 index 00000000000..3156af97870 --- /dev/null +++ b/extras/LinuxRPM/make_glusterrpms @@ -0,0 +1,9 @@ +#!/bin/sh + +( + cd $(realpath $(dirname $0))/../.. || exit 1 + rm -rf autom4te.cache + ./autogen.sh || exit 1 + ./configure --with-previous-options || exit 1 +) +make -C $(realpath $(dirname $0)) glusterrpms_without_autogen diff --git a/extras/MacOSX/Portfile b/extras/MacOSX/Portfile deleted file mode 100644 index 1262a44daea..00000000000 --- a/extras/MacOSX/Portfile +++ /dev/null @@ -1,26 +0,0 @@ -# $Id$ - -PortSystem 1.0 - -name glusterfs -version 2.0.0rc8 -categories fuse -maintainers amar@gluster.com -description GlusterFS -long_description GlusterFS is a cluster file system, flexible to tune it for your needs. -homepage http://www.gluster.org/ -platforms darwin -master_sites http://ftp.gluster.com/pub/gluster/glusterfs/2.0/2.0.0 - -configure.args --disable-bdb -checksums md5 33c2d02344d4fab422e80cfb637e0b48 - -post-destroot { - file mkdir ${destroot}/Library/LaunchDaemons/ - file copy ${worksrcpath}/extras/glusterfs-server.plist \ - ${destroot}/Library/LaunchDaemons/com.gluster.glusterfs.plist - - file mkdir ${destroot}/sbin/ - file copy ${worksrcpath}/xlators/mount/fuse/utils/mount_glusterfs \ - ${destroot}/sbin/ -}
\ No newline at end of file diff --git a/extras/MacOSX/README.MacOSX b/extras/MacOSX/README.MacOSX deleted file mode 100644 index ec7abc2cc78..00000000000 --- a/extras/MacOSX/README.MacOSX +++ /dev/null @@ -1,84 +0,0 @@ - -Mostly the usage is over OS-X. - -Important links - -GlusterFS -========= -* http://www.gluster.org -* http://gluster.org/docs/index.php/GlusterFS -* http://gluster.org/docs/index.php/GlusterFS_on_MAC_OS_X - -MacFUSE -======= -* http://code.google.com/p/macfuse/ -* http://code.google.com/p/macfuse/wiki/FAQ - - -Important steps: -================ - -Make sure that there is no previous installation of MacFUSE exists in the -system. Run, - - bash# sudo /Library/Filesystems/fusefs.fs/Support/uninstall-macfuse-core.sh - -After this, install MacFUSE (mostly through .dmg available in macfuse homepage -or if Gluster Inc provides any custom built .dmg) - -Make sure the .dmg of glusterfs is built against the installed MacFUSE version -(if not, any operations over mountpoint gives EIO ie, Input/Output Error). If -glusterfs tarball is used then compile it only after the MacFUSE installation -is complete. - -To make an entry in /etc/fstab for glusterfs mount, use 'vifs' command - - bash# sudo vifs - -after the entry is added in /etc/fstab, it can be mounted by 'mount' command. - -To start the server process one can use the 'launchd' mechanism. Follow below -steps after installation - - bash# launchctl load /Library/LaunchDaemons/com.gluster.glusterfs.plist - -No need to run the command if the machine reboots, as it will be loaded -automatically by launchd process. - -Now copy the server volume file in the proper path - - bash# sudo vi /opt/local/etc/glusterfs/server.vol - -NOTE: (If glusterfs is installed in different path other than '/opt/local' - update the volume file at the corresponding path, and also need to - update the /Library/LaunchDaemons/com.gluster.glusterfs.plist with - the proper path) - -Once the volume file is updated, administrator can start the server process by -running, - - bash# launchctl start com.gluster.glusterfs - -and stop like - - bash# launchctl stop com.gluster.glusterfs - -NOTE: To start the process by default when the loaded, one need to add the -following lines to .plist file - ----- - <key>KeepAlive</key> - <true/> - ----- - - - - -Install using .dmg in Terminal -================================= - - bash# hdiutil attach <package>.dmg - bash# cd /Volumes/<package>/ - bash# installer -pkg <package>.pkg -installer / - bash# cd - bash# hdiutil detach /Volumes/<package>/ - diff --git a/extras/Makefile.am b/extras/Makefile.am index 8ad2f6302ff..983f014cca6 100644 --- a/extras/Makefile.am +++ b/extras/Makefile.am @@ -1,9 +1,77 @@ +addonexecdir = $(GLUSTERFS_LIBEXECDIR) +addonexec_SCRIPTS = +if WITH_SERVER +addonexec_SCRIPTS += peer_add_secret_pub +if USE_SYSTEMD +addonexec_SCRIPTS += mount-shared-storage.sh +endif +endif -docdir = $(datadir)/doc/glusterfs/ -EditorModedir = $(docdir)/ +EditorModedir = $(docdir) EditorMode_DATA = glusterfs-mode.el glusterfs.vim -SUBDIRS = init.d benchmarking volgen +SUBDIRS = init.d systemd benchmarking hook-scripts $(OCF_SUBDIR) LinuxRPM \ + $(GEOREP_EXTRAS_SUBDIR) snap_scheduler firewalld cliutils python \ + ganesha -EXTRA_DIST = specgen.scm MacOSX/Portfile glusterfs-mode.el glusterfs.vim migrate-unify-to-distribute.sh backend-xattr-sanitize.sh backend-cleanup.sh defrag.sh scale-n-defrag.sh +confdir = $(sysconfdir)/glusterfs +if WITH_SERVER +conf_DATA = glusterfs-logrotate gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf \ + logger.conf.example glusterfs-georep-logrotate group-virt.example \ + group-metadata-cache group-gluster-block group-nl-cache \ + group-db-workload group-distributed-virt group-samba +endif +voldir = $(sysconfdir)/glusterfs +vol_DATA = thin-arbiter/thin-arbiter.vol +if WITH_SERVER +vol_DATA += glusterd.vol +endif + +scriptsdir = $(datadir)/glusterfs/scripts +scripts_SCRIPTS = thin-arbiter/setup-thin-arbiter.sh +if WITH_SERVER +scripts_SCRIPTS += post-upgrade-script-for-quota.sh \ + pre-upgrade-script-for-quota.sh stop-all-gluster-processes.sh +if USE_SYSTEMD +scripts_SCRIPTS += control-cpu-load.sh +scripts_SCRIPTS += control-mem.sh +endif +endif + +EXTRA_DIST = glusterfs-logrotate gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf \ + logger.conf.example glusterfs-georep-logrotate group-virt.example \ + group-metadata-cache group-gluster-block group-nl-cache \ + group-db-workload group-samba specgen.scm glusterfs-mode.el glusterfs.vim \ + migrate-unify-to-distribute.sh backend-xattr-sanitize.sh \ + backend-cleanup.sh disk_usage_sync.sh clear_xattrs.sh \ + glusterd-sysconfig glusterd.vol post-upgrade-script-for-quota.sh \ + pre-upgrade-script-for-quota.sh command-completion/gluster.bash \ + command-completion/Makefile command-completion/README \ + stop-all-gluster-processes.sh clang-checker.sh mount-shared-storage.sh \ + control-cpu-load.sh control-mem.sh group-distributed-virt \ + thin-arbiter/thin-arbiter.vol thin-arbiter/setup-thin-arbiter.sh + +if WITH_SERVER +install-data-local: + if [ -n "$(tmpfilesdir)" ]; then \ + $(mkdir_p) $(DESTDIR)$(tmpfilesdir); \ + $(INSTALL_DATA) run-gluster.tmpfiles \ + $(DESTDIR)$(tmpfilesdir)/gluster.conf; \ + fi + $(mkdir_p) $(DESTDIR)$(GLUSTERD_WORKDIR)/groups + $(INSTALL_DATA) $(top_srcdir)/extras/group-virt.example \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/virt + $(INSTALL_DATA) $(top_srcdir)/extras/group-metadata-cache \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/metadata-cache + $(INSTALL_DATA) $(top_srcdir)/extras/group-gluster-block \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/gluster-block + $(INSTALL_DATA) $(top_srcdir)/extras/group-nl-cache \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/nl-cache + $(INSTALL_DATA) $(top_srcdir)/extras/group-db-workload \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/db-workload + $(INSTALL_DATA) $(top_srcdir)/extras/group-distributed-virt \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/distributed-virt + $(INSTALL_DATA) $(top_srcdir)/extras/group-samba \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/samba +endif diff --git a/extras/Solaris/Prototype b/extras/Solaris/Prototype index 369234210a9..cb2d99d8c67 100644 --- a/extras/Solaris/Prototype +++ b/extras/Solaris/Prototype @@ -9,6 +9,9 @@ d none /usr/sfw/share 0755 root bin d none /usr/sfw/share/man 0755 root bin d none /usr/sfw/share/man/man8 0755 root bin f none /usr/sfw/share/man/man8/glusterfs.8 0644 root root +f none /usr/sfw/share/man/man8/gluster.8 0644 root root +f none /usr/sfw/share/man/man8/glusterd.8 0644 root root +f none /usr/sfw/share/man/man8/glusterfs-volgen.8 0644 root root d none /usr/sfw/share/doc 0755 root bin d none /usr/sfw/share/doc/glusterfs 0755 root root d none /usr/sfw/share/doc/glusterfs/examples 0755 root root diff --git a/extras/Solaris/README.solaris b/extras/Solaris/README.solaris index ddb70c22c0e..5a4e4336d24 100644 --- a/extras/Solaris/README.solaris +++ b/extras/Solaris/README.solaris @@ -10,7 +10,7 @@ $ which glusterfs $ glusterfs --version glusterfs 2.0.0rc1 built on Jan 16 2009 03:36:59 Repository revision: glusterfs--mainline--3.0--patch-844 -Copyright (c) 2006, 2007, 2008 Gluster Inc. <http://www.gluster.com> +Copyright (c) 2006-2011 Gluster Inc. <http://www.gluster.com> GlusterFS comes with ABSOLUTELY NO WARRANTY. You may redistribute copies of GlusterFS under the terms of the GNU General Public License. diff --git a/extras/Ubuntu/README.Ubuntu b/extras/Ubuntu/README.Ubuntu new file mode 100644 index 00000000000..890da3ca614 --- /dev/null +++ b/extras/Ubuntu/README.Ubuntu @@ -0,0 +1,26 @@ +Bug 765014 - Mounting from localhost in fstab fails at boot on ubuntu +(original bug: https://bugzilla.redhat.com/show_bug.cgi?id=765014) +(updated in: https://bugzilla.redhat.com/show_bug.cgi?id=1047007) +(https://bugs.launchpad.net/ubuntu/+source/glusterfs/+bug/876648) + +Ubuntu uses upstart instead of init to bootstrap the system and it has a unique +way of handling fstab, using a program called mountall(8). As a result of using +a debian initscript to start glusterd, glusterfs mounts in fstab are tried before +the glusterd service is running. In the case where the client is also a server +and the volume is mounted from localhost, the mount fails at boot time. To +correct this we need to launch glusterd using upstart and block the glusterfs +mounting event until glusterd is started. + +The glusterfs-server.conf file contains the necessary configuration for upstart to +manage the glusterd service. It should be placed in /etc/init/glusterfs-server.conf +on Ubuntu systems, and then the old initscript /etc/init.d/glusterd can be +removed. An additional upstart job, mounting-glusterfs.conf, is also required +to block mounting glusterfs volumes until the network interfaces are available. +Both of these upstart jobs need to be placed in /etc/init to resolve the issue. + +Starting with Ubuntu 14.04, Trusty Tahr, these upstart jobs will be included +with the glusterfs-server and glusterfs-client packages in the Ubuntu +universe repository. + +This affects all versions of glusterfs on the Ubuntu platform since at least +10.04, Lucid Lynx. diff --git a/extras/Ubuntu/glusterfs-server.conf b/extras/Ubuntu/glusterfs-server.conf new file mode 100644 index 00000000000..aa99502b0a7 --- /dev/null +++ b/extras/Ubuntu/glusterfs-server.conf @@ -0,0 +1,10 @@ +author "Louis Zuckerman <me@louiszuckerman.com>" +description "GlusterFS Management Daemon" + +start on runlevel [2345] +stop on runlevel [016] + +expect fork + +exec /usr/sbin/glusterd -p /var/run/glusterd.pid + diff --git a/extras/Ubuntu/mounting-glusterfs.conf b/extras/Ubuntu/mounting-glusterfs.conf new file mode 100644 index 00000000000..786ef16df04 --- /dev/null +++ b/extras/Ubuntu/mounting-glusterfs.conf @@ -0,0 +1,6 @@ +author "Louis Zuckerman <me@louiszuckerman.com>" +description "Block the mounting event for glusterfs filesystems until the network interfaces are running" + +start on mounting TYPE=glusterfs +task +exec start wait-for-state WAIT_FOR=static-network-up WAITER=mounting-glusterfs diff --git a/extras/backend-cleanup.sh b/extras/backend-cleanup.sh index 755161f18c9..cb0a1d8871f 100644 --- a/extras/backend-cleanup.sh +++ b/extras/backend-cleanup.sh @@ -7,7 +7,7 @@ # This script has to be run on the servers, which are exporting the data to # GlusterFS # -# (c) 2009 Gluster Inc <http://www.gluster.com/> +# (c) 2010 Gluster Inc <http://www.gluster.com/> set -e @@ -17,7 +17,7 @@ export_directory="/export/glusterfs" clean_dir() { # Clean the 'link' files on backend - find "${export_directory}" -type f -perm +01000 -exec rm -v '{}' \; + find "${export_directory}" -type f -perm /01000 -exec rm -v '{}' \; } main() diff --git a/extras/benchmarking/Makefile.am b/extras/benchmarking/Makefile.am index 04cc06182ab..bfcc59277a5 100644 --- a/extras/benchmarking/Makefile.am +++ b/extras/benchmarking/Makefile.am @@ -1,7 +1,5 @@ -docdir = $(datadir)/doc/$(PACKAGE_NAME)/benchmarking - -benchmarkingdir = $(docdir) +benchmarkingdir = $(docdir)/benchmarking benchmarking_DATA = rdd.c glfs-bm.c README launch-script.sh local-script.sh diff --git a/extras/benchmarking/glfs-bm.c b/extras/benchmarking/glfs-bm.c index 42e367d7fa4..f7f5873f84d 100644 --- a/extras/benchmarking/glfs-bm.c +++ b/extras/benchmarking/glfs-bm.c @@ -1,28 +1,18 @@ -/* - Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com> - This file is part of GlusterFS. - GlusterFS is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - GlusterFS is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see - <http://www.gnu.org/licenses/>. +/* + Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. */ - #define _GNU_SOURCE #define __USE_FILE_OFFSET64 #define _FILE_OFFSET_BITS 64 #include <stdio.h> #include <argp.h> -#include <libglusterfsclient.h> #include <stdlib.h> #include <unistd.h> #include <sys/types.h> @@ -35,593 +25,338 @@ #include <sys/time.h> struct state { - char need_op_write:1; - char need_op_read:1; + char need_op_write : 1; + char need_op_read : 1; - char need_iface_fileio:1; - char need_iface_xattr:1; + char need_iface_fileio : 1; + char need_iface_xattr : 1; - char need_mode_posix:1; - char need_mode_libglusterfsclient:1; + char need_mode_posix : 1; - char prefix[512]; - long int count; + char prefix[512]; + long int count; - size_t block_size; + size_t block_size; - char *specfile; - void *libglusterfsclient_context; + char *specfile; - long int io_size; + long int io_size; }; - -#define MEASURE(func, arg) measure (func, #func, arg) - +#define MEASURE(func, arg) measure(func, #func, arg) void -tv_difference (struct timeval *tv_stop, - struct timeval *tv_start, - struct timeval *tv_diff) +tv_difference(struct timeval *tv_stop, struct timeval *tv_start, + struct timeval *tv_diff) { - if (tv_stop->tv_usec < tv_start->tv_usec) { - tv_diff->tv_usec = (tv_stop->tv_usec + 1000000) - tv_start->tv_usec; - tv_diff->tv_sec = (tv_stop->tv_sec - 1 - tv_start->tv_sec); - } else { - tv_diff->tv_usec = tv_stop->tv_usec - tv_start->tv_usec; - tv_diff->tv_sec = tv_stop->tv_sec - tv_start->tv_sec; - } + if (tv_stop->tv_usec < tv_start->tv_usec) { + tv_diff->tv_usec = (tv_stop->tv_usec + 1000000) - tv_start->tv_usec; + tv_diff->tv_sec = (tv_stop->tv_sec - 1 - tv_start->tv_sec); + } else { + tv_diff->tv_usec = tv_stop->tv_usec - tv_start->tv_usec; + tv_diff->tv_sec = tv_stop->tv_sec - tv_start->tv_sec; + } } - void -measure (int (*func)(struct state *state), - char *func_name, struct state *state) +measure(int (*func)(struct state *state), char *func_name, struct state *state) { - struct timeval tv_start, tv_stop, tv_diff; - state->io_size = 0; - long int count; + struct timeval tv_start, tv_stop, tv_diff; + state->io_size = 0; + long int count; - gettimeofday (&tv_start, NULL); - count = func (state); - gettimeofday (&tv_stop, NULL); + gettimeofday(&tv_start, NULL); + count = func(state); + gettimeofday(&tv_stop, NULL); - tv_difference (&tv_stop, &tv_start, &tv_diff); + tv_difference(&tv_stop, &tv_start, &tv_diff); - fprintf (stdout, "%s: count=%ld, size=%ld, time=%ld:%ld\n", - func_name, count, state->io_size, - tv_diff.tv_sec, tv_diff.tv_usec); + fprintf(stdout, "%s: count=%ld, size=%ld, time=%ld:%ld\n", func_name, count, + state->io_size, tv_diff.tv_sec, tv_diff.tv_usec); } - static error_t -parse_opts (int key, char *arg, - struct argp_state *_state) +parse_opts(int key, char *arg, struct argp_state *_state) { - struct state *state = _state->input; + struct state *state = _state->input; - switch (key) - { + switch (key) { case 'o': - if (strcasecmp (arg, "read") == 0) { - state->need_op_write = 0; - state->need_op_read = 1; - } else if (strcasecmp (arg, "write") == 0) { - state->need_op_write = 1; - state->need_op_read = 0; - } else if (strcasecmp (arg, "both") == 0) { - state->need_op_write = 1; - state->need_op_read = 1; - } else { - fprintf (stderr, "unknown op: %s\n", arg); - return -1; - } - break; + if (strcasecmp(arg, "read") == 0) { + state->need_op_write = 0; + state->need_op_read = 1; + } else if (strcasecmp(arg, "write") == 0) { + state->need_op_write = 1; + state->need_op_read = 0; + } else if (strcasecmp(arg, "both") == 0) { + state->need_op_write = 1; + state->need_op_read = 1; + } else { + fprintf(stderr, "unknown op: %s\n", arg); + return -1; + } + break; case 'i': - if (strcasecmp (arg, "fileio") == 0) { - state->need_iface_fileio = 1; - state->need_iface_xattr = 0; - } else if (strcasecmp (arg, "xattr") == 0) { - state->need_iface_fileio = 0; - state->need_iface_xattr = 1; - } else if (strcasecmp (arg, "both") == 0) { - state->need_iface_fileio = 1; - state->need_iface_xattr = 1; - } else { - fprintf (stderr, "unknown interface: %s\n", arg); - return -1; - } - break; - case 'm': - if (strcasecmp (arg, "posix") == 0) { - state->need_mode_posix = 1; - state->need_mode_libglusterfsclient = 0; - } else if (strcasecmp (arg, "libglusterfsclient") == 0) { - state->need_mode_posix = 0; - state->need_mode_libglusterfsclient = 1; - } else if (strcasecmp (arg, "both") == 0) { - state->need_mode_posix = 1; - state->need_mode_libglusterfsclient = 1; - } else { - fprintf (stderr, "unknown mode: %s\n", arg); - return -1; - } - break; - case 'b': - { - size_t block_size = atoi (arg); - if (!block_size) { - fprintf (stderr, "incorrect size: %s\n", arg); - return -1; - } - state->block_size = block_size; - } - break; + if (strcasecmp(arg, "fileio") == 0) { + state->need_iface_fileio = 1; + state->need_iface_xattr = 0; + } else if (strcasecmp(arg, "xattr") == 0) { + state->need_iface_fileio = 0; + state->need_iface_xattr = 1; + } else if (strcasecmp(arg, "both") == 0) { + state->need_iface_fileio = 1; + state->need_iface_xattr = 1; + } else { + fprintf(stderr, "unknown interface: %s\n", arg); + return -1; + } + break; + case 'b': { + size_t block_size = atoi(arg); + if (!block_size) { + fprintf(stderr, "incorrect size: %s\n", arg); + return -1; + } + state->block_size = block_size; + } break; case 's': - state->specfile = strdup (arg); - break; + state->specfile = strdup(arg); + break; case 'p': - fprintf (stderr, "using prefix: %s\n", arg); - strncpy (state->prefix, arg, 512); - break; - case 'c': - { - long count = atol (arg); - if (!count) { - fprintf (stderr, "incorrect count: %s\n", arg); - return -1; - } - state->count = count; - } - break; + fprintf(stderr, "using prefix: %s\n", arg); + strncpy(state->prefix, arg, 512); + break; + case 'c': { + long count = atol(arg); + if (!count) { + fprintf(stderr, "incorrect count: %s\n", arg); + return -1; + } + state->count = count; + } break; case ARGP_KEY_NO_ARGS: - break; + break; case ARGP_KEY_ARG: - break; - } + break; + } - return 0; + return 0; } int -do_mode_posix_iface_fileio_write (struct state *state) +do_mode_posix_iface_fileio_write(struct state *state) { - long int i; - int ret = -1; - char block[state->block_size]; - - for (i=0; i<state->count; i++) { - int fd = -1; - char filename[512]; - - sprintf (filename, "%s.%06ld", state->prefix, i); - - fd = open (filename, O_CREAT|O_WRONLY, 00600); - if (fd == -1) { - fprintf (stderr, "open(%s) => %s\n", filename, strerror (errno)); - break; - } - ret = write (fd, block, state->block_size); - if (ret != state->block_size) { - fprintf (stderr, "write (%s) => %d/%s\n", filename, ret, - strerror (errno)); - close (fd); - break; - } - close (fd); - state->io_size += ret; - } + long int i; + int ret = -1; + char block[state->block_size]; - return i; -} + for (i = 0; i < state->count; i++) { + int fd = -1; + char filename[512]; + sprintf(filename, "%s.%06ld", state->prefix, i); -int -do_mode_posix_iface_fileio_read (struct state *state) -{ - long int i; - int ret = -1; - char block[state->block_size]; - - for (i=0; i<state->count; i++) { - int fd = -1; - char filename[512]; - - sprintf (filename, "%s.%06ld", state->prefix, i); - - fd = open (filename, O_RDONLY); - if (fd == -1) { - fprintf (stderr, "open(%s) => %s\n", filename, strerror (errno)); - break; - } - ret = read (fd, block, state->block_size); - if (ret == -1) { - fprintf (stderr, "read(%s) => %d/%s\n", filename, ret, strerror (errno)); - close (fd); - break; - } - close (fd); - state->io_size += ret; + fd = open(filename, O_CREAT | O_WRONLY, 00600); + if (fd == -1) { + fprintf(stderr, "open(%s) => %s\n", filename, strerror(errno)); + break; } + ret = write(fd, block, state->block_size); + if (ret != state->block_size) { + fprintf(stderr, "write (%s) => %d/%s\n", filename, ret, + strerror(errno)); + close(fd); + break; + } + close(fd); + state->io_size += ret; + } - return i; + return i; } - int -do_mode_posix_iface_fileio (struct state *state) +do_mode_posix_iface_fileio_read(struct state *state) { - if (state->need_op_write) - MEASURE (do_mode_posix_iface_fileio_write, state); + long int i; + int ret = -1; + char block[state->block_size]; - if (state->need_op_read) - MEASURE (do_mode_posix_iface_fileio_read, state); - - return 0; -} + for (i = 0; i < state->count; i++) { + int fd = -1; + char filename[512]; + sprintf(filename, "%s.%06ld", state->prefix, i); -int -do_mode_posix_iface_xattr_write (struct state *state) -{ - long int i; - int ret = -1; - char block[state->block_size]; - char *dname = NULL, *dirc = NULL; - char *bname = NULL, *basec = NULL; - - dirc = strdup (state->prefix); - basec = strdup (state->prefix); - dname = dirname (dirc); - bname = basename (basec); - - for (i=0; i<state->count; i++) { - char key[512]; - - sprintf (key, "glusterfs.file.%s.%06ld", bname, i); - - ret = lsetxattr (dname, key, block, state->block_size, 0); - - if (ret != 0) { - fprintf (stderr, "lsetxattr (%s, %s, %p) => %s\n", - dname, key, block, strerror (errno)); - break; - } - state->io_size += state->block_size; + fd = open(filename, O_RDONLY); + if (fd == -1) { + fprintf(stderr, "open(%s) => %s\n", filename, strerror(errno)); + break; } - - free (dirc); - free (basec); - - return i; -} - - -int -do_mode_posix_iface_xattr_read (struct state *state) -{ - long int i; - int ret = -1; - char block[state->block_size]; - char *dname = NULL, *dirc = NULL; - char *bname = NULL, *basec = NULL; - - dirc = strdup (state->prefix); - basec = strdup (state->prefix); - dname = dirname (dirc); - bname = basename (basec); - - for (i=0; i<state->count; i++) { - char key[512]; - - sprintf (key, "glusterfs.file.%s.%06ld", bname, i); - - ret = lgetxattr (dname, key, block, state->block_size); - - if (ret < 0) { - fprintf (stderr, "lgetxattr (%s, %s, %p) => %s\n", - dname, key, block, strerror (errno)); - break; - } - state->io_size += ret; + ret = read(fd, block, state->block_size); + if (ret == -1) { + fprintf(stderr, "read(%s) => %d/%s\n", filename, ret, + strerror(errno)); + close(fd); + break; } + close(fd); + state->io_size += ret; + } - return i; + return i; } - int -do_mode_posix_iface_xattr (struct state *state) +do_mode_posix_iface_fileio(struct state *state) { - if (state->need_op_write) - MEASURE (do_mode_posix_iface_xattr_write, state); + if (state->need_op_write) + MEASURE(do_mode_posix_iface_fileio_write, state); - if (state->need_op_read) - MEASURE (do_mode_posix_iface_xattr_read, state); + if (state->need_op_read) + MEASURE(do_mode_posix_iface_fileio_read, state); - return 0; + return 0; } - int -do_mode_libglusterfsclient_iface_fileio_write (struct state *state) +do_mode_posix_iface_xattr_write(struct state *state) { - long int i; - int ret = -1; - char block[state->block_size]; - - for (i=0; i<state->count; i++) { - glusterfs_file_t fd = 0; - char filename[512]; - - sprintf (filename, "/%s.%06ld", state->prefix, i); - - fd = glusterfs_glh_open (state->libglusterfsclient_context, - filename, O_CREAT|O_WRONLY, 0); - - if (fd == 0) { - fprintf (stderr, "open(%s) => %s\n", filename, strerror (errno)); - break; - } - ret = glusterfs_write (fd, block, state->block_size); - if (ret == -1) { - fprintf (stderr, "glusterfs_write(%s) => %s\n", filename, strerror (errno)); - glusterfs_close (fd); - break; - } - glusterfs_close (fd); - state->io_size += ret; - } + long int i; + int ret = -1; + char block[state->block_size]; + char *dname = NULL, *dirc = NULL; + char *bname = NULL, *basec = NULL; - return i; -} - - -int -do_mode_libglusterfsclient_iface_fileio_read (struct state *state) -{ - long int i; - int ret = -1; - char block[state->block_size]; - - for (i=0; i<state->count; i++) { - glusterfs_file_t fd = 0; - char filename[512]; - - sprintf (filename, "/%s.%06ld", state->prefix, i); - - fd = glusterfs_glh_open (state->libglusterfsclient_context, - filename, O_RDONLY, 0); - - if (fd == 0) { - fprintf (stderr, "glusterfs_glh_open(%s) => %s\n", - filename, strerror (errno)); - break; - } - ret = glusterfs_read (fd, block, state->block_size); - if (ret == -1) { - fprintf (stderr, "glusterfs_read(%s) => %s\n", filename, - strerror (errno)); - glusterfs_close (fd); - break; - } - glusterfs_close (fd); - state->io_size += ret; - } - - return i; -} + dirc = strdup(state->prefix); + basec = strdup(state->prefix); + dname = dirname(dirc); + bname = basename(basec); + for (i = 0; i < state->count; i++) { + char key[512]; -int -do_mode_libglusterfsclient_iface_fileio (struct state *state) -{ - if (state->need_op_write) - MEASURE (do_mode_libglusterfsclient_iface_fileio_write, state); - - if (state->need_op_read) - MEASURE (do_mode_libglusterfsclient_iface_fileio_read, state); - - return 0; -} + sprintf(key, "glusterfs.file.%s.%06ld", bname, i); + ret = lsetxattr(dname, key, block, state->block_size, 0); -int -do_mode_libglusterfsclient_iface_xattr_write (struct state *state) -{ - long int i; - int ret = -1; - char block[state->block_size]; - char *dname = NULL, *dirc = NULL; - char *bname = NULL, *basec = NULL; - - asprintf (&dirc, "/%s", state->prefix); - asprintf (&basec, "/%s", state->prefix); - dname = dirname (dirc); - bname = basename (basec); - - for (i=0; i<state->count; i++) { - char key[512]; - - sprintf (key, "glusterfs.file.%s.%06ld", bname, i); - - ret = glusterfs_glh_setxattr (state->libglusterfsclient_context, - dname, key, block, - state->block_size, 0); - - if (ret < 0) { - fprintf (stderr, "glusterfs_glh_setxattr (%s, %s, %p) " - "=> %s\n", dname, key, block, - strerror (errno)); - break; - } - state->io_size += state->block_size; + if (ret != 0) { + fprintf(stderr, "lsetxattr (%s, %s, %p) => %s\n", dname, key, block, + strerror(errno)); + break; } + state->io_size += state->block_size; + } - return i; + free(dirc); + free(basec); + return i; } - int -do_mode_libglusterfsclient_iface_xattr_read (struct state *state) +do_mode_posix_iface_xattr_read(struct state *state) { - long int i; - int ret = -1; - char block[state->block_size]; - char *dname = NULL, *dirc = NULL; - char *bname = NULL, *basec = NULL; - - dirc = strdup (state->prefix); - basec = strdup (state->prefix); - dname = dirname (dirc); - bname = basename (basec); - - for (i=0; i<state->count; i++) { - char key[512]; - - sprintf (key, "glusterfs.file.%s.%06ld", bname, i); - - ret = glusterfs_glh_getxattr (state->libglusterfsclient_context, - dname, key, block, - state->block_size); - - if (ret < 0) { - fprintf (stderr, "glusterfs_glh_getxattr (%s, %s, %p) " - "=> %s\n", dname, key, block, - strerror (errno)); - break; - } - state->io_size += ret; - } + long int i; + int ret = -1; + char block[state->block_size]; + char *dname = NULL, *dirc = NULL; + char *bname = NULL, *basec = NULL; - return i; -} + dirc = strdup(state->prefix); + basec = strdup(state->prefix); + dname = dirname(dirc); + bname = basename(basec); + for (i = 0; i < state->count; i++) { + char key[512]; -int -do_mode_libglusterfsclient_iface_xattr (struct state *state) -{ - if (state->need_op_write) - MEASURE (do_mode_libglusterfsclient_iface_xattr_write, state); + sprintf(key, "glusterfs.file.%s.%06ld", bname, i); - if (state->need_op_read) - MEASURE (do_mode_libglusterfsclient_iface_xattr_read, state); + ret = lgetxattr(dname, key, block, state->block_size); - return 0; -} + if (ret < 0) { + fprintf(stderr, "lgetxattr (%s, %s, %p) => %s\n", dname, key, block, + strerror(errno)); + break; + } + state->io_size += ret; + } + return i; +} int -do_mode_posix (struct state *state) +do_mode_posix_iface_xattr(struct state *state) { - if (state->need_iface_fileio) - do_mode_posix_iface_fileio (state); + if (state->need_op_write) + MEASURE(do_mode_posix_iface_xattr_write, state); - if (state->need_iface_xattr) - do_mode_posix_iface_xattr (state); + if (state->need_op_read) + MEASURE(do_mode_posix_iface_xattr_read, state); - return 0; + return 0; } - int -do_mode_libglusterfsclient (struct state *state) +do_mode_posix(struct state *state) { - glusterfs_init_params_t ctx = { - .logfile = "/dev/stderr", - .loglevel = "error", - .lookup_timeout = 60, - .stat_timeout = 60, - }; - - ctx.specfile = state->specfile; - if (state->specfile) { - state->libglusterfsclient_context = glusterfs_init (&ctx, 1); - - if (!state->libglusterfsclient_context) { - fprintf (stdout, "Unable to initialize glusterfs " - "context, skipping libglusterfsclient mode\n"); - return -1; - } - } else { - fprintf (stdout, "glusterfs volume specification file not " - "provided, skipping libglusterfsclient mode\n"); - return -1; - } + if (state->need_iface_fileio) + do_mode_posix_iface_fileio(state); - if (state->need_iface_fileio) - do_mode_libglusterfsclient_iface_fileio (state); + if (state->need_iface_xattr) + do_mode_posix_iface_xattr(state); - if (state->need_iface_xattr) - do_mode_libglusterfsclient_iface_xattr (state); - - return 0; + return 0; } - int -do_actions (struct state *state) +do_actions(struct state *state) { - if (state->need_mode_libglusterfsclient) - do_mode_libglusterfsclient (state); + if (state->need_mode_posix) + do_mode_posix(state); - if (state->need_mode_posix) - do_mode_posix (state); - - return 0; + return 0; } static struct argp_option options[] = { - {"op", 'o', "OPERATIONS", 0, - "WRITE|READ|BOTH - defaults to BOTH"}, - {"iface", 'i', "INTERFACE", 0, - "FILEIO|XATTR|BOTH - defaults to FILEIO"}, - {"mode", 'm', "MODE", 0, - "POSIX|LIBGLUSTERFSCLIENT|BOTH - defaults to POSIX"}, - {"block", 'b', "BLOCKSIZE", 0, - "<NUM> - defaults to 4096"}, - {"specfile", 's', "SPECFILE", 0, - "absolute path to specfile"}, - {"prefix", 'p', "PREFIX", 0, - "filename prefix"}, - {"count", 'c', "COUNT", 0, - "number of files"}, - {0, 0, 0, 0, 0} -}; + {"op", 'o', "OPERATIONS", 0, "WRITE|READ|BOTH - defaults to BOTH"}, + {"iface", 'i', "INTERFACE", 0, "FILEIO|XATTR|BOTH - defaults to FILEIO"}, + {"block", 'b', "BLOCKSIZE", 0, "<NUM> - defaults to 4096"}, + {"specfile", 's', "SPECFILE", 0, "absolute path to specfile"}, + {"prefix", 'p', "PREFIX", 0, "filename prefix"}, + {"count", 'c', "COUNT", 0, "number of files"}, + {0, 0, 0, 0, 0}}; -static struct argp argp = { - options, - parse_opts, - "tool", - "tool to benchmark small file performance" -}; +static struct argp argp = {options, parse_opts, "tool", + "tool to benchmark small file performance"}; int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - struct state state = {0, }; + struct state state = { + 0, + }; - state.need_op_write = 1; - state.need_op_read = 1; + state.need_op_write = 1; + state.need_op_read = 1; - state.need_iface_fileio = 1; - state.need_iface_xattr = 0; + state.need_iface_fileio = 1; + state.need_iface_xattr = 0; - state.need_mode_posix = 1; - state.need_mode_libglusterfsclient = 0; + state.need_mode_posix = 1; - state.block_size = 4096; + state.block_size = 4096; - strcpy (state.prefix, "tmpfile"); - state.count = 1048576; + strcpy(state.prefix, "tmpfile"); + state.count = 1048576; - if (argp_parse (&argp, argc, argv, 0, 0, &state) != 0) { - fprintf (stderr, "argp_parse() failed\n"); - return 1; - } + if (argp_parse(&argp, argc, argv, 0, 0, &state) != 0) { + fprintf(stderr, "argp_parse() failed\n"); + return 1; + } - do_actions (&state); + do_actions(&state); - return 0; + return 0; } diff --git a/extras/benchmarking/rdd.c b/extras/benchmarking/rdd.c index dc7aafa76db..efc9d342a37 100644 --- a/extras/benchmarking/rdd.c +++ b/extras/benchmarking/rdd.c @@ -1,22 +1,12 @@ /* - Copyright (c) 2008 Gluster, Inc. <http://www.gluster.com> - This file is part of GlusterFS. - - GlusterFS is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - GlusterFS is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see - <http://www.gnu.org/licenses/>. -*/ + Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ #include <stdio.h> #include <sys/stat.h> #include <sys/types.h> @@ -30,463 +20,586 @@ #define TWO_POWER(power) (2UL << (power)) -#define RDD_INTEGER_VALUE ((TWO_POWER ((sizeof (int) * 8))) - 1) +#define RDD_INTEGER_VALUE ((TWO_POWER((sizeof(int) * 8))) - 1) #ifndef UNIX_PATH_MAX #define UNIX_PATH_MAX 108 #endif +#define UNIT_KB 1024ULL +#define UNIT_MB UNIT_KB * 1024ULL +#define UNIT_GB UNIT_MB * 1024ULL +#define UNIT_TB UNIT_GB * 1024ULL +#define UNIT_PB UNIT_TB * 1024ULL + +#define UNIT_KB_STRING "KB" +#define UNIT_MB_STRING "MB" +#define UNIT_GB_STRING "GB" +#define UNIT_TB_STRING "TB" +#define UNIT_PB_STRING "PB" + struct rdd_file { - char path[UNIX_PATH_MAX]; - struct stat st; - int fd; + char path[UNIX_PATH_MAX]; + struct stat st; + int fd; }; - + struct rdd_config { - long iters; - long max_ops_per_seq; - size_t max_bs; - size_t min_bs; - int thread_count; - pthread_t *threads; - pthread_barrier_t barrier; - pthread_mutex_t lock; - struct rdd_file in_file; - struct rdd_file out_file; + long iters; + long max_ops_per_seq; + size_t max_bs; + size_t min_bs; + int thread_count; + pthread_t *threads; + pthread_barrier_t barrier; + pthread_mutex_t lock; + struct rdd_file in_file; + struct rdd_file out_file; + ssize_t file_size; }; static struct rdd_config rdd_config; enum rdd_keys { - RDD_MIN_BS_KEY = 1, - RDD_MAX_BS_KEY, + RDD_MIN_BS_KEY = 1, + RDD_MAX_BS_KEY, }; - + static error_t -rdd_parse_opts (int key, char *arg, - struct argp_state *_state) +rdd_parse_opts(int key, char *arg, struct argp_state *_state) { - switch (key) { - case 'o': - { - int len = 0; - len = strlen (arg); - if (len > UNIX_PATH_MAX) { - fprintf (stderr, "output file name too long (%s)\n", - arg); - return -1; - } - - strncpy (rdd_config.out_file.path, arg, len); - } - break; - - case 'i': - { - int len = 0; - len = strlen (arg); - if (len > UNIX_PATH_MAX) { - fprintf (stderr, "input file name too long (%s)\n", - arg); - return -1; - } - - strncpy (rdd_config.in_file.path, arg, len); - } - break; - - case RDD_MIN_BS_KEY: - { - char *tmp = NULL; - long bs = 0; - bs = strtol (arg, &tmp, 10); - if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) { - fprintf (stderr, "invalid argument for minimum block" - "size (%s)\n", arg); - return -1; - } - - rdd_config.min_bs = bs; - } - break; - - case RDD_MAX_BS_KEY: - { - char *tmp = NULL; - long bs = 0; - bs = strtol (arg, &tmp, 10); - if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) { - fprintf (stderr, "invalid argument for maximum block" - "size (%s)\n", arg); - return -1; - } - - rdd_config.max_bs = bs; - } - break; - - case 'r': - { - char *tmp = NULL; - long iters = 0; - iters = strtol (arg, &tmp, 10); - if ((iters == LONG_MAX) || - (iters == LONG_MIN) || - (tmp && *tmp)) { - fprintf (stderr, "invalid argument for iterations" - "(%s)\n", arg); - return -1; - } - - rdd_config.iters = iters; - } - break; - - case 'm': - { - char *tmp = NULL; - long max_ops = 0; - max_ops = strtol (arg, &tmp, 10); - if ((max_ops == LONG_MAX) || - (max_ops == LONG_MIN) || - (tmp && *tmp)) { - fprintf (stderr, "invalid argument for max-ops" - "(%s)\n", arg); - return -1; - } - - rdd_config.max_ops_per_seq = max_ops; - } - break; - - case 't': - { - char *tmp = NULL; - long threads = 0; - threads = strtol (arg, &tmp, 10); - if ((threads == LONG_MAX) || - (threads == LONG_MIN) || - (tmp && *tmp)) { - fprintf (stderr, "invalid argument for thread count" - "(%s)\n", arg); - return -1; - } - - rdd_config.thread_count = threads; - } - break; - - case ARGP_KEY_NO_ARGS: - break; - case ARGP_KEY_ARG: - break; - case ARGP_KEY_END: - if (_state->argc == 1) { - argp_usage (_state); - } - - } - - return 0; + switch (key) { + case 'o': { + int len = 0; + len = strlen(arg); + if (len > UNIX_PATH_MAX) { + fprintf(stderr, "output file name too long (%s)\n", arg); + return -1; + } + + strncpy(rdd_config.out_file.path, arg, len); + } break; + + case 'i': { + int len = 0; + len = strlen(arg); + if (len > UNIX_PATH_MAX) { + fprintf(stderr, "input file name too long (%s)\n", arg); + return -1; + } + + strncpy(rdd_config.in_file.path, arg, len); + rdd_config.in_file.path[len] = '\0'; + } break; + + case 'f': { + char *tmp = NULL; + unsigned long long fs = 0; + if (string2bytesize(arg, &fs) == -1) { + fprintf(stderr, + "invalid argument for file size " + "(%s)\n", + arg); + return -1; + } + + rdd_config.file_size = fs; + } break; + + case RDD_MIN_BS_KEY: { + char *tmp = NULL; + long bs = 0; + bs = strtol(arg, &tmp, 10); + if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for minimum block" + "size (%s)\n", + arg); + return -1; + } + + rdd_config.min_bs = bs; + } break; + + case RDD_MAX_BS_KEY: { + char *tmp = NULL; + long bs = 0; + bs = strtol(arg, &tmp, 10); + if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for maximum block" + "size (%s)\n", + arg); + return -1; + } + + rdd_config.max_bs = bs; + } break; + + case 'r': { + char *tmp = NULL; + long iters = 0; + iters = strtol(arg, &tmp, 10); + if ((iters == LONG_MAX) || (iters == LONG_MIN) || (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for iterations" + "(%s)\n", + arg); + return -1; + } + + rdd_config.iters = iters; + } break; + + case 'm': { + char *tmp = NULL; + long max_ops = 0; + max_ops = strtol(arg, &tmp, 10); + if ((max_ops == LONG_MAX) || (max_ops == LONG_MIN) || + (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for max-ops" + "(%s)\n", + arg); + return -1; + } + + rdd_config.max_ops_per_seq = max_ops; + } break; + + case 't': { + char *tmp = NULL; + long threads = 0; + threads = strtol(arg, &tmp, 10); + if ((threads == LONG_MAX) || (threads == LONG_MIN) || + (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for thread count" + "(%s)\n", + arg); + return -1; + } + + rdd_config.thread_count = threads; + } break; + + case ARGP_KEY_NO_ARGS: + break; + case ARGP_KEY_ARG: + break; + case ARGP_KEY_END: + if (_state->argc == 1) { + argp_usage(_state); + } + } + + return 0; +} + +int +string2bytesize(const char *str, unsigned long long *n) +{ + unsigned long long value = 0ULL; + char *tail = NULL; + int old_errno = 0; + const char *s = NULL; + + if (str == NULL || n == NULL) { + errno = EINVAL; + return -1; + } + + for (s = str; *s != '\0'; s++) { + if (isspace(*s)) { + continue; + } + if (*s == '-') { + return -1; + } + break; + } + + old_errno = errno; + errno = 0; + value = strtoull(str, &tail, 10); + + if (errno == ERANGE || errno == EINVAL) { + return -1; + } + + if (errno == 0) { + errno = old_errno; + } + + if (tail[0] != '\0') { + if (strcasecmp(tail, UNIT_KB_STRING) == 0) { + value *= UNIT_KB; + } else if (strcasecmp(tail, UNIT_MB_STRING) == 0) { + value *= UNIT_MB; + } else if (strcasecmp(tail, UNIT_GB_STRING) == 0) { + value *= UNIT_GB; + } else if (strcasecmp(tail, UNIT_TB_STRING) == 0) { + value *= UNIT_TB; + } else if (strcasecmp(tail, UNIT_PB_STRING) == 0) { + value *= UNIT_PB; + } + + else { + return -1; + } + } + + *n = value; + + return 0; } static struct argp_option rdd_options[] = { - {"if", 'i', "INPUT_FILE", 0, "input-file"}, - {"of", 'o', "OUTPUT_FILE", 0, "output-file"}, - {"threads", 't', "COUNT", 0, "number of threads to spawn (defaults to 2)"}, - {"min-bs", RDD_MIN_BS_KEY, "MIN_BLOCK_SIZE", 0, - "Minimum block size in bytes (defaults to 1024)"}, - {"max-bs", RDD_MAX_BS_KEY, "MAX_BLOCK_SIZE", 0, - "Maximum block size in bytes (defaults to 4096)"}, - {"iters", 'r', "ITERS", 0, - "Number of read-write sequences (defaults to 1000000)"}, - {"max-ops", 'm', "MAXOPS", 0, - "maximum number of read-writes to be performed in a sequence (defaults to 1)"}, - {0, 0, 0, 0, 0} -}; + {"if", 'i', "INPUT_FILE", 0, "input-file"}, + {"of", 'o', "OUTPUT_FILE", 0, "output-file"}, + {"threads", 't', "COUNT", 0, "number of threads to spawn (defaults to 2)"}, + {"min-bs", RDD_MIN_BS_KEY, "MIN_BLOCK_SIZE", 0, + "Minimum block size in bytes (defaults to 1024)"}, + {"max-bs", RDD_MAX_BS_KEY, "MAX_BLOCK_SIZE", 0, + "Maximum block size in bytes (defaults to 4096)"}, + {"iters", 'r', "ITERS", 0, + "Number of read-write sequences (defaults to 1000000)"}, + {"max-ops", 'm', "MAXOPS", 0, + "maximum number of read-writes to be performed in a sequence (defaults to " + "1)"}, + {"file-size", 'f', "FILESIZE", 0, + "the size of the file which will be created and upon it I/O will be done" + " (defaults to 100MB"}, + {0, 0, 0, 0, 0}}; static struct argp argp = { - rdd_options, - rdd_parse_opts, - "", - "random dd - tool to do a sequence of random block-sized continuous" - "read writes starting at a random offset" -}; - + rdd_options, rdd_parse_opts, "", + "random dd - tool to do a sequence of random block-sized continuous" + "read writes starting at a random offset"}; static void -rdd_default_config (void) +rdd_default_config(void) { - rdd_config.thread_count = 2; - rdd_config.iters = 1000000; - rdd_config.max_bs = 4096; - rdd_config.min_bs = 1024; - rdd_config.in_file.fd = rdd_config.out_file.fd = -1; - rdd_config.max_ops_per_seq = 1; - - return; + char *tmp_path = "rdd.in"; + + rdd_config.thread_count = 2; + rdd_config.iters = 1000000; + rdd_config.max_bs = 4096; + rdd_config.min_bs = 1024; + rdd_config.in_file.fd = rdd_config.out_file.fd = -1; + rdd_config.max_ops_per_seq = 1; + strncpy(rdd_config.in_file.path, tmp_path, strlen(tmp_path)); + rdd_config.file_size = 104857600; + + return; } - static char -rdd_valid_config (void) +rdd_valid_config(void) { - char ret = 1; - int fd = -1; - - fd = open (rdd_config.in_file.path, O_RDONLY); - if (fd == -1) { - ret = 0; - goto out; - } - close (fd); - - if (rdd_config.min_bs > rdd_config.max_bs) { - ret = 0; - goto out; - } - - if (strlen (rdd_config.out_file.path) == 0) { - sprintf (rdd_config.out_file.path, "%s.rddout", - rdd_config.in_file.path); - } + char ret = 1; + int fd = -1; + + fd = open(rdd_config.in_file.path, O_RDONLY); + if (fd == -1 && (errno != ENOENT)) { + fprintf(stderr, "open: (%s)", strerror(errno)); + ret = 0; + goto out; + } + close(fd); + + if (rdd_config.min_bs > rdd_config.max_bs) { + fprintf(stderr, + "minimum blocksize %ld is greater than the " + "maximum blocksize %ld", + rdd_config.min_bs, rdd_config.max_bs); + ret = 0; + goto out; + } + + if (strlen(rdd_config.out_file.path) == 0) { + sprintf(rdd_config.out_file.path, "%s.rddout", rdd_config.in_file.path); + } out: - return ret; + return ret; } - static void * -rdd_read_write (void *arg) +rdd_read_write(void *arg) { - int i = 0, ret = 0; - size_t bs = 0; - off_t offset = 0; - long rand = 0; - long max_ops = 0; - char *buf = NULL; - - buf = calloc (1, rdd_config.max_bs); - if (!buf) { - fprintf (stderr, "calloc failed (%s)\n", strerror (errno)); - ret = -1; - goto out; - } - - for (i = 0; i < rdd_config.iters; i++) - { - pthread_mutex_lock (&rdd_config.lock); - { - int bytes = 0; - rand = random (); - - if (rdd_config.min_bs == rdd_config.max_bs) { - bs = rdd_config.max_bs; - } else { - bs = rdd_config.min_bs + - (rand % - (rdd_config.max_bs - - rdd_config.min_bs)); - } - - offset = rand % rdd_config.in_file.st.st_size; - max_ops = rand % rdd_config.max_ops_per_seq; - if (!max_ops) { - max_ops ++; - } - - ret = lseek (rdd_config.in_file.fd, offset, SEEK_SET); - if (ret != offset) { - fprintf (stderr, "lseek failed (%s)\n", - strerror (errno)); - ret = -1; - goto unlock; - } - - ret = lseek (rdd_config.out_file.fd, offset, SEEK_SET); - if (ret != offset) { - fprintf (stderr, "lseek failed (%s)\n", - strerror (errno)); - ret = -1; - goto unlock; - } - - while (max_ops--) - { - bytes = read (rdd_config.in_file.fd, buf, bs); - if (!bytes) { - break; - } - - if (bytes == -1) { - fprintf (stderr, "read failed (%s)\n", - strerror (errno)); - ret = -1; - goto unlock; - } - - if (write (rdd_config.out_file.fd, buf, bytes) - != bytes) { - fprintf (stderr, "write failed (%s)\n", - strerror (errno)); - ret = -1; - goto unlock; - } - } - } - unlock: - pthread_mutex_unlock (&rdd_config.lock); - if (ret == -1) { - goto out; - } - ret = 0; - } + int i = 0, ret = 0; + size_t bs = 0; + off_t offset = 0; + long rand = 0; + long max_ops = 0; + char *buf = NULL; + + buf = calloc(1, rdd_config.max_bs); + if (!buf) { + fprintf(stderr, "calloc failed (%s)\n", strerror(errno)); + ret = -1; + goto out; + } + + for (i = 0; i < rdd_config.iters; i++) { + pthread_mutex_lock(&rdd_config.lock); + { + int bytes = 0; + rand = random(); + + if (rdd_config.min_bs == rdd_config.max_bs) { + bs = rdd_config.max_bs; + } else { + bs = rdd_config.min_bs + + (rand % (rdd_config.max_bs - rdd_config.min_bs)); + } + + offset = rand % rdd_config.in_file.st.st_size; + max_ops = rand % rdd_config.max_ops_per_seq; + if (!max_ops) { + max_ops++; + } + + ret = lseek(rdd_config.in_file.fd, offset, SEEK_SET); + if (ret != offset) { + fprintf(stderr, "lseek failed (%s)\n", strerror(errno)); + ret = -1; + goto unlock; + } + + ret = lseek(rdd_config.out_file.fd, offset, SEEK_SET); + if (ret != offset) { + fprintf(stderr, "lseek failed (%s)\n", strerror(errno)); + ret = -1; + goto unlock; + } + + while (max_ops--) { + bytes = read(rdd_config.in_file.fd, buf, bs); + if (!bytes) { + break; + } + + if (bytes == -1) { + fprintf(stderr, "read failed (%s)\n", strerror(errno)); + ret = -1; + goto unlock; + } + + if (write(rdd_config.out_file.fd, buf, bytes) != bytes) { + fprintf(stderr, "write failed (%s)\n", strerror(errno)); + ret = -1; + goto unlock; + } + } + } + unlock: + pthread_mutex_unlock(&rdd_config.lock); + if (ret == -1) { + goto out; + } + ret = 0; + } out: - free (buf); - pthread_barrier_wait (&rdd_config.barrier); + free(buf); + pthread_barrier_wait(&rdd_config.barrier); - return NULL; + return NULL; } +static void +cleanup(void) +{ + close(rdd_config.in_file.fd); + close(rdd_config.out_file.fd); + rdd_config.in_file.fd = rdd_config.out_file.fd = -1; +} static int -rdd_spawn_threads (void) +check_and_create(void) { - int i = 0, ret = -1, fd = -1; - char buf[4096]; - - fd = open (rdd_config.in_file.path, O_RDONLY); - if (fd < 0) { - fprintf (stderr, "cannot open %s (%s)\n", - rdd_config.in_file.path, strerror (errno)); - ret = -1; - goto out; - } - ret = fstat (fd, &rdd_config.in_file.st); - if (ret != 0) { - close (fd); - fprintf (stderr, "cannot stat %s (%s)\n", - rdd_config.in_file.path, strerror (errno)); - ret = -1; - goto out; - } - rdd_config.in_file.fd = fd; - - fd = open (rdd_config.out_file.path, O_WRONLY | O_CREAT, - S_IRWXU | S_IROTH); - if (fd < 0) { - close (rdd_config.in_file.fd); - rdd_config.in_file.fd = -1; - fprintf (stderr, "cannot open %s (%s)\n", - rdd_config.out_file.path, strerror (errno)); - ret = -1; - goto out; - } - rdd_config.out_file.fd = fd; - - while ((ret = read (rdd_config.in_file.fd, buf, 4096)) > 0) { - if (write (rdd_config.out_file.fd, buf, ret) != ret) { - fprintf (stderr, "write failed (%s)\n", - strerror (errno)); - close (rdd_config.in_file.fd); - close (rdd_config.out_file.fd); - rdd_config.in_file.fd = rdd_config.out_file.fd = -1; - ret = -1; - goto out; - } - } - - rdd_config.threads = calloc (rdd_config.thread_count, - sizeof (pthread_t)); - if (rdd_config.threads == NULL) { - fprintf (stderr, "calloc() failed (%s)\n", strerror (errno)); - - ret = -1; - close (rdd_config.in_file.fd); - close (rdd_config.out_file.fd); - rdd_config.in_file.fd = rdd_config.out_file.fd = -1; - goto out; - } - - ret = pthread_barrier_init (&rdd_config.barrier, NULL, - rdd_config.thread_count + 1); - if (ret != 0) { - fprintf (stderr, "pthread_barrier_init() failed (%s)\n", - strerror (ret)); - - free (rdd_config.threads); - close (rdd_config.in_file.fd); - close (rdd_config.out_file.fd); - rdd_config.in_file.fd = rdd_config.out_file.fd = -1; - ret = -1; - goto out; - } - - ret = pthread_mutex_init (&rdd_config.lock, NULL); - if (ret != 0) { - fprintf (stderr, "pthread_mutex_init() failed (%s)\n", - strerror (ret)); - - free (rdd_config.threads); - pthread_barrier_destroy (&rdd_config.barrier); - close (rdd_config.in_file.fd); - close (rdd_config.out_file.fd); - rdd_config.in_file.fd = rdd_config.out_file.fd = -1; - ret = -1; - goto out; - } - - for (i = 0; i < rdd_config.thread_count; i++) - { - ret = pthread_create (&rdd_config.threads[i], NULL, - rdd_read_write, NULL); - if (ret != 0) { - fprintf (stderr, "pthread_create failed (%s)\n", - strerror (errno)); - exit (1); - } - } + int ret = -1; + char buf[4096] = { + 0, + }; + struct stat stbuf = { + 0, + }; + int fd[2] = { + -1, + }; + size_t total_size = -1; + + total_size = rdd_config.file_size; + + ret = stat(rdd_config.in_file.path, &stbuf); + if (ret == -1 && (errno != ENOENT)) + goto out; + + fd[1] = open(rdd_config.in_file.path, O_CREAT | O_WRONLY | O_TRUNC); + if (fd[1] == -1) + goto out; + + fd[0] = open("/dev/urandom", O_RDONLY); + if (fd[0] == -1) + goto out; + + while (total_size > 0) { + if (total_size >= 4096) { + ret = read(fd[0], buf, 4096); + if (ret == -1) + goto out; + ret = write(fd[1], buf, 4096); + if (ret == -1) + goto out; + total_size = total_size - 4096; + } else { + ret = read(fd[0], buf, total_size); + if (ret == -1) + goto out; + ret = write(fd[1], buf, total_size); + if (ret == -1) + goto out; + total_size = total_size - total_size; + } + } + + ret = 0; out: - return ret; + if (fd[0] > 0) + close(fd[0]); + if (fd[1] > 0) + close(fd[1]); + return ret; } +static int +rdd_spawn_threads(void) +{ + int i = 0, ret = -1, fd = -1; + char buf[4096]; + + ret = check_and_create(); + if (ret == -1) + goto out; + + fd = open(rdd_config.in_file.path, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "cannot open %s (%s)\n", rdd_config.in_file.path, + strerror(errno)); + ret = -1; + goto out; + } + ret = fstat(fd, &rdd_config.in_file.st); + if (ret != 0) { + close(fd); + fprintf(stderr, "cannot stat %s (%s)\n", rdd_config.in_file.path, + strerror(errno)); + ret = -1; + goto out; + } + rdd_config.in_file.fd = fd; + + fd = open(rdd_config.out_file.path, O_WRONLY | O_CREAT | O_TRUNC, + S_IRWXU | S_IROTH); + if (fd < 0) { + close(rdd_config.in_file.fd); + rdd_config.in_file.fd = -1; + fprintf(stderr, "cannot open %s (%s)\n", rdd_config.out_file.path, + strerror(errno)); + ret = -1; + goto out; + } + rdd_config.out_file.fd = fd; + + while ((ret = read(rdd_config.in_file.fd, buf, 4096)) > 0) { + if (write(rdd_config.out_file.fd, buf, ret) != ret) { + fprintf(stderr, "write failed (%s)\n", strerror(errno)); + cleanup(); + ret = -1; + goto out; + } + } + + rdd_config.threads = calloc(rdd_config.thread_count, sizeof(pthread_t)); + if (rdd_config.threads == NULL) { + fprintf(stderr, "calloc() failed (%s)\n", strerror(errno)); + + ret = -1; + cleanup(); + goto out; + } + + ret = pthread_barrier_init(&rdd_config.barrier, NULL, + rdd_config.thread_count + 1); + if (ret != 0) { + fprintf(stderr, "pthread_barrier_init() failed (%s)\n", strerror(ret)); + + free(rdd_config.threads); + cleanup(); + ret = -1; + goto out; + } + + ret = pthread_mutex_init(&rdd_config.lock, NULL); + if (ret != 0) { + fprintf(stderr, "pthread_mutex_init() failed (%s)\n", strerror(ret)); + + free(rdd_config.threads); + pthread_barrier_destroy(&rdd_config.barrier); + cleanup(); + ret = -1; + goto out; + } + + for (i = 0; i < rdd_config.thread_count; i++) { + ret = pthread_create(&rdd_config.threads[i], NULL, rdd_read_write, + NULL); + if (ret != 0) { + fprintf(stderr, "pthread_create failed (%s)\n", strerror(errno)); + exit(1); + } + } + +out: + return ret; +} static void -rdd_wait_for_completion (void) +rdd_wait_for_completion(void) { - pthread_barrier_wait (&rdd_config.barrier); + pthread_barrier_wait(&rdd_config.barrier); } - -int -main (int argc, char *argv[]) +int +main(int argc, char *argv[]) { - int ret = -1; + int ret = -1; - rdd_default_config (); + rdd_default_config(); - ret = argp_parse (&argp, argc, argv, 0, 0, NULL); - if (ret != 0) { - ret = -1; - fprintf (stderr, "%s: argp_parse() failed\n", argv[0]); - goto err; - } + ret = argp_parse(&argp, argc, argv, 0, 0, NULL); + if (ret != 0) { + ret = -1; + fprintf(stderr, "%s: argp_parse() failed\n", argv[0]); + goto err; + } - if (!rdd_valid_config ()) { - ret = -1; - fprintf (stderr, "%s: configuration validation failed\n", - argv[0]); - goto err; - } + if (!rdd_valid_config()) { + ret = -1; + fprintf(stderr, "%s: configuration validation failed\n", argv[0]); + goto err; + } - ret = rdd_spawn_threads (); - if (ret != 0) { - fprintf (stderr, "%s: spawning threads failed\n", argv[0]); - goto err; - } + ret = rdd_spawn_threads(); + if (ret != 0) { + fprintf(stderr, "%s: spawning threads failed\n", argv[0]); + goto err; + } - rdd_wait_for_completion (); + rdd_wait_for_completion(); err: - return ret; -} + return ret; +} diff --git a/extras/check_goto.pl b/extras/check_goto.pl new file mode 100755 index 00000000000..fa71bfc6683 --- /dev/null +++ b/extras/check_goto.pl @@ -0,0 +1,45 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +my @ignore_labels = qw (TODO retry fetch_data again try_again sp_state_read_proghdr redo disabled_loop fd_alloc_try_again); +my @ignore_files = qw (y.tab.c lex.c); +my @c_files; +my $line; +my @labels; +my $in_comments; + +{ + local $" = "|"; + my $cmd = "find . -type f -name '*.c' | grep -vE '(@ignore_files)'"; + @c_files = `$cmd`; +} + +foreach my $file (@c_files) { + chomp ($file); + open FD, $file or die ("Failed to read file $file: $!"); + @labels = (); + $in_comments = 0; + while ($line = <FD>) { + chomp ($line); + + next if $line =~ /^\s*(#|\/\/)/; + $in_comments = 1 if ($line =~ /\/\*/); + $in_comments = 0 if ($line =~ /\*\//); + + next if $in_comments; + if ($line =~ /^\s*(([a-zA-Z]|_)\w*)\s*:/) { + push (@labels, $1) unless grep (/$1/, @ignore_labels); + } + @labels = () if $line =~ /^}/; + + next unless @labels; + if ($line =~ /^\s*goto\s*(\w+)/) { + print "$file:$.: $line\n" if grep /^$1$/, @labels; + } + } + + close FD; +} + diff --git a/extras/clang-checker.sh b/extras/clang-checker.sh new file mode 100755 index 00000000000..4909d3adfcd --- /dev/null +++ b/extras/clang-checker.sh @@ -0,0 +1,301 @@ +#!/usr/bin/env bash +#******************************************************************************* +# * +# Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> * +# This file is part of GlusterFS. * +# * +# This file is licensed to you under your choice of the GNU Lesser * +# General Public License, version 3 or any later version (LGPLv3 or * +# later), or the GNU General Public License, version 2 (GPLv2), in all * +# cases as published by the Free Software Foundation. * +#------------------------------------------------------------------------------* +# * +# clang-checker.sh: This script runs clang static analyzer using 'scan-build' * +# a perl wrapper. After you commit your patch i.e. right * +# before executing rfc.sh in order to push the patch to * +# repository, it is recommended that you execute * +# clang-checker.sh to perform static analysis inorder to * +# check if there are any possible bugs in the code. * +# * +# This script performs the static analysis with and * +# without HEAD commit, it runs the analyzer only in the * +# directory where changes have been made and finally diff's * +# the number of bugs using both cases (i.e. with and * +# without your commit) and gives a summary, which explain's * +# about the eligibility of your patch. * +# * +# Usage: $ cd $PATH_TO_GLUSTERFS * +# $ extras/clang-checker.sh (or) $ make clang-check * +# * +# Author: Prasanna Kumar Kalever <prasanna.kalever@redhat.com> * +# * +#******************************************************************************* + +REPORTS_DIR=$(pwd) +BASELINE_DIR=${REPORTS_DIR}/baseline +BRESULTS_DIR=${BASELINE_DIR}/results +BBACKUP_DIR=${BASELINE_DIR}/backup +TARGET_DIR=${REPORTS_DIR}/target +TRESULTS_DIR=${TARGET_DIR}/results +TBACKUP_DIR=${TARGET_DIR}/backup + +declare -A DICT_B +declare -A DICT_T +declare -A ARR +declare -A FILES + +function identify_changes () { + MODIFIED_DATA=$(git show --name-status --oneline | tail -n +2) + FLAG=0 + for i in ${MODIFIED_DATA}; do + if [ $FLAG -eq 1 ]; then + ARR+="$(dirname $i) "; + FLAG=0; + fi + if [ $i = 'M' ] || [ $i = 'A' ]; then + FLAG=1; + fi + done + + MODIFIED_DIR=$(echo "${ARR[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ') + for i in $MODIFIED_DIR; do + # run only in directories which has Makefile + if [ $(find ./$i -iname "makefile*" | wc -c) -gt 0 ]; then + # skip 'doc' and '.'(top) directory + if [ "xx$i" != "xxdoc" ] && [ "xx$i" != "xx." ]; then + FILES+="$i " + fi + fi + done + if [ -z $FILES ]; then + echo "Probably no changes made to 'c' files" + exit; + fi +} + +function check_prerequisites () { + if ! type "clang" 2> /dev/null; then + echo -e "\ntry after installing clang and scan-build..." + echo "useful info at http://clang-analyzer.llvm.org/installation.html\n" + echo -e "hint: 'dnf -y install clang-analyzer.noarch'\n" + exit 1; + elif ! type "scan-build" 2> /dev/null; then + echo -e "\ntry after installing scan-build..." + echo "useful info at http://clang-analyzer.llvm.org/installation.html" + echo -e "hint: 'dnf -y install clang-analyzer.noarch'\n" + exit 1; + fi +} + +function force_terminate () { + echo -e "\nreceived a signal to force terminate ..\n" + git am --abort 2> /dev/null + git am ${PATCH_NAME} + rm -f ${REPORTS_DIR}/${PATCH_NAME} + exit 1; +} + +function run_scanbuild () { + local CLANG=$(which clang) + local SCAN_BUILD=$(which scan-build) + local ORIG_COMMIT=$(git rev-parse --verify HEAD^) + PATCH_NAME=$(git format-patch $ORIG_COMMIT) + + echo -e "\n| Performing clang analysis on:" \ + "$(git log --pretty=format:"%h - '%s' by %an" -1) ... |\n" + echo -e "Changes are identified in '${FILES[@]}' directorie[s]\n" + + if [ -d "${BRESULTS_DIR}" ]; then + mkdir -p ${BBACKUP_DIR} ${TBACKUP_DIR} + mv ${BRESULTS_DIR} \ + ${BBACKUP_DIR}/results_$(ls -l ${BBACKUP_DIR} | wc -l) + mv ${TRESULTS_DIR} \ + ${TBACKUP_DIR}/results_$(ls -l ${TBACKUP_DIR} | wc -l) + fi + for DIR in ${FILES[@]}; do + mkdir -p ${BRESULTS_DIR}/$(echo ${DIR} | sed 's/\//_/g') + mkdir -p ${TRESULTS_DIR}/$(echo ${DIR} | sed 's/\//_/g') + done + # get nproc info + case $(uname -s) in + 'Linux') + local NPROC=$(getconf _NPROCESSORS_ONLN) + ;; + 'NetBSD') + local NPROC=$(getconf NPROCESSORS_ONLN) + ;; + esac + + trap force_terminate INT TERM QUIT EXIT + + git reset --hard HEAD^ + + # build complete source code for sake of dependencies + echo -e "\n# make -j${NPROC} ..." + make -j${NPROC} 1>/dev/null + + for DIR in ${FILES[@]}; do + if [ $(find ./$i -iname "makefile*" | wc -c) -gt 0 ]; then + make clean -C ${DIR} 1>/dev/null + echo -e "\n| Analyzing ${DIR} without commit ... |\n" + # run only in directory where changes are made + ${SCAN_BUILD} -o ${BRESULTS_DIR}/$(echo ${DIR} | sed 's/\//_/g') \ + --use-analyzer=${CLANG} make -j${NPROC} -C ${DIR} + fi + done + + echo -e "\n| Analyzing without commit complete ... |\n" + + git am ${PATCH_NAME} + trap - INT TERM QUIT EXIT + + # In case commit has changes to configure stuff ? + echo -e "\n# make clean ..." + make clean 1>/dev/null + echo -e "\n# ./autogen.sh && ./configure --with-previous-options ..." + ${REPORTS_DIR}/autogen.sh 2>/dev/null + ${REPORTS_DIR}/configure --with-previous-options 1>/dev/null + echo -e "\n# make -j${NPROC} ..." + make -j${NPROC} 1>/dev/null + + for DIR in ${FILES[@]}; do + if [ $(find ./$i -iname "makefile*" | wc -c) -gt 0 ]; then + make clean -C ${DIR} 1>/dev/null + echo -e "\n| Analyzing ${DIR} with commit ... |\n" + # run only in directory where changes are made + ${SCAN_BUILD} -o ${TRESULTS_DIR}/$(echo ${DIR} | sed 's/\//_/g') \ + --use-analyzer=${CLANG} make -j${NPROC} -C ${DIR} + fi + done + + echo -e "\n| Analyzing with commit complete ... |\n" + + rm -f ${REPORTS_DIR}/${PATCH_NAME} +} + +function count_for_baseline () { + for DIR in ${FILES[@]}; do + HTMLS_DIR=${BRESULTS_DIR}/$(echo ${DIR} | + sed 's/\//_/g')/$(ls ${BRESULTS_DIR}/$(echo ${DIR} | + sed 's/\//_/g')/); + + local NAMES_OF_BUGS_B=$(grep -n "SUMM_DESC" ${HTMLS_DIR}/index.html | + cut -d"<" -f3 | cut -d">" -f2 | + sed 's/[^a-zA-Z0]/_/g' | tr '\n' ' ') + local NO_OF_BUGS_B=$(grep -n "SUMM_DESC" ${HTMLS_DIR}/index.html | + cut -d"<" -f5 | cut -d">" -f2 | tr '\n' ' ') + local count_B=0; + + read -a BUG_NAME_B <<<$NAMES_OF_BUGS_B + read -a BUG_COUNT_B <<<$NO_OF_BUGS_B + for i in ${BUG_NAME_B[@]}; + do + if [ ! -z ${DICT_B[$i]} ]; then + DICT_B[$i]=$(expr ${BUG_COUNT_B[count_B]} + ${DICT_B[$i]}); + else + DICT_B+=([$i]=${BUG_COUNT_B[count_B]}); + fi + count_B=$(expr $count_B + 1) + done + done + + echo -e "\nBASELINE BUGS LIST (before applying patch):" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + for key_B in ${!DICT_B[@]}; do + echo "${key_B} --> ${DICT_B[${key_B}]}" | sed 's/_/ /g' | tr -s ' ' + done +} + +function count_for_target () { + for DIR in ${FILES[@]}; do + HTMLS_DIR=${TRESULTS_DIR}/$(echo ${DIR} | + sed 's/\//_/g')/$(ls ${TRESULTS_DIR}/$(echo ${DIR} | + sed 's/\//_/g')/); + + local NAME_OF_BUGS_T=$(grep -n "SUMM_DESC" ${HTMLS_DIR}/index.html | + cut -d"<" -f3 | cut -d">" -f2 | + sed 's/[^a-zA-Z0]/_/g'| tr '\n' ' ') + local NO_OF_BUGS_T=$(grep -n "SUMM_DESC" ${HTMLS_DIR}/index.html | + cut -d"<" -f5 | cut -d">" -f2 | tr '\n' ' ') + local count_T=0; + + read -a BUG_NAME_T <<<$NAME_OF_BUGS_T + read -a BUG_COUNT_T <<<$NO_OF_BUGS_T + + for i in ${BUG_NAME_T[@]}; + do + if [ ! -z ${DICT_T[$i]} ]; then + DICT_T[$i]=$(expr ${BUG_COUNT_T[count_T]} + ${DICT_T[$i]}); + else + DICT_T+=([$i]=${BUG_COUNT_T[count_T]}); + fi + count_T=$(expr $count_T + 1) + done + done + + echo -e "\nTARGET BUGS LIST (after applying patch):" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + for key_T in ${!DICT_T[@]}; do + echo "${key_T} --> ${DICT_T[${key_T}]}" | sed 's/_/ /g' | tr -s ' ' + done +} + +function array_contains () { + local SEEKING=$1; shift + local IN=1 + for ELEMENT; do + if [[ $ELEMENT == $SEEKING ]]; then + IN=0 + break + fi + done + return $IN +} + +function main () { + echo -e "\n================ Clang analyzer in progress ================\n" + check_prerequisites + identify_changes + run_scanbuild + clear + count_for_baseline + count_for_target + echo -e "\nSUMMARY OF CLANG-ANALYZER:" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" + + FLAG=0 + for BUG in ${!DICT_T[@]}; do + array_contains $BUG "${!DICT_B[@]}" + if [ $? -eq 1 ]; then + echo "New ${DICT_T[${BUG}]} Bug[s] introduced: $(echo $BUG | + sed 's/_/ /g' | + tr -s ' ')" + FLAG=1 + else + if [ ${BUG} != "All_Bugs" ]; then + if [ ${DICT_B[${BUG}]} -lt \ + ${DICT_T[${BUG}]} ]; then + echo "Extra $(expr ${DICT_T[${BUG}]} - \ + ${DICT_B[${BUG}]}) Bug[s] Introduced in: $(echo $BUG | + sed 's/_/ /g' | tr -s ' ')" + FLAG=1 + fi + fi + fi + done + + echo + if [ $FLAG -eq 0 ]; then + echo -e "Patch Value given by Clang analyzer '+1'\n" + else + echo -e "Patch Value given by Clang analyzer '-1'\n" + fi + echo -e "\nExplore complete results at:" + find ${BRESULTS_DIR}/ -iname "index.html" + find ${TRESULTS_DIR}/ -iname "index.html" + echo -e "\n================= Done with Clang Analysis =================\n" + + exit ${FLAG} +} + +main diff --git a/extras/clear_xattrs.sh b/extras/clear_xattrs.sh new file mode 100755 index 00000000000..dd04731e8bd --- /dev/null +++ b/extras/clear_xattrs.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Clear the trusted.gfid xattr in the brick tree + +# This script must be run only on a stopped brick/volume +# Stop the volume to make sure no rebalance/replace-brick +# operations are on-going + +# Not much error checking +remove_xattrs () +{ + find "$1" -exec setfattr -h -x "trusted.gfid" '{}' \; > /dev/null 2>&1; + find "$1" -exec setfattr -h -x "trusted.glusterfs.volume-id" '{}' \; > /dev/null 2>&1; +} + +main () +{ + if [ -z "$1" ]; then + echo "Usage: $0 <brick_path(s)>"; + exit 1; + fi + + export PATH; + which getfattr > /dev/null 2>&1; + if [ $? -ne 0 ]; then + echo "attr package missing"; + exit 2; + fi + + which setfattr > /dev/null 2>&1; + if [ $? -ne 0 ]; then + echo "attr package missing"; + exit 2; + fi + + for brick in "$@"; + do + stat "$brick" > /dev/null 2>&1; + if [ $? -ne 0 ]; then + echo "brick: $brick does not exist"; + exit 3; + fi + if [ ! -d "$brick" ]; then + echo "$brick: not a directory"; + exit 4; + fi + echo "xattr clean-up in progress: $brick"; + remove_xattrs "$brick"; + echo "$brick ready to be used as a glusterfs brick"; + done; +} + +main "$@";
\ No newline at end of file diff --git a/extras/cliutils/Makefile.am b/extras/cliutils/Makefile.am new file mode 100644 index 00000000000..7039703e275 --- /dev/null +++ b/extras/cliutils/Makefile.am @@ -0,0 +1,4 @@ +EXTRA_DIST= cliutils.py __init__.py + +cliutilsdir = @BUILD_PYTHON_SITE_PACKAGES@/gluster/cliutils +cliutils_PYTHON = cliutils.py __init__.py diff --git a/extras/cliutils/README.md b/extras/cliutils/README.md new file mode 100644 index 00000000000..309beb1ca25 --- /dev/null +++ b/extras/cliutils/README.md @@ -0,0 +1,233 @@ +# CLI utility for creating Cluster aware CLI tools for Gluster +cliutils is a Python library which provides wrapper around `gluster system:: +execute` command to extend the functionalities of Gluster. + +Example use cases: +- Start a service in all peer nodes of Cluster +- Collect the status of a service from all peer nodes +- Collect the config values from each peer nodes and display latest + config based on version. +- Copy a file present in GLUSTERD_WORKDIR from one peer node to all + other peer nodes.(Geo-replication create push-pem is using this to + distribute the SSH public keys from all master nodes to all slave + nodes) +- Generate pem keys in all peer nodes and collect all the public keys + to one place(Geo-replication gsec_create is doing this) +- Provide Config sync CLIs for new features like `gluster-eventsapi`, + `gluster-restapi`, `gluster-mountbroker` etc. + +## Introduction + +If a executable file present in `$GLUSTER_LIBEXEC` directory in all +peer nodes(Filename startswith `peer_`) then it can be executed by +running `gluster system:: execute` command from any one peer node. + +- This command will not copy any executables to peer nodes, Script + should exist in all peer nodes to use this infrastructure. Raises + error in case script not exists in any one of the peer node. +- Filename should start with `peer_` and should exist in + `$GLUSTER_LIBEXEC` directory. +- This command can not be called from outside the cluster. + +To understand the functionality, create a executable file `peer_hello` +under $GLUSTER_LIBEXEC directory and copy to all peer nodes. + + #!/usr/bin/env bash + echo "Hello from $(gluster system:: uuid get)" + +Now run the following command from any one gluster node, + + gluster system:: execute hello + +**Note:** Gluster will not copy the executable script to all nodes, + copy `peer_hello` script to all peer nodes to use `gluster system:: + execute` infrastructure. + +It will run `peer_hello` executable in all peer nodes and shows the +output from each node(Below example shows output from my two nodes +cluster) + + Hello from UUID: e7a3c5c8-e7ad-47ad-aa9c-c13907c4da84 + Hello from UUID: c680fc0a-01f9-4c93-a062-df91cc02e40f + +## cliutils +A Python wrapper around `gluster system:: execute` command is created +to address the following issues + +- If a node is down in the cluster, `system:: execute` just skips it + and runs only in up nodes. +- `system:: execute` commands are not user friendly +- It captures only stdout, so handling errors is tricky. + +**Advantages of cliutils:** + +- Single executable file will act as node component as well as User CLI. +- `execute_in_peers` utility function will merge the `gluster system:: + execute` output with `gluster peer status` to identify offline nodes. +- Easy CLI Arguments handling. +- If node component returns non zero return value then, `gluster + system:: execute` will fail to aggregate the output from other + nodes. `node_output_ok` or `node_output_notok` utility functions + returns zero both in case of success or error, but returns json + with ok: true or ok:false respectively. +- Easy to iterate on the node outputs. +- Better error handling - Geo-rep CLIs `gluster system:: execute + mountbroker`, `gluster system:: execute gsec_create` and `gluster + system:: add_secret_pub` are suffering from error handling. These + tools are not notifying user if any failures during execute or if a node + is down during execute. + +### Hello World +Create a file in `$LIBEXEC/glusterfs/peer_message.py` with following +content. + + #!/usr/bin/python3 + from gluster.cliutils import Cmd, runcli, execute_in_peers, node_output_ok + + class NodeHello(Cmd): + name = "node-hello" + + def run(self, args): + node_output_ok("Hello") + + class Hello(Cmd): + name = "hello" + + def run(self, args): + out = execute_in_peers("node-hello") + for row in out: + print ("{0} from {1}".format(row.output, row.hostname)) + + if __name__ == "__main__": + runcli() + +When we run `python peer_message.py`, it will have two subcommands, +"node-hello" and "hello". This file should be copied to +`$LIBEXEC/glusterfs` directory in all peer nodes. User will call +subcommand "hello" from any one peer node, which internally call +`gluster system:: execute message.py node-hello`(This runs in all peer +nodes and collect the outputs) + +For node component do not print the output directly, use +`node_output_ok` or `node_output_notok` functions. `node_output_ok` +additionally collects the node UUID and prints in JSON +format. `execute_in_peers` function will collect this output and +merges with `peers list` so that we don't miss the node information if +that node is offline. + +If you observed already, function `args` is optional, if you don't +have arguments then no need to create a function. When we run the +file, we will have two subcommands. For example, + + python peer_message.py hello + python peer_message.py node-hello + +First subcommand calls second subcommand in all peer nodes. Basically +`execute_in_peers(NAME, ARGS)` will be converted into + + CMD_NAME = FILENAME without "peers_" + gluster system:: execute <CMD_NAME> <SUBCOMMAND> <ARGS> + +In our example, + + filename = "peer_message.py" + cmd_name = "message.py" + gluster system:: execute ${cmd_name} node-hello + +Now create symlink in `/usr/bin` or `/usr/sbin` directory depending on +the usecase.(Optional step for usability) + + ln -s /usr/libexec/glusterfs/peer_message.py /usr/bin/gluster-message + +Now users can use `gluster-message` instead of calling +`/usr/libexec/glusterfs/peer_message.py` + + gluster-message hello + +### Showing CLI output as Table + +Following example uses prettytable library, which can be installed +using `pip install prettytable` or `dnf install python-prettytable` + + #!/usr/bin/python3 + from prettytable import PrettyTable + from gluster.cliutils import Cmd, runcli, execute_in_peers, node_output_ok + + class NodeHello(Cmd): + name = "node-hello" + + def run(self, args): + node_output_ok("Hello") + + class Hello(Cmd): + name = "hello" + + def run(self, args): + out = execute_in_peers("node-hello") + # Initialize the CLI table + table = PrettyTable(["ID", "NODE", "NODE STATUS", "MESSAGE"]) + table.align["NODE STATUS"] = "r" + for row in out: + table.add_row([row.nodeid, + row.hostname, + "UP" if row.node_up else "DOWN", + row.output if row.ok else row.error]) + + print table + + if __name__ == "__main__": + runcli() + + +Example output, + + +--------------------------------------+-----------+-------------+---------+ + | ID | NODE | NODE STATUS | MESSAGE | + +--------------------------------------+-----------+-------------+---------+ + | e7a3c5c8-e7ad-47ad-aa9c-c13907c4da84 | localhost | UP | Hello | + | bb57a4c4-86eb-4af5-865d-932148c2759b | vm2 | UP | Hello | + | f69b918f-1ffa-4fe5-b554-ee10f051294e | vm3 | DOWN | N/A | + +--------------------------------------+-----------+-------------+---------+ + +## How to package in Gluster +If the project is created in `$GLUSTER_SRC/tools/message` + +Add "message" to SUBDIRS list in `$GLUSTER_SRC/tools/Makefile.am` + +and then create a `Makefile.am` in `$GLUSTER_SRC/tools/message` +directory with following content. + + EXTRA_DIST = peer_message.py + + peertoolsdir = $(libexecdir)/glusterfs/ + peertools_SCRIPTS = peer_message.py + + install-exec-hook: + $(mkdir_p) $(DESTDIR)$(bindir) + rm -f $(DESTDIR)$(bindir)/gluster-message + ln -s $(libexecdir)/glusterfs/peer_message.py \ + $(DESTDIR)$(bindir)/gluster-message + + uninstall-hook: + rm -f $(DESTDIR)$(bindir)/gluster-message + +Thats all. Add following files in `glusterfs.spec.in` if packaging is +required.(Under `%files` section) + + %{_libexecdir}/glusterfs/peer_message.py* + %{_bindir}/gluster-message + +## Who is using cliutils +- gluster-mountbroker http://review.gluster.org/14544 +- gluster-eventsapi http://review.gluster.org/14248 +- gluster-georep-sshkey http://review.gluster.org/14732 +- gluster-restapi https://github.com/gluster/restapi + +## Limitations/TODOs +- Not yet possible to create CLI without any subcommand, For example + `gluster-message` without any arguments +- Hiding node subcommands in `--help`(`gluster-message --help` will + show all subcommands including node subcommands) +- Only positional arguments supported for node arguments, Optional + arguments can be used for other commands. +- API documentation diff --git a/extras/cliutils/__init__.py b/extras/cliutils/__init__.py new file mode 100644 index 00000000000..8765cc85099 --- /dev/null +++ b/extras/cliutils/__init__.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Reexporting the utility funcs and classes +from .cliutils import (runcli, + sync_file_to_peers, + execute_in_peers, + execute, + node_output_ok, + node_output_notok, + output_error, + oknotok, + yesno, + get_node_uuid, + Cmd, + GlusterCmdException, + set_common_args_func) + + +# This will be useful when `from cliutils import *` +__all__ = ["runcli", + "sync_file_to_peers", + "execute_in_peers", + "execute", + "node_output_ok", + "node_output_notok", + "output_error", + "oknotok", + "yesno", + "get_node_uuid", + "Cmd", + "GlusterCmdException", + "set_common_args_func"] diff --git a/extras/cliutils/cliutils.py b/extras/cliutils/cliutils.py new file mode 100644 index 00000000000..55fbaf56704 --- /dev/null +++ b/extras/cliutils/cliutils.py @@ -0,0 +1,237 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +from argparse import ArgumentParser, RawDescriptionHelpFormatter +import inspect +import subprocess +import os +import xml.etree.cElementTree as etree +import json +import sys + +MY_UUID = None +parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter, + description=__doc__) +subparsers = parser.add_subparsers(dest="mode") + +subcommands = {} +cache_data = {} +ParseError = etree.ParseError if hasattr(etree, 'ParseError') else SyntaxError +_common_args_func = lambda p: True + + +class GlusterCmdException(Exception): + def __init__(self, message): + self.message = message + try: + # Python 3 + super().__init__(message) + except TypeError: + # Python 2 + super(GlusterCmdException, self).__init__(message) + + +def get_node_uuid(): + # Caches the Node UUID in global variable, + # Executes gluster system:: uuid get command only if + # calling this function for first time + global MY_UUID + if MY_UUID is not None: + return MY_UUID + + cmd = ["gluster", "system::", "uuid", "get", "--xml"] + rc, out, err = execute(cmd) + + if rc != 0: + return None + + tree = etree.fromstring(out) + uuid_el = tree.find("uuidGenerate/uuid") + MY_UUID = uuid_el.text + return MY_UUID + + +def yesno(flag): + return "Yes" if flag else "No" + + +def oknotok(flag): + return "OK" if flag else "NOT OK" + + +def output_error(message, errcode=1): + print (message, file=sys.stderr) + sys.exit(errcode) + + +def node_output_ok(message=""): + # Prints Success JSON output and exits with returncode zero + out = {"ok": True, "nodeid": get_node_uuid(), "output": message} + print (json.dumps(out)) + sys.exit(0) + + +def node_output_notok(message): + # Prints Error JSON output and exits with returncode zero + out = {"ok": False, "nodeid": get_node_uuid(), "error": message} + print (json.dumps(out)) + sys.exit(0) + + +def execute(cmd): + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + out, err = p.communicate() + return p.returncode, out, err + + +def get_pool_list(): + cmd = ["gluster", "--mode=script", "pool", "list", "--xml"] + rc, out, err = execute(cmd) + if rc != 0: + output_error("Failed to get Pool Info: {0}".format(err)) + + tree = etree.fromstring(out) + + pool = [] + try: + for p in tree.findall('peerStatus/peer'): + pool.append({"nodeid": p.find("uuid").text, + "hostname": p.find("hostname").text, + "connected": (True if p.find("connected").text == "1" + else False)}) + except (ParseError, AttributeError, ValueError) as e: + output_error("Failed to parse Pool Info: {0}".format(e)) + + return pool + + +class NodeOutput(object): + def __init__(self, **kwargs): + self.nodeid = kwargs.get("nodeid", "") + self.hostname = kwargs.get("hostname", "") + self.node_up = kwargs.get("node_up", False) + self.ok = kwargs.get("ok", False) + self.output = kwargs.get("output", "N/A") + self.error = kwargs.get("error", "N/A") + + +def execute_in_peers(name, args=[]): + # Get the file name of Caller function, If the file name is peer_example.py + # then Gluster peer command will be gluster system:: execute example.py + # Command name is without peer_ + frame = inspect.stack()[1] + module = inspect.getmodule(frame[0]) + actual_file = module.__file__ + # If file is symlink then find actual file + if os.path.islink(actual_file): + actual_file = os.readlink(actual_file) + + # Get the name of file without peer_ + cmd_name = os.path.basename(actual_file).replace("peer_", "") + cmd = ["gluster", "system::", "execute", cmd_name, name] + args + rc, out, err = execute(cmd) + if rc != 0: + raise GlusterCmdException((rc, out, err, " ".join(cmd))) + + out = out.strip().splitlines() + + # JSON decode each line and construct one object with node id as key + all_nodes_data = {} + for node_data in out: + data = json.loads(node_data) + all_nodes_data[data["nodeid"]] = { + "nodeid": data.get("nodeid"), + "ok": data.get("ok"), + "output": data.get("output", ""), + "error": data.get("error", "")} + + # gluster pool list + pool_list = get_pool_list() + + data_out = [] + # Iterate pool_list and merge all_nodes_data collected above + # If a peer node is down then set node_up = False + for p in pool_list: + p_data = all_nodes_data.get(p.get("nodeid"), None) + row_data = NodeOutput(node_up=False, + hostname=p.get("hostname"), + nodeid=p.get("nodeid"), + ok=False) + + if p_data is not None: + # Node is UP + row_data.node_up = True + row_data.ok = p_data.get("ok") + row_data.output = p_data.get("output") + row_data.error = p_data.get("error") + + data_out.append(row_data) + + return data_out + + +def sync_file_to_peers(fname): + # Copy file from current node to all peer nodes, fname + # is path after GLUSTERD_WORKDIR + cmd = ["gluster", "system::", "copy", "file", fname] + rc, out, err = execute(cmd) + if rc != 0: + raise GlusterCmdException((rc, out, err)) + + +class Cmd(object): + name = "" + + def run(self, args): + # Must required method. Raise NotImplementedError if derived class + # not implemented this method + raise NotImplementedError("\"run(self, args)\" method is " + "not implemented by \"{0}\"".format( + self.__class__.__name__)) + + +def runcli(): + # Get list of Classes derived from class "Cmd" and create + # a subcommand as specified in the Class name. Call the args + # method by passing subcommand parser, Derived class can add + # arguments to the subcommand parser. + metavar_data = [] + for c in Cmd.__subclasses__(): + cls = c() + if getattr(cls, "name", "") == "": + raise NotImplementedError("\"name\" is not added " + "to \"{0}\"".format( + cls.__class__.__name__)) + + # Do not show in help message if subcommand starts with node- + if not cls.name.startswith("node-"): + metavar_data.append(cls.name) + + p = subparsers.add_parser(cls.name) + args_func = getattr(cls, "args", None) + if args_func is not None: + args_func(p) + + # Apply common args if any + _common_args_func(p) + + # A dict to save subcommands, key is name of the subcommand + subcommands[cls.name] = cls + + # Hide node commands in Help message + subparsers.metavar = "{" + ",".join(metavar_data) + "}" + + # Get all parsed arguments + args = parser.parse_args() + + # Get the subcommand to execute + cls = subcommands.get(args.mode, None) + + # Run + if cls is not None: + cls.run(args) + + +def set_common_args_func(func): + global _common_args_func + _common_args_func = func diff --git a/extras/collect-system-stats.sh b/extras/collect-system-stats.sh new file mode 100755 index 00000000000..865e70bbc11 --- /dev/null +++ b/extras/collect-system-stats.sh @@ -0,0 +1,52 @@ +#!/bin/bash +################################################################################ +# Usage: collect-system-stats.sh <delay-in-seconds> +# This script starts sar/top/iostat/vmstat processes which collect system stats +# with the interval <delay-in-seconds> given as argument to the script. When +# the script is stopped either by entering any input or Ctrl+C the list of +# files where output is captured will be printed on the screen which can be +# observed to find any problems/bottlenecks. +############################################################################### + +function stop_processes { + echo "Stopping the monitoring processes" + echo "sar pid:$sar_pid", "top pid: $top_pid", "iostat pid: $iostat_pid", "vmstat pid: $vmstat_pid" + kill "$sar_pid" "$top_pid" "$iostat_pid" "$vmstat_pid" + echo "Files created: ${timestamp}-network.out, ${timestamp}-top.out, ${timestamp}-iostat.out, ${timestamp}-vmstat.out" +} + +function check_dependent_commands_exist() +{ + declare -a arr=("sar" "top" "iostat" "vmstat") + for i in "${arr[@]}" + do + if ! command -v "$i" > /dev/null 2>&1 + then + echo "ERROR: '$i' command is not found" + exit 1 + fi + done + +} + +case "$1" in + ''|*[!0-9]*) echo "Usage: $0 <delay-between-successive-metrics-collection-in-seconds>"; exit 1 ;; + *) interval="$1" ;; +esac + +timestamp=$(date +"%s") + +check_dependent_commands_exist +sar -n DEV "$interval" > "${timestamp}"-network.out & +sar_pid="$!" +top -bHd "$interval" > "${timestamp}"-top.out & +top_pid="$!" +iostat -Ntkdx "$interval" > "${timestamp}"-iostat.out & +iostat_pid="$!" +vmstat -t "$interval" > "${timestamp}"-vmstat.out & +vmstat_pid="$!" +echo "Started sar, vmstat, iostat, top for collecting stats" + + +trap stop_processes EXIT +read -r -p "Press anything and ENTER to exit"; diff --git a/extras/command-completion/Makefile b/extras/command-completion/Makefile new file mode 100644 index 00000000000..06cbfe0672f --- /dev/null +++ b/extras/command-completion/Makefile @@ -0,0 +1,6 @@ +install: + mkdir -p /etc/bash_completion.d + cp gluster.bash /etc/bash_completion.d/gluster + +uninstall: + rm -f /etc/bash_completion.d/gluster diff --git a/extras/command-completion/README b/extras/command-completion/README new file mode 100644 index 00000000000..0acba38168f --- /dev/null +++ b/extras/command-completion/README @@ -0,0 +1,5 @@ +This file is not moved to /etc/bash_completion.d/ with the source installation. +Please execute make install explicity from here to move this file to +/etc/bash_completion.d + +Similarly, use make uninstall to remove it from /etc/bash_completion.d diff --git a/extras/command-completion/gluster.bash b/extras/command-completion/gluster.bash new file mode 100644 index 00000000000..73d16098875 --- /dev/null +++ b/extras/command-completion/gluster.bash @@ -0,0 +1,492 @@ +#!/bin/bash + +if pidof glusterd > /dev/null 2>&1; then + GLUSTER_SET_OPTIONS=" + $(for token in `gluster volume set help 2>/dev/null | grep "^Option:" | cut -d ' ' -f 2` + do + echo "{$token}," + done) + " + GLUSTER_RESET_OPTIONS="$GLUSTER_SET_OPTIONS" +fi + +GLUSTER_TOP_SUBOPTIONS1=" + {nfs}, + {brick}, + {list-cnt} +" +GLUSTER_TOP_SUBOPTIONS2=" + {bs + {__SIZE + {count} + } + }, + {brick}, + {list-cnt} +" +GLUSTER_TOP_OPTIONS=" + {open + [ $GLUSTER_TOP_SUBOPTIONS1 ] + }, + {read + [ $GLUSTER_TOP_SUBOPTIONS1 ] + }, + {write + [ $GLUSTER_TOP_SUBOPTIONS1 ] + }, + {opendir + [ $GLUSTER_TOP_SUBOPTIONS1 ] + }, + {readdir + [ $GLUSTER_TOP_SUBOPTIONS1 ] + }, + {clear + [ $GLUSTER_TOP_SUBOPTIONS1 ] + }, + {read-perf + [ $GLUSTER_TOP_SUBOPTIONS2 ] + }, + {write-perf + [ $GLUSTER_TOP_SUBOPTIONS2 ] + } +" + +GLUSTER_QUOTA_OPTIONS=" + {enable}, + {disable}, + {list}, + {remove}, + {default-soft-limit}, + {limit-usage}, + {alert-time}, + {soft-timeout}, + {hard-timeout} +" + +GLUSTER_PROFILE_OPTIONS=" + {start}, + {info [ + {peek}, + {incremental + {peek} + }, + {cumulative}, + {clear}, + ] + }, + {stop} +" + +GLUSTER_BARRIER_OPTIONS=" + {enable}, + {disable} +" + +GLUSTER_GEO_REPLICATION_SUBOPTIONS=" +" +GLUSTER_GEO_REPLICATION_OPTIONS=" + {__VOLNAME [ + {__SLAVEURL [ + {create [ + {push-pem + {force} + }, + {force} + ] + }, + {start {force} }, + {status {detail} }, + {config}, + {pause {force} }, + {resume {force} }, + {stop {force} }, + {delete {force} } + ] + }, + {status} + ] + }, + {status} +" + +GLUSTER_VOLUME_OPTIONS=" + {volume [ + {add-brick + {__VOLNAME} + }, + {barrier + {__VOLNAME + [ $GLUSTER_BARRIER_OPTIONS ] + } + }, + {clear-locks + {__VOLNAME} + }, + {create}, + {delete + {__VOLNAME} + }, + {geo-replication + [ $GLUSTER_GEO_REPLICATION_OPTIONS ] + }, + {heal + {__VOLNAME} + }, + {help}, + {info + {__VOLNAME} + }, + {list}, + {log + {__VOLNAME} + }, + {profile + {__VOLNAME + [ $GLUSTER_PROFILE_OPTIONS ] + } + }, + {quota + {__VOLNAME + [ $GLUSTER_QUOTA_OPTIONS ] + } + }, + {rebalance + {__VOLNAME} + }, + {remove-brick + {__VOLNAME} + }, + {replace-brick + {__VOLNAME} + }, + {reset + {__VOLNAME + [ $GLUSTER_RESET_OPTIONS ] + } + }, + {set + {__VOLNAME + [ $GLUSTER_SET_OPTIONS ] + } + }, + {start + {__VOLNAME + {force} + } + }, + {statedump + {__VOLNAME} + }, + {status + {__VOLNAME} + }, + {stop + {__VOLNAME + {force} + } + }, + {sync + {__HOSTNAME} + }, + {top + {__VOLNAME + [ $GLUSTER_TOP_OPTIONS ] + } + } + ] + } +" + +GLUSTER_COMMAND_TREE=" +{gluster [ + $GLUSTER_VOLUME_OPTIONS , + {peer [ + {probe + {__HOSTNAME} + }, + {detach + {__HOSTNAME + {force} + } + }, + {status} + ] + }, + {pool + {list} + }, + {help} + ] +}" + +__SIZE () +{ + return 0 +} + +__SLAVEURL () +{ + return 0 +} + +__HOSTNAME () +{ + local zero=0 + local ret=0 + local cur_word="$2" + + if [ "$1" == "X" ]; then + return + + elif [ "$1" == "match" ]; then + return 0 + + elif [ "$1" == "complete" ]; then + COMPREPLY=($(compgen -A hostname -- $cur_word)) + fi + return 0 +} + +__VOLNAME () +{ + local zero=0 + local ret=0 + local cur_word="$2" + local list="" + + if [ "X$1" == "X" ]; then + return + + elif [ "$1" == "match" ]; then + return 0 + + elif [ "$1" == "complete" ]; then + if ! pidof glusterd > /dev/null 2>&1; then + list=''; + + else + list=`gluster volume list 2> /dev/null` + fi + + else + return 0 + fi + + COMPREPLY=($(compgen -W "$list" -- $cur_word)) + return 0 +} + +_gluster_throw () { +#echo $1 >&2 + COMPREPLY='' + exit +} + +declare GLUSTER_FINAL_LIST='' +declare GLUSTER_LIST='' +declare -i GLUSTER_TOP=0 +_gluster_push () { + GLUSTER_TOP=$((GLUSTER_TOP + 1)) + return $GLUSTER_TOP +} +_gluster_pop () { + GLUSTER_TOP=$((GLUSTER_TOP - 1)) + return $GLUSTER_TOP +} + +_gluster_goto_end () +{ + local prev_top=$1 + local top=$1 + local token='' + + while [ $top -ge $prev_top ]; do + read -r token + case $token in + '{' | '[') + _gluster_push + top=$? + ;; + '}' | ']') + _gluster_pop + top=$? + ;; + esac + done + + return +} + +_gluster_form_list () +{ + local token='' + local top=0 + local comma='' + local cur_word="$1" + + read -r token + case $token in + ']') + ;; + '{') + _gluster_push + top=$? + read -r key + if [ "X$cur_word" == "X" -o "${cur_word:0:1}" == "${key:0:1}" -o "${key:0:1}" == "_" ]; then + GLUSTER_LIST="$GLUSTER_LIST $key" + fi + + _gluster_goto_end $top + read -r comma + if [ "$comma" == "," ]; then + _gluster_form_list $cur_word + fi + ;; + *) + _gluster_throw "Expected '{' but received $token" + ;; + esac + + return +} + +_gluster_goto_child () +{ + local match_string="$1" + local token='' + local top=0 + local comma='' + + read -r token + case $token in + '{') + _gluster_push + top=$? + ;; + *) + _gluster_throw "Expected '{' but received $token" + ;; + esac + + read -r token + case `echo $token` in + '[' | ']' | '{' | '}') + _gluster_throw "Expected string but received $token" + ;; + _*) + $token "match" $match_string + ret=$? + if [ $ret -eq 0 ]; then + return + else + _gluster_goto_end $top + + read -r comma + if [ "$comma" == "," ]; then + _gluster_goto_child $match_string + fi + fi + ;; + + "$match_string") + return + ;; + *) + _gluster_goto_end $top + + read -r comma + if [ "$comma" == "," ]; then + _gluster_goto_child $match_string + fi + ;; + esac + + return +} + +_gluster_does_match () +{ + local token="$1" + local key="$2" + + if [ "${token:0:1}" == "_" ]; then + $token $2 + return $? + fi + + [ "$token" == "$key" ] && return 0 + + return 1 +} + +_gluster_parse () +{ + local i=0 + local token='' + local tmp_token='' + local word='' + + while [ $i -lt $COMP_CWORD ]; do + read -r token + case $token in + '[') + _gluster_push + _gluster_goto_child ${COMP_WORDS[$i]} + ;; + '{') + _gluster_push + read -r tmp_token + _gluster_does_match $tmp_token ${COMP_WORDS[$i]} + if [ $? -ne 0 ]; then + _gluster_throw "No match" + fi + ;; + esac + i=$((i+1)) + done + + read -r token + if [ "$token" == '[' ]; then + _gluster_push + _gluster_form_list ${COMP_WORDS[COMP_CWORD]} + + elif [ "$token" == '{' ]; then + read -r tmp_token + GLUSTER_LIST="$tmp_token" + fi + + echo $GLUSTER_LIST +} + +_gluster_handle_list () +{ + local list="${!1}" + local cur_word=$2 + local count=0 + local i=0 + + for i in `echo $list`; do + count=$((count + 1)) + done + + if [ $count -eq 1 ] && [ "${i:0:1}" == "_" ]; then + $i "complete" $cur_word + else + COMPREPLY=($(compgen -W "$list" -- $cur_word)) + fi + return +} + +_gluster_completion () +{ + GLUSTER_FINAL_LIST=`echo $GLUSTER_COMMAND_TREE | \ + egrep -ao --color=never "([A-Za-z0-9_.-]+)|[[:space:]]+|." | \ + egrep -v --color=never "^[[:space:]]*$" | \ + _gluster_parse` + + ARG="GLUSTER_FINAL_LIST" + _gluster_handle_list $ARG ${COMP_WORDS[COMP_CWORD]} + return +} + +complete -F _gluster_completion gluster diff --git a/extras/control-cpu-load.sh b/extras/control-cpu-load.sh new file mode 100755 index 00000000000..52dcf62fd9f --- /dev/null +++ b/extras/control-cpu-load.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +USAGE="This script provides a utility to control CPU utilization for any +gluster daemon.In this, we use cgroup framework to configure CPU quota +for a process(like selfheal daemon). Before running this script, make +sure that daemon is running.Every time daemon restarts, it is required +to rerun this command to set CPU quota on new daemon process id. +User can enter any value between 10 to 100 for CPU quota. +Recommended value of quota period is 25. 25 means, kernel will allocate +25 ms period to this group of tasks in every 100 ms period. This 25ms +could be considered as the maximum percentage of CPU quota daemon can take. +This value will be reflected on CPU usage of "top" command.If provided pid +is the only process and no other process is in competition to get CPU, more + than 25% could be allocated to daemon to speed up the process." + +if [ $# -ge 1 ]; then + case $1 in + -h|--help) echo " " "$USAGE" | sed -r -e 's/^[ ]+//g' + exit 0; + ;; + *) echo "Please Provide correct input for script." + echo "For help correct options are -h or --help." + exit 1; + ;; + esac +fi + +DIR_EXIST=0 +LOC="/sys/fs/cgroup/cpu,cpuacct/system.slice/glusterd.service" +echo "Enter gluster daemon pid for which you want to control CPU." +read daemon_pid + +if expr ${daemon_pid} + 0 > /dev/null 2>&1 ;then + CHECK_PID=$(pgrep -f gluster | grep ${daemon_pid}) + if [ -z "${CHECK_PID}" ]; then + echo "No daemon is running or pid ${daemon_pid} does not match." + echo "with running gluster processes." + exit 1 + fi +else + echo "Entered daemon_pid is not numeric so Rerun the script." + exit 1 +fi + + +if [ -f ${LOC}/tasks ];then + CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/tasks) + if [ ${CHECK_CGROUP} ]; then + echo "pid ${daemon_pid} is attached with glusterd.service cgroup." + fi +fi + +cgroup_name=cgroup_gluster_${daemon_pid} +if [ -f ${LOC}/${cgroup_name}/tasks ]; then + CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/${cgroup_name}/tasks) + if [ ${CHECK_CGROUP} ]; then + val=`cat ${LOC}/${cgroup_name}/cpu.cfs_quota_us` + qval=$((val / 1000)) + echo "pid ${daemon_pid} is already attached ${cgroup_name} with quota value ${qval}." + echo "Press n if you don't want to reassign ${daemon_pid} with new quota value." + DIR_EXIST=1 + else + echo "pid ${daemon_pid} is not attached with ${cgroup_name}." + fi +fi + +read -p "If you want to continue the script to attach ${daemon_pid} with new ${cgroup_name} cgroup Press (y/n)?" choice +case "$choice" in + y|Y ) echo "yes";; + n|N ) echo "no";exit;; + * ) echo "invalid";exit;; +esac + +systemctl set-property glusterd.service CPUShares=1024 + +if [ ${DIR_EXIST} -eq 0 ];then + echo "Creating child cgroup directory '${cgroup_name} cgroup' for glusterd.service." + mkdir -p ${LOC}/${cgroup_name} + if [ ! -f ${LOC}/${cgroup_name}/tasks ];then + echo "Not able to create ${cgroup_name} directory so exit." + exit 1 + fi +fi + +echo "Enter quota value in range [10,100]: " + +read quota_value +if expr ${quota_value} + 0 > /dev/null 2>&1 ;then + if [ ${quota_value} -lt 10 ] || [ ${quota_value} -gt 100 ]; then + echo "Entered quota value is not correct,it should be in the range ." + echo "10-100. Ideal value is 25." + echo "Rerun the sript with correct value." + exit 1 + else + echo "Entered quota value is $quota_value" + fi +else + echo "Entered quota value is not numeric so Rerun the script." + exit 1 +fi + +quota_value=$((quota_value * 1000)) +echo "Setting $quota_value to cpu.cfs_quota_us for gluster_cgroup." +echo ${quota_value} > ${LOC}/${cgroup_name}/cpu.cfs_quota_us + +if ps -T -p ${daemon_pid} | grep gluster > /dev/null; then + for thid in `ps -T -p ${daemon_pid} | grep -v SPID | awk -F " " '{print $2}'`; + do + echo ${thid} > ${LOC}/${cgroup_name}/tasks ; + done + if cat /proc/${daemon_pid}/cgroup | grep -w ${cgroup_name} > /dev/null; then + echo "Tasks are attached successfully specific to ${daemon_pid} to ${cgroup_name}." + else + echo "Tasks are not attached successfully." + fi +fi diff --git a/extras/control-mem.sh b/extras/control-mem.sh new file mode 100755 index 00000000000..91b36f8107a --- /dev/null +++ b/extras/control-mem.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +USAGE="This commands provides a utility to control MEMORY utilization for any +gluster daemon.In this, we use cgroup framework to configure MEMORY limit for +a process. Before running this script, make sure that daemon is running.Every +time daemon restarts, it is required to rerun this command to set memory limit +(in bytes) on new daemon process id.User can enter any value between 100 +(in Mega bytes) to 8000000000000 for Memory limit in Mega bytes. +Memory limit value is depends on how much maximum memory user wants to restrict +for specific daemon process.If a process will try to consume memore more than +configured value then cgroup will hang/sleep this task and to resume the task +rerun the script with new increase memory limit value ." + +if [ $# -ge 1 ]; then + case $1 in + -h|--help) echo " " "$USAGE" | sed -r -e 's/^[ ]+//g' + exit 0; + ;; + *) echo "Please Provide correct input for script." + echo "For help correct options are -h of --help." + exit 1; + ;; + esac +fi + +DIR_EXIST=0 +LOC="/sys/fs/cgroup/memory/system.slice/glusterd.service" +echo "Enter Any gluster daemon pid for that you want to control MEMORY." +read daemon_pid + +if expr ${daemon_pid} + 0 > /dev/null 2>&1 ;then + CHECK_PID=$(pgrep -f gluster | grep ${daemon_pid}) + if [ -z "${CHECK_PID}" ]; then + echo "No daemon is running or pid ${daemon_pid} does not match." + echo "with running gluster processes." + exit 1 + fi +else + echo "Entered daemon_pid is not numeric so Rerun the script." + exit 1 +fi + + +if [ -f ${LOC}/tasks ]; then + CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/tasks) + if [ ${CHECK_CGROUP} ] ;then + echo "pid ${daemon_pid} is attached with default glusterd.service cgroup." + fi +fi + +cgroup_name=cgroup_gluster_${daemon_pid} +if [ -f ${LOC}/${cgroup_name}/tasks ];then + CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/${cgroup_name}/tasks) + if [ ${CHECK_CGROUP} ]; then + val=`cat ${LOC}/${cgroup_name}/memory.limit_in_bytes` + mval=$((val / 1024 / 1024)) + echo "pid ${daemon_pid} is already attached ${cgroup_name} with mem value ${mval}." + echo "Press n if you don't want to reassign ${daemon_pid} with new mem value." + DIR_EXIST=1 + else + echo "pid ${daemon_pid} is not attached with ${cgroup_name}." + fi +fi + +read -p "If you want to continue the script to attach daeomon with new cgroup. Press (y/n)?" choice +case "$choice" in + y|Y ) echo "yes";; + n|N ) echo "no";exit;; + * ) echo "invalid";exit;; +esac + +systemctl set-property glusterd.service CPUShares=1024 + +if [ ${DIR_EXIST} -eq 0 ];then + echo "Creating child cgroup directory '${cgroup_name} cgroup' for glusterd.service." + mkdir -p ${LOC}/${cgroup_name} + if [ ! -f ${LOC}/${cgroup_name}/tasks ];then + echo "Not able to create ${LOC}/${cgroup_name} directory so exit." + exit 1 + fi +fi + +echo "Enter Memory value in Mega bytes [100,8000000000000]: " + +read mem_value +if expr ${mem_value} + 0 > /dev/null 2>&1 ;then + if [ ${mem_value} -lt 100 ] || [ ${mem_value} -gt 8000000000000 ]; then + echo "Entered memory value is not correct,it should be in the range ." + echo "100-8000000000000, Rerun the script with correct value ." + exit 1 + else + echo "Entered memory limit value is ${mem_value}." + fi +else + echo "Entered memory value is not numeric so Rerun the script." + exit 1 +fi + +mem_value=$(($mem_value * 1024 * 1024)) +if [ ${DIR_EXIST} -eq 0 ];then + echo "Setting ${mem_value} to memory.limit_in_bytes for ${LOC}/${cgroup_name}." + echo ${mem_value} > ${LOC}/${cgroup_name}/memory.limit_in_bytes + #Set memory value to memory.memsw.limit_in_bytes + echo ${mem_value} > ${LOC}/${cgroup_name}/memory.memsw.limit_in_bytes + # disable oom_control so that kernel will not send kill signal to the + # task once limit has reached + echo 1 > ${LOC}/${cgroup_name}/memory.oom_control +else + #Increase mem_value to memory.memsw.limit_in_bytes + echo ${mem_value} > ${LOC}/${cgroup_name}/memory.memsw.limit_in_bytes + echo "Increase ${mem_value} to memory.limit_in_bytes for ${LOC}/${cgroup_name}." + echo ${mem_value} > ${LOC}/${cgroup_name}/memory.limit_in_bytes + # disable oom_control so that kernel will not send kill signal to the + # task once limit has reached + echo 1 > ${LOC}/${cgroup_name}/memory.oom_control +fi + +if ps -T -p ${daemon_pid} | grep gluster > /dev/null; then + for thid in `ps -T -p ${daemon_pid} | grep -v SPID | awk -F " " '{print $2}'`; + do + echo ${thid} > ${LOC}/${cgroup_name}/tasks ; + done + if cat /proc/${daemon_pid}/cgroup | grep -iw ${cgroup_name} > /dev/null; then + echo "Tasks are attached successfully specific to ${daemon_pid} to ${cgroup_name}." + else + echo "Tasks are not attached successfully." + fi +fi diff --git a/extras/create_new_xlator/README.md b/extras/create_new_xlator/README.md new file mode 100644 index 00000000000..fdc1ba00812 --- /dev/null +++ b/extras/create_new_xlator/README.md @@ -0,0 +1,24 @@ +####This document explains how to create a template for your new xlator. + +`$ python ./generate_xlator.py <XLATOR_DIRECTORY> <XLATOR_NAME> <FOP_PREFIX>` + * XLATOR_DIRECTORY: Directory path where the new xlator folder will reside + * XLATOR_NAME: Name of the xlator you wish to create + * FOP_PREFIX: This is the fop prefix that you wish to prefix every fop definition in your xlator, fop prefix is generally different than xlator name, if the xlator name is too long. + +Eg: `python ./generate_xlator.py /home/u1/glusterfs/xlators/features compression cmpr` +This command will create the following files with some initial contents like copyright, fops definition etc. +Note that there shouldn't be a "/" specified at the end of the <XLATOR_DIRECTORY> + `* /home/u1/glusterfs/xlators/features/compression/Makefile.am + * /home/u1/glusterfs/xlators/features/compression/src/Makefile.am + * /home/u1/glusterfs/xlators/features/compression/src/compression.c + * /home/u1/glusterfs/xlators/features/compression/src/compression.h + * /home/u1/glusterfs/xlators/features/compression/src/compression-mem-types.h + * /home/u1/glusterfs/xlators/features/compression/src/compression-messages.h` + +By default all the fops and functions are generated, if you wish to not implement certain fops and functions, comment those lines (by adding '#' at the start of the line) in libglusterfs/src/generate_xlator.py + +Few other manual steps required to get the new xlator completely functional: +* Change configure.ac +* Change `<XLATOR_DIRECTORY>/Makefile.am` to include the new xlator directory. + Eg: `/home/u1/glusterfs/xlators/features/Makefile.am` +* Change vol file or glusterd volgen to include the new xlator in volfile diff --git a/extras/create_new_xlator/generate_xlator.py b/extras/create_new_xlator/generate_xlator.py new file mode 100755 index 00000000000..983868c04db --- /dev/null +++ b/extras/create_new_xlator/generate_xlator.py @@ -0,0 +1,208 @@ +#!/usr/bin/python3 + +from __future__ import print_function +import os +import re +import sys +import string +import time +path = os.path.abspath(os.path.dirname(__file__)) + '/../../libglusterfs/src' +sys.path.append(path) +from generator import ops, xlator_cbks, xlator_dumpops + +MAKEFILE_FMT = """ +xlator_LTLIBRARIES = @XL_NAME@.la +xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/@XL_TYPE@ +@XL_NAME_NO_HYPHEN@_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) +@XL_NAME_NO_HYPHEN@_la_SOURCES = @XL_NAME@.c +@XL_NAME_NO_HYPHEN@_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la +noinst_HEADERS = @XL_NAME@.h @XL_NAME@-mem-types.h @XL_NAME@-messages.h +AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ + -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src +AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS) +CLEANFILES = +""" + +fop_subs = {} +cbk_subs = {} +fn_subs = {} + + +def get_error_arg(type_str): + if type_str.find(" *") != -1: + return "NULL" + return "-1" + + +def get_param(names, types): + # Convert two separate tuples to one of (name, type) sub-tuples. + as_tuples = list(zip(types, names)) + # Convert each sub-tuple into a "type name" string. + as_strings = [' '.join(item) for item in as_tuples] + # Join all of those into one big string. + return ',\n\t'.join(as_strings) + + +def generate(tmpl, name, table): + w_arg_names = [a[1] for a in table[name] if a[0] == 'fop-arg'] + w_arg_types = [a[2] for a in table[name] if a[0] == 'fop-arg'] + u_arg_names = [a[1] for a in table[name] if a[0] == 'cbk-arg'] + u_arg_types = [a[2] for a in table[name] if a[0] == 'cbk-arg'] + fn_arg_names = [a[1] for a in table[name] if a[0] == 'fn-arg'] + fn_arg_types = [a[2] for a in table[name] if a[0] == 'fn-arg'] + ret_type = [a[1] for a in table[name] if a[0] == 'ret-val'] + ret_var = [a[2] for a in table[name] if a[0] == 'ret-val'] + + sdict = {} + #Parameters are (t1, var1), (t2, var2)... + #Args are (var1, var2,...) + sdict["@WIND_ARGS@"] = ', '.join(w_arg_names) + sdict["@UNWIND_ARGS@"] = ', '.join(u_arg_names) + sdict["@ERROR_ARGS@"] = ', '.join(list(map(get_error_arg, u_arg_types))) + sdict["@WIND_PARAMS@"] = get_param(w_arg_names, w_arg_types) + sdict["@UNWIND_PARAMS@"] = get_param(u_arg_names, u_arg_types) + sdict["@FUNC_PARAMS@"] = get_param(fn_arg_names, fn_arg_types) + sdict["@NAME@"] = name + sdict["@FOP_PREFIX@"] = fop_prefix + sdict["@RET_TYPE@"] = ''.join(ret_type) + sdict["@RET_VAR@"] = ''.join(ret_var) + + for old, new in sdict.items(): + tmpl = tmpl.replace(old, new) + # TBD: reindent/reformat the result for maximum readability. + return tmpl + + +def gen_xlator(): + xl = open(src_dir_path+"/"+xl_name+".c", 'w+') + + print(COPYRIGHT, file=xl) + print(fragments["INCLUDE_IN_SRC_FILE"].replace("@XL_NAME@", + xl_name), file=xl) + + #Generate cbks and fops + for fop in ops: + print(generate(fragments["CBK_TEMPLATE"], fop, ops), file=xl) + print(generate(fragments["FOP_TEMPLATE"], fop, ops), file=xl) + + for cbk in xlator_cbks: + print(generate(fragments["FUNC_TEMPLATE"], cbk, + xlator_cbks), file=xl) + + for dops in xlator_dumpops: + print(generate(fragments["FUNC_TEMPLATE"], dops, + xlator_dumpops), file=xl) + + #Generate fop table + print("struct xlator_fops fops = {", file=xl) + for fop in ops: + print(" .{0:20} = {1}_{2},".format(fop, fop_prefix, fop), file=xl) + print("};", file=xl) + + #Generate xlator_cbks table + print("struct xlator_cbks cbks = {", file=xl) + for cbk in xlator_cbks: + print(" .{0:20} = {1}_{2},".format(cbk, fop_prefix, cbk), file=xl) + print("};", file=xl) + + #Generate xlator_dumpops table + print("struct xlator_dumpops dumpops = {", file=xl) + for dops in xlator_dumpops: + print(" .{0:20} = {1}_{2},".format(dops, fop_prefix, dops), file=xl) + print("};", file=xl) + + xlator_methods = fragments["XLATOR_METHODS"].replace("@XL_NAME@", xl_name) + xlator_methods = xlator_methods.replace("@FOP_PREFIX@", fop_prefix) + print(xlator_methods, file=xl) + + xl.close() + + +def create_dir_struct(): + if not os.path.exists(dir_path+"/src"): + os.makedirs(dir_path+"/src") + + +def gen_header_files(): + upname = xl_name_no_hyphen.upper() + h = open(src_dir_path+"/"+xl_name+".h", 'w+') + print(COPYRIGHT, file=h) + txt = fragments["HEADER_FMT"].replace("@HFL_NAME@", upname) + txt = txt.replace("@XL_NAME@", xl_name) + print(txt, file=h) + h.close() + + h = open(src_dir_path+"/"+xl_name+"-mem-types.h", 'w+') + print(COPYRIGHT, file=h) + txt = fragments["MEM_HEADER_FMT"].replace("@HFL_NAME@", upname+"_MEM_TYPES") + txt = txt.replace("@FOP_PREFIX@", fop_prefix) + print(txt, file=h) + h.close() + + h = open(src_dir_path+"/"+xl_name+"-messages.h", 'w+') + print(COPYRIGHT, file=h) + txt = fragments["MSG_HEADER_FMT"].replace("@HFL_NAME@", upname+"_MESSAGES") + txt = txt.replace("@FOP_PREFIX@", fop_prefix.upper()) + print(txt, file=h) + h.close() + + +def gen_makefiles(): + m = open(dir_path+"/Makefile.am", 'w+') + print("SUBDIRS = src\n\nCLEANFILES =", file=m) + m.close() + + m = open(src_dir_path+"/Makefile.am", 'w+') + txt = MAKEFILE_FMT.replace("@XL_NAME@", xl_name) + txt = txt.replace("@XL_NAME_NO_HYPHEN@", xl_name_no_hyphen) + txt = txt.replace("@XL_TYPE@", xlator_type) + print(txt, file=m) + m.close() + +def get_copyright (): + return fragments["CP"].replace("@CURRENT_YEAR@", + time.strftime("%Y")) + +def load_fragments (): + pragma_re = re.compile('pragma fragment (.*)') + cur_symbol = None + cur_value = "" + result = {} + basepath = os.path.abspath(os.path.dirname(__file__)) + fragpath = basepath + "/new-xlator.c.tmpl" + for line in open(fragpath, "r").readlines(): + m = pragma_re.search(line) + if m: + if cur_symbol: + result[cur_symbol] = cur_value + cur_symbol = m.group(1) + cur_value = "" + else: + cur_value += line + if cur_symbol: + result[cur_symbol] = cur_value + return result + +if __name__ == '__main__': + + if len(sys.argv) < 3: + print("USAGE: ./gen_xlator <XLATOR_DIR> <XLATOR_NAME> <FOP_PREFIX>") + sys.exit(0) + + xl_name = sys.argv[2] + xl_name_no_hyphen = xl_name.replace("-", "_") + if sys.argv[1].endswith('/'): + dir_path = sys.argv[1] + xl_name + else: + dir_path = sys.argv[1] + "/" + xl_name + xlator_type = os.path.basename(sys.argv[1]) + fop_prefix = sys.argv[3] + src_dir_path = dir_path + "/src" + + fragments = load_fragments() + + COPYRIGHT = get_copyright() + create_dir_struct() + gen_xlator() + gen_header_files() + gen_makefiles() diff --git a/extras/create_new_xlator/new-xlator.c.tmpl b/extras/create_new_xlator/new-xlator.c.tmpl new file mode 100644 index 00000000000..fe9735bfcf1 --- /dev/null +++ b/extras/create_new_xlator/new-xlator.c.tmpl @@ -0,0 +1,151 @@ +#pragma fragment CBK_TEMPLATE +int32_t @FOP_PREFIX@_@NAME@_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, @UNWIND_PARAMS@) +{ + STACK_UNWIND_STRICT(@NAME@, frame, op_ret, op_errno, @UNWIND_ARGS@); + return 0; +} + +#pragma fragment COMMENT +If you are generating the leaf xlators, remove the STACK_WIND and replace the + @ERROR_ARGS@ to @UNWIND_ARGS@ if necessary + +#pragma fragment FOP_TEMPLATE + int32_t @FOP_PREFIX@_@NAME@(call_frame_t *frame, xlator_t *this, @WIND_PARAMS@) +{ + STACK_WIND(frame, @FOP_PREFIX@_@NAME@_cbk, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->@NAME@, @WIND_ARGS@); + return 0; +err: + STACK_UNWIND_STRICT(@NAME@, frame, -1, errno, @ERROR_ARGS@); + return 0; +} + +#pragma fragment FUNC_TEMPLATE +@RET_TYPE@ @FOP_PREFIX@_@NAME@(@FUNC_PARAMS@) +{ + return @RET_VAR@; +} + +#pragma fragment CP +/* + * Copyright (c) @CURRENT_YEAR@ Red Hat, Inc. <http://www.redhat.com> + * This file is part of GlusterFS. + * + * This file is licensed to you under your choice of the GNU Lesser + * General Public License, version 3 or any later version (LGPLv3 or + * later), or the GNU General Public License, version 2 (GPLv2), in all + * cases as published by the Free Software Foundation. + */ + +#pragma fragment INCLUDE_IN_SRC_FILE +#include "@XL_NAME@.h" + +#pragma fragment XLATOR_METHODS + +static int32_t @FOP_PREFIX@_init(xlator_t *this) +{ + return 0; +} + +static void @FOP_PREFIX@_fini(xlator_t *this) +{ + return; +} + +static int32_t @FOP_PREFIX@_reconfigure(xlator_t *this, dict_t *dict) +{ + return 0; +} + +static int @FOP_PREFIX@_notify(xlator_t *this, int event, void *data, ...) +{ + return default_notify(this, event, data); +} + +static int32_t @FOP_PREFIX@_mem_acct_init(xlator_t *this) +{ + int ret = -1; + + ret = xlator_mem_acct_init(this, gf_@FOP_PREFIX@_mt_end + 1); + return ret; +} + +static int32_t @FOP_PREFIX@_dump_metrics(xlator_t *this, int fd) +{ + return 0; +} + +struct volume_options @FOP_PREFIX@_options[] = { + /*{ .key = {""}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "", + .op_version = {GD_OP_VERSION_}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT, + .tags = {""}, + .description = "", + .category = GF_EXPERIMENTAL, + }, + { .key = {NULL} }, + */ +}; + +xlator_api_t xlator_api = { + .init = @FOP_PREFIX@_init, + .fini = @FOP_PREFIX@_fini, + .notify = @FOP_PREFIX@_notify, + .reconfigure = @FOP_PREFIX@_reconfigure, + .mem_acct_init = @FOP_PREFIX@_mem_acct_init, + .dump_metrics = @FOP_PREFIX@_dump_metrics, + .op_version = {GD_OP_VERSION_}, + .dumpops = &@FOP_PREFIX@_dumpops, + .fops = &@FOP_PREFIX@_fops, + .cbks = &@FOP_PREFIX @_cbks, + .options = @FOP_PREFIX@_options, + .identifier = "@XL_NAME@", + .category = GF_EXPERIMENTAL, +}; +#pragma fragment HEADER_FMT +#ifndef __ @HFL_NAME@_H__ +#define __ @HFL_NAME@_H__ + +#include "@XL_NAME@-mem-types.h" +#include "@XL_NAME@-messages.h" +#include <glusterfs/glusterfs.h> +#include <glusterfs/xlator.h> +#include <glusterfs/defaults.h> + +#endif /* __@HFL_NAME@_H__ */ + +#pragma fragment MEM_HEADER_FMT +#ifndef __ @HFL_NAME@_H__ +#define __ @HFL_NAME@_H__ + +#include <glusterfs/mem-types.h> + +enum gf_mdc_mem_types_ { + gf_@FOP_PREFIX@_mt_ = gf_common_mt_end + 1, + gf_@FOP_PREFIX@_mt_end +}; + +#endif /* __@HFL_NAME@_H__ */ + +#pragma fragment MSG_HEADER_FMT +#ifndef __@HFL_NAME@_H__ +#define __@HFL_NAME@_H__ + +#include <glusterfs/glfs-message-id.h> + +/* To add new message IDs, append new identifiers at the end of the list. + * + * Never remove a message ID. If it's not used anymore, you can rename it or + * leave it as it is, but not delete it. This is to prevent reutilization of + * IDs by other messages. + * + * The component name must match one of the entries defined in + * glfs-message-id.h. + */ + +GLFS_MSGID(@FOP_PREFIX@, @FOP_PREFIX@_MSG_NO_MEMORY); + +#endif /* __@HFL_NAME@_H__ */ diff --git a/extras/defrag.sh b/extras/defrag.sh deleted file mode 100644 index 465b0979488..00000000000 --- a/extras/defrag.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh - -# This script gets called from 'scale-n-defrag.sh' script. -# Don't run this stand alone. -# -# - -set -e - -CP="cp" -MV="mv" - -scan_dir() -{ - path=$1; - find "$path" -type f -perm +01000 -exec $0 '{}' \; -} - -rsync_filename() -{ - path=$1 - dir=$(dirname "$path"); - file=$(basename "$path"); - - echo "$dir/.$file.zr$$"; -} - -relocate_file() -{ - path=$1; - tmp_path=$(rsync_filename "$path"); - - pre_mtime=$(stat -c '%Y' "$path"); - $CP -a "$path" "$tmp_path"; - post_mtime=$(stat -c '%Y' "$path"); - - if [ $pre_mtime = $post_mtime ]; then - chmod -t "$tmp_path"; - $MV "$tmp_path" "$path"; - echo "file '$path' relocated" - else - echo "file '$path' modified during defrag. skipping" - rm -f "$tmp_path"; - fi -} - -main() -{ - path="$1"; - - if [ -d "$path" ]; then - scan_dir "$path"; - else - relocate_file "$@"; - fi - - usleep 500000 # 500ms -} - -main "$1" diff --git a/extras/devel-tools/devel-vagrant/Vagrantfile b/extras/devel-tools/devel-vagrant/Vagrantfile new file mode 100644 index 00000000000..78dc29bdc68 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/Vagrantfile @@ -0,0 +1,165 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# +# Author: Rajesh Joseph (rjoseph@redhat.com) +# Author: Christopher Blum (cblum@redhat.com) +# + + +#Variables +box_name = "gluster-dev-fedora" +box_url = "http://download.gluster.org/pub/gluster/glusterfs/vagrant/gluster-dev-fedora/boxes/gluster-dev-fedora.json" +node_count = 0 +disk_count = -1 +node_name = "Node" +ipbase="192.168.99." +source_path = "/source/glusterfs" +target_path = "/mnt/src" + +if ARGV[0] == "up" + environment = open('vagrant_env.conf', 'w') + + print "\n\e[1;37mEnter Node (or VM) Count? Default: 1 \e[32m" + while node_count < 1 or node_count > 99 + node_count = $stdin.gets.strip.to_i + if node_count == 0 # The user pressed enter without input or we cannot parse the input to a number + node_count = 1 + elsif node_count < 1 + print "\e[31mWe need at least 1 VM ;) Try again \e[32m" + elsif node_count > 99 + print "\e[31mWe don't support more than 99 VMs - Try again \e[32m" + end + end + + print "\e[1;37mEnter per Node Disc (Brick) Count? Default: 2 \e[32m" + + while disk_count < 1 + disk_count = $stdin.gets.strip.to_i + if disk_count == 0 # The user pressed enter without input or we cannot parse the input to a number + disk_count = 2 + elsif disk_count < 1 + print "\e[31mWe need at least 1 disk ;) Try again \e[32m" + end + end + + print "\e[1;37mEnter GlusterFS source location? Default: \"#{source_path}\" : \e[32m" + tmploc = $stdin.gets.strip.to_s + if tmploc != "" + source_path = "#{tmploc}" + end + + environment.puts("# BEWARE: Do NOT modify ANY settings in here or your vagrant environment will be messed up") + environment.puts(node_count.to_s) + environment.puts(disk_count.to_s) + environment.puts(source_path) + + print "\e[32m\nOK I will provision #{node_count} VMs for you and each one will have #{disk_count} disks for bricks\e[37m\n\n" + system "sleep 1" +else # So that we destroy and can connect to all VMs... + environment = open('vagrant_env.conf', 'r') + + environment.readline # Skip the comment on top + node_count = environment.readline.to_i + disk_count = environment.readline.to_i + source_path = environment.readline.gsub(/\s+/, "") + + if ARGV[0] != "ssh-config" + puts "Detected settings from previous vagrant up:" + puts " We deployed #{node_count} VMs with each #{disk_count} disks" + puts "" + end +end + +environment.close + +$ansivar = Hash.new{ |hash,key| hash[key] = [] } +$devnamecreated = false + +# +# Function to create and attach disks +# +def attachDisks(numDisk, provider) + suffix = "bcdefghijklmn".split("") + for i in 1..numDisk.to_i + devname = "vd" + (suffix[i-1]).to_s + if $devnamecreated == false + $ansivar["device"].push "#{devname}" + end + provider.storage :file, + :size => '1G', + :device => "vd" + (suffix[i-1]).to_s, + :type => "qcow2", + :bus => "virtio", + :cache => "default" + end + $devnamecreated = true +end + + +$ansivar["src_path"].push "#{source_path}" +$ansivar["trg_path"].push "#{target_path}" + +groups = Hash.new{ |hash,key| hash[key] = [] } + +groups["origin"].push "#{node_name}1" +groups["all"].push "#{node_name}1" + +(2..node_count).each do |num| + $ansivar["peer_nodes"].push "#{node_name}#{num}" + groups["all"].push "#{node_name}#{num}" +end + +hostsFile = "\n" +(1..node_count).each do |num| + hostsFile += "#{ipbase}#{( 100 + num).to_s} #{node_name}#{num.to_s}\n" +end + +Vagrant.configure("2") do |config| + (1..node_count).each do |num| + config.vm.define "#{node_name}#{num}" do |node| + ip_addr = "#{ipbase}#{(100 + num).to_s}" + node.vm.network "private_network", ip: "#{ip_addr}" + node.vm.box = box_name + node.vm.box_url = box_url + node.vm.hostname = "#{node_name}#{num}" + node.ssh.insert_key = false + node.vm.synced_folder "#{source_path}", "#{target_path}", type: "nfs" + + # Define basic config for VM, memory, cpu, storage pool + node.vm.provider "libvirt" do |virt| + virt.storage_pool_name = "default" + virt.memory = 1024 + virt.cpus = 1 + + attachDisks( disk_count, virt ) + end + + node.vm.post_up_message = "\e[37mBuilding of this VM is finished \n" + "You can access it now with: \n" + "vagrant ssh #{node_name}#{num.to_s}\n\n" + "#{target_path} directory in VM #{node_name}#{num.to_s}" + "is synced with Host machine. \nSo any changes done in this" + "directory will be reflected in the host machine as well\n" + "Beware of this when you delete content from this directory\e[32m" + + node.vm.provision :shell, path: "bootstrap.sh" + + node.vm.provision "shell", inline: <<-SHELL + echo '#{hostsFile}' | sudo tee -a /etc/hosts + SHELL + + if num == node_count + # Let's provision + node.vm.provision "ansible" do |setup| + setup.verbose = "v" + setup.playbook = "ansible/setup.yml" + setup.limit = "all" + setup.sudo = "true" + setup.groups = groups + setup.extra_vars = $ansivar + end + end + + end + end +end diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/cluster/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/cluster/tasks/main.yml new file mode 100644 index 00000000000..3306c7a3dc2 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/cluster/tasks/main.yml @@ -0,0 +1,5 @@ +--- +- name: gluster peer probe + shell: gluster peer probe {{ item }} + with_items: "{{ peer_nodes | default([]) }}" + diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/compile-gluster/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/compile-gluster/tasks/main.yml new file mode 100644 index 00000000000..6ee258c7780 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/compile-gluster/tasks/main.yml @@ -0,0 +1,29 @@ +--- +- name: autogen.sh + shell: chdir={{ item }} ./autogen.sh + with_items: "{{ trg_path }}" + +- name: configure + shell: chdir={{ item }} CFLAGS="-g -O0 -Werror -Wall -Wno-error=cpp -Wno-error=maybe-uninitialized" \ + ./configure \ + --prefix=/usr \ + --exec-prefix=/usr \ + --bindir=/usr/bin \ + --sbindir=/usr/sbin \ + --sysconfdir=/etc \ + --datadir=/usr/share \ + --includedir=/usr/include \ + --libdir=/usr/lib64 \ + --libexecdir=/usr/libexec \ + --localstatedir=/var \ + --sharedstatedir=/var/lib \ + --mandir=/usr/share/man \ + --infodir=/usr/share/info \ + --libdir=/usr/lib64 \ + --enable-debug + with_items: "{{ trg_path }}" + +- name: make install + shell: chdir={{ item }} make install + with_items: "{{ trg_path }}" + diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/install-pkgs/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/install-pkgs/tasks/main.yml new file mode 100644 index 00000000000..3944054dd25 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/install-pkgs/tasks/main.yml @@ -0,0 +1,72 @@ +--- +- name: install deltarpm + dnf: name=deltarpm state=present + +- name: update system + shell: dnf update -y + +- name: install other packages + dnf: name={{ item }} state=present + with_items: + - attr + - autoconf + - automake + - bison + - cifs-utils + - cscope + - ctags + - dbench + - dos2unix + - e2fsprogs + - findutils + - flex + - fuse-devel + - fuse-libs + - gcc + - gdb + - git + - glib2-devel + - hostname + - libacl-devel + - libaio-devel + - libattr-devel + - libibverbs-devel + - librdmacm-devel + - libtool + - libxml2-devel + - lvm2-devel + - make + - man-db + - mock + - net-tools + - nfs-utils + - openssh-server + - openssl-devel + - perl-Test-Harness + - pkgconfig + - procps-ng + - psmisc + - python-devel + - python-eventlet + - python-netifaces + - python-paste-deploy + - python-setuptools + - python-simplejson + - python-sphinx + - python-webob + - pyxattr + - readline-devel + - rpm-build + - screen + - strace + - supervisor + - systemtap-sdt-devel + - sqlite-devel + - samba* + - userspace-rcu-devel + - vim + - wget + - which + - xfsprogs + - yajl-devel + diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/iptables/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/iptables/tasks/main.yml new file mode 100644 index 00000000000..768cb0e8668 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/iptables/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: disable iptables, need to add specific rules later + shell: iptables -F diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/prepare-brick/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/prepare-brick/tasks/main.yml new file mode 100644 index 00000000000..a3a6c463468 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/prepare-brick/tasks/main.yml @@ -0,0 +1,30 @@ +--- +- name: Create physical device + shell: pvcreate /dev/{{ item }} + with_items: "{{ device }}" + +- name: Create volume group + shell: vgcreate vg{{ item }} /dev/{{ item }} + with_items: "{{ device }}" + +- name: Create thin pool + shell: lvcreate -L 950M -T vg{{ item }}/thinpool + with_items: "{{ device }}" + +- name: Create thin volume + shell: lvcreate -V900M -T vg{{ item }}/thinpool -n thinp1 + with_items: "{{ device }}" + +- name: Format backend + filesystem: fstype=xfs dev=/dev/vg{{ item }}/thinp1 + with_items: "{{ device }}" + +- name: Create mount directory + file: path=/bricks/br{{ item }} state=directory recurse=yes + with_items: "{{ device }}" + +- name: Add entry to fstab and mount + mount: name=/bricks/br{{ item }} src=/dev/vg{{ item }}/thinp1 fstype=xfs state=mounted + with_items: "{{ device }}" + + diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/selinux/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/selinux/tasks/main.yml new file mode 100644 index 00000000000..c9ba9618428 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/selinux/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: Allow gfapi in Samba to bind to other ports than well known smb ports + seboolean: name=samba_load_libgfapi state=yes persistent=yes diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/service/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/service/tasks/main.yml new file mode 100644 index 00000000000..ef78a125ae8 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/service/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: disable kernel nfs + service: name=nfs-server enabled=no + +- name: stop kernel nfs + service: name=nfs-server state=stopped + +- name: enable rpcbind + service: name=rpcbind enabled=yes + +- name: start rpcbind + service: name=rpcbind state=started + +- name: enable glusterd + service: name=glusterd enabled=yes + +- name: start glusterd + service: name=glusterd state=started + + + diff --git a/extras/devel-tools/devel-vagrant/ansible/setup.yml b/extras/devel-tools/devel-vagrant/ansible/setup.yml new file mode 100644 index 00000000000..c26bd7d6051 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/setup.yml @@ -0,0 +1,23 @@ +--- +- hosts: all + become: yes + become_method: sudo + roles: + - install-pkgs + - prepare-brick + - selinux + - iptables + +- hosts: all + become: yes + become_method: sudo + serial: 1 + roles: + - compile-gluster + - service + +- hosts: origin + become: yes + become_method: sudo + roles: + - cluster diff --git a/extras/devel-tools/devel-vagrant/bootstrap.sh b/extras/devel-tools/devel-vagrant/bootstrap.sh new file mode 100644 index 00000000000..bbf9fa2c063 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/bootstrap.sh @@ -0,0 +1,3 @@ +dnf install -y nfs-utils +dnf install -y vim +dnf install -y python2 python2-dnf libselinux-python libsemanage-python diff --git a/extras/devel-tools/devel-vagrant/up.sh b/extras/devel-tools/devel-vagrant/up.sh new file mode 100755 index 00000000000..35cbe79d835 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/up.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +vagrant up --no-provision $@ +vagrant provision diff --git a/extras/devel-tools/gdb_macros b/extras/devel-tools/gdb_macros new file mode 100644 index 00000000000..aae4ccb3b37 --- /dev/null +++ b/extras/devel-tools/gdb_macros @@ -0,0 +1,84 @@ + +# +# gdb_macros is a gdb script file which can assist +# developer in debugging. This script provides +# following functions for debugging gluster processes +# effectively: +# +# pdict : This function will iterate through all the +# dictionary (dict_t) members and print the +# key-value pair. +# +# plist : This function will print address of each member +# of gluster list (list_head). +# +# gdb script should be loaded in gdb before using these +# functions. gdb script can be loaded on-demand or on gdb +# start-up. Use the following command on gdb prompt for +# loading it on-demand. +# source <script filename> +# e.g. +# (gdb) source /mnt/gdb_macros +# +# To automatically load gdb script on startup use .gdbinit +# file. This file is automatically loaded by gdb on start. +# Edit (or create) ~/.gdbinit file and add the following +# entry: +# source /mnt/gdb_macros +# + + + +# This is an internal helper function +define _print_dict_data + set $data = ($arg0) + set $len = $data->len - 1 + set $i = 0 + set $ishex = 0 + while ($i < $len) + set $ch = $data->data [$i] + + # Print only alpha-numeric values as char + # and remaining as hex value. This is done + # in this way because using %s screws up + # the display if the string has non-displayable + # characters + if ($ishex == 0) && ($ch > 31) && ($ch < 127) + printf "%c", $ch + else + printf "%x", ($ch & 0xFF) + set $ishex = 1 + end + set $i = $i + 1 + end +end + + +define pdict + set $cnt = 1 + set $temp = *($arg0->members) + while ($temp) + printf "[%d](%s::",$cnt, $temp->key + _print_dict_data ($temp)->value + printf ")\n" + set $temp = $temp->next + set $cnt = $cnt + 1 + end + printf "Done\n" +end + + +define plist + set $node = &($arg0) + set $head = $node + + printf "[0x%lx]", $node + set $node = $node->next + + while ($node != $head) + printf "--> [0x%lx]", $node + set $node = $node->next + end + printf "\n" +end + diff --git a/extras/devel-tools/print-backtrace.sh b/extras/devel-tools/print-backtrace.sh new file mode 100755 index 00000000000..33fbae288bc --- /dev/null +++ b/extras/devel-tools/print-backtrace.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# sample unresolved backtrace lines picked up from a brick log that should go +# into a backtrace file eg. bt-file.txt: +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3ec81)[0x7fe4bc271c81] +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3eecd)[0x7fe4bc271ecd] +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x404cb)[0x7fe4bc2734cb] +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d2b6)[0x7fe4bc2702b6] +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d323)[0x7fe4bc270323] +# +# following is the output of the script for the above backtrace lines: +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3ec81)[0x7fe4bc271c81] __afr_selfheal_data_finalize_source inlined at /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:684 in __afr_selfheal_data_prepare /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:603 +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3eecd)[0x7fe4bc271ecd] __afr_selfheal_data /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:740 +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x404cb)[0x7fe4bc2734cb] afr_selfheal_data /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:883 +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d2b6)[0x7fe4bc2702b6] afr_selfheal_do /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-common.c:1968 +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d323)[0x7fe4bc270323] afr_selfheal /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-common.c:2015 +# +# Usage with debuginfo RPM: +# print-backtrace.sh $HOME/Downloads/glusterfs-debuginfo-3.8.4-10.el7.x86_64.rpm bt-file.txt +# +# Usage with source install: +# print-packtrace.sh none bt-file.txt + +function version_compare() { test $(echo $1|awk -F '.' '{print $1 $2 $3}') -gt $(echo $2|awk -F '.' '{print $1 $2 $3}'); } + +function Usage() +{ + echo -e "Usage:\n\t$0 { none | <debuginfo-rpm> } <backtrace-file>" + echo "none: implies we don't have a debuginfo rpm but want to resolve" + echo " against a source install which already has the debuginfo" + echo " NOTE: in this case you should have configured the build" + echo " with --enable-debug and the linker options should" + echo " have the option -rdynamic" +} + +debuginfo_rpm=$1 +backtrace_file=$2 + +if [ ! $debuginfo_rpm ] || [ ! $backtrace_file ]; then + Usage + exit 1 +fi + +if [ $debuginfo_rpm != "none" ]; then + if [ ! -f $debuginfo_rpm ]; then + echo "no such rpm file: $debuginfo_rpm" + exit 1 + fi +fi + +if [ ! -f $backtrace_file ]; then + echo "no such backtrace file: $backtrace_file" + exit 1 +fi + +if [ "$debuginfo_rpm" != "none" ]; then + if ! file $debuginfo_rpm | grep RPM >/dev/null 2>&1 ; then + echo "file does not look like an rpm: $debuginfo_rpm" + exit 1 + fi +fi + +cpio_version=$(cpio --version|grep cpio|cut -f 2 -d ')'|sed -e 's/^[[:space:]]*//') +rpm_name="" +debuginfo_path="" +debuginfo_extension="" + +if [ $debuginfo_rpm != "none" ]; then + # extract the gluster debuginfo rpm to resolve the symbols against + rpm_name=$(basename $debuginfo_rpm '.rpm') + if [ -d $rpm_name ]; then + echo "directory already exists: $rpm_name" + echo "please remove/move it and reattempt" + exit 1 + fi + mkdir -p $rpm_name + if version_compare $cpio_version "2.11"; then + rpm2cpio $debuginfo_rpm | cpio --quiet --extract --make-directories --preserve-modification-time --directory=$rpm_name + ret=$? + else + current_dir="$PWD" + cd $rpm_name + rpm2cpio $debuginfo_rpm | cpio --quiet --extract --make-directories --preserve-modification-time + ret=$? + cd $current_dir + fi + if [ $ret -eq 1 ]; then + echo "failed to extract rpm $debuginfo_rpm to $PWD/$rpm_name directory" + rm -rf $rpm_name + exit 1 + fi + debuginfo_path="$PWD/$rpm_name/usr/lib/debug" + debuginfo_extension=".debug" +else + debuginfo_path="" + debuginfo_extension="" +fi + +# NOTE: backtrace file should contain only the lines which need to be resolved +for bt in $(cat $backtrace_file) +do + libname=$(echo $bt | cut -f 1 -d '(') + addr=$(echo $bt | cut -f 2 -d '(' | cut -f 1 -d ')') + libpath=${debuginfo_path}${libname}${debuginfo_extension} + if [ ! -f $libpath ]; then + continue + fi + newbt=( $(eu-addr2line --functions --exe=$libpath $addr) ) + echo "$bt ${newbt[*]}" +done + +# remove the temporary directory +if [ -d $rpm_name ]; then + rm -rf $rpm_name +fi + diff --git a/extras/devel-tools/strace-brick.sh b/extras/devel-tools/strace-brick.sh new file mode 100755 index 00000000000..a140729111c --- /dev/null +++ b/extras/devel-tools/strace-brick.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Usage: +# nice -n -19 strace-brick.sh glusterfsd 50 + +brick_process_name=$1 +min_watch_cpu=$2 +if [ ! $brick_process_name ]; then + brick_process_name=glusterfsd +fi + +if [ ! $min_watch_cpu ]; then + min_watch_cpu=50 +fi + +echo "min_watch_cpu: $min_watch_cpu" + +break=false + +while ! $break; +do + mypids=( $(pgrep $brick_process_name) ) + echo "mypids: ${mypids[*]}" + + pid_args=$(echo ${mypids[*]} | sed -e 's/ / -p /g;s/^/-p /') + echo "pid_args: $pid_args" + + pcpu=( $(ps $pid_args -o pcpu -h ) ) + echo "pcpu: ${pcpu[*]}" + + wait_longer=false + + for i in $( seq 0 $((${#pcpu[*]} - 1)) ) + do + echo "i: $i" + echo "mypids[$i]: ${mypids[$i]}" + + int_pcpu=$(echo ${pcpu[$i]} | cut -f 1 -d '.') + echo "int_pcpu: $int_pcpu" + if [ ! $int_pcpu ] || [ ! $min_watch_cpu ]; then + break=true + echo "breaking" + fi + if [ $int_pcpu -ge $min_watch_cpu ]; then + wait_longer=true + mydirname="${brick_process_name}-${mypids[$i]}-$(date --utc +'%Y%m%d-%H%M%S.%N')" + $(mkdir $mydirname && cd $mydirname && timeout --kill-after=5 --signal=KILL 60 nice -n -19 strace -p ${mypids[$i]} -ff -tt -T -o $brick_process_name) & + fi + done + + if $wait_longer; then + sleep 90 + else + sleep 15 + fi +done diff --git a/extras/disk_usage_sync.sh b/extras/disk_usage_sync.sh new file mode 100755 index 00000000000..85ee158f888 --- /dev/null +++ b/extras/disk_usage_sync.sh @@ -0,0 +1,86 @@ +#!/bin/sh + +# This script can be used to sync disk usage before activating quotas on GlusterFS. +# There are two scenarios where quotas are used and this script has to be used accordingly - +# +# 1. Server side +# The script needs to be run with backend export directory as the argument. This script updates +# the current disk usage of the backend directory in a way that is intelligible to the GlusterFS quota +# translator. Make sure you run this script before starting glusterfsd (GlusterFS server process): +# * for the first time +# * after any server outage (reboot, etc.) + +# 2. Client side +# The script needs to be run with the client mount point as the argument. It updates the current disk +# of the GlusterFS volume is a way that is intelligible to the GlusterFS quota translator. Make sure +# you run this script after a fresh mount of the GlusterFS volume on the client: +# * For the first time +# * After any client outage (reboot, remount, etc.) + +# Please note that this script is dependent on the 'attr' package, more specifically 'setfattr' to set +# extended attributes of files. +# GlusterFS +# +# (c) 2010 Gluster Inc <http://www.gluster.com/> + +PROGRAM_NAME="disk_usage_sync.sh" + +#check if setfattr is available +check_for_attr () +{ + `command -v setfattr >/dev/null` + if [ "${?}" -gt 0 ]; then + echo >&2 "This script requires the 'attr' package to run. Either it has not been installed or is not present currently in the system path." + exit 1 + fi + +} + +usage () { + echo >&2 "$PROGRAM_NAME - Command used to sync disk usage information before activating quotas on GlusterFS. + +usage: $PROGRAM_NAME <target-directory>" + +exit 1 +} + +TARGET_DIR=$1 +EXT_ATTR_NAME="trusted.glusterfs-quota-du" + +#output of du in number of bytes +get_disk_usage () +{ + if [ ! -d $TARGET_DIR ]; then + echo >&2 "Error: $TARGET_DIR does not exist." + exit 1 + fi + + DISK_USAGE=`du -bc $TARGET_DIR | grep 'total' | cut -f1` + if [ "${?}" -gt 0 ]; then + exit 1 + fi + +} + +#set the extended attribute of the root directory with du size in bytes +set_disk_usage () +{ + ` setfattr -n $EXT_ATTR_NAME -v $DISK_USAGE $TARGET_DIR` + if [ "${?}" -gt 0 ]; then + exit 1 + fi +} + +main () +{ + [ $# -lt 1 ] && usage + + check_for_attr + + get_disk_usage + set_disk_usage + + printf "Disk Usage information has been sync'd successfully.\n" +} + +main "$@" diff --git a/extras/distributed-testing/README b/extras/distributed-testing/README new file mode 100644 index 00000000000..928d943f211 --- /dev/null +++ b/extras/distributed-testing/README @@ -0,0 +1,28 @@ +PROBLEM + +The testing methodology of Gluster is extremely slow. It takes a very long time (6+ hrs) to run the basic tests on a single machine. It takes about 20+ hours to run code analysis version of tests like valgrind, asan, tsan etc. + +SOLUTION + +The fundamental problem is that the tests cannot be parallelized on a single machine. The natural solution is to run these tests on a cluster of machines. In a nutshell, apply map-reduce to run unit tests. + +WORK @ Facebook + +At Facebook we have applied the map-reduce approach to testing and have observed 10X improvements. + +The solution supports the following + +Distribute tests across machines, collect results/logs +Share worker pool across different testers +Try failure 3 times on 3 different machines before calling it a failure +Support running asan, valgrind, asan-noleaks +Self management of worker pools. The clients will manage the worker pool including version update, no manual maintenance required +WORK + +Port the code from gluster-fb-3.8 to gluster master + +HOW TO RUN + +./extras/distributed-testing/distributed-test.sh --hosts '<h1> <h2> <h3>' + +All hosts should have no password for ssh via root. This can be achieved with keys setup on the client and the server machines. diff --git a/extras/distributed-testing/distributed-test-build-env b/extras/distributed-testing/distributed-test-build-env new file mode 100644 index 00000000000..cd68ff717da --- /dev/null +++ b/extras/distributed-testing/distributed-test-build-env @@ -0,0 +1,20 @@ +#!/bin/bash + +GF_CONF_OPTS="--localstatedir=/var --sysconfdir /var/lib --prefix /usr --libdir /usr/lib64 \ + --enable-bd-xlator=yes --enable-debug --enable-gnfs" + +if [ -x /usr/lib/rpm/redhat/dist.sh ]; then + REDHAT_MAJOR=$(/usr/lib/rpm/redhat/dist.sh --distnum) +else + REDHAT_MAJOR=0 +fi + +ASAN_ENABLED=${ASAN_ENABLED:=0} +if [ "$ASAN_ENABLED" -eq "1" ]; then + GF_CONF_OPTS="$GF_CONF_OPTS --with-asan" +fi + +GF_CONF_OPTS="$GF_CONF_OPTS --with-systemd" +export GF_CONF_OPTS + +export CFLAGS="-O0 -ggdb -fPIC -Wall" diff --git a/extras/distributed-testing/distributed-test-build.sh b/extras/distributed-testing/distributed-test-build.sh new file mode 100755 index 00000000000..e8910d8425c --- /dev/null +++ b/extras/distributed-testing/distributed-test-build.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +EXTRA_CONFIGURE_ARGS="$@" +ASAN_REQUESTED=false +for arg in $EXTRA_CONFIGURE_ARGS; do + if [ $arg == "--with-asan" ]; then + echo "Requested ASAN, cleaning build first." + make -j distclean || true + touch .with_asan + ASAN_REQUESTED=true + fi +done + +if [ $ASAN_REQUESTED == false ]; then + if [ -f .with_asan ]; then + echo "Previous build was with ASAN, cleaning build first." + make -j distclean || true + rm -v .with_asan + fi +fi + +source extras/distributed-testing/distributed-test-build-env +./autogen.sh +./configure $GF_CONF_OPTS $EXTRA_CONFIGURE_ARGS +make -j diff --git a/extras/distributed-testing/distributed-test-env b/extras/distributed-testing/distributed-test-env new file mode 100644 index 00000000000..36fdd82e5dd --- /dev/null +++ b/extras/distributed-testing/distributed-test-env @@ -0,0 +1,48 @@ +#!/bin/bash + +SMOKE_TESTS="\ + tests/basic/*.t\ + tests/basic/afr/*.t\ + tests/basic/distribute/*.t\ + tests/bugs/fb*.t\ + tests/features/brick-min-free-space.t\ +" + +KNOWN_FLAKY_TESTS="\ +" + +BROKEN_TESTS="\ + tests/features/lock_revocation.t\ + tests/features/recon.t\ + tests/features/fdl-overflow.t\ + tests/features/fdl.t\ + tests/features/ipc.t\ + tests/bugs/distribute/bug-1247563.t\ + tests/bugs/distribute/bug-1543279.t\ + tests/bugs/distribute/bug-1066798.t\ + tests/bugs/ec/bug-1304988.t\ + tests/bugs/unclassified/bug-1357397.t\ + tests/bugs/quota/bug-1235182.t\ + tests/bugs/fuse/bug-1309462.t\ + tests/bugs/glusterd/bug-1238706-daemons-stop-on-peer-cleanup.t\ + tests/bugs/stripe/bug-1002207.t\ + tests/bugs/stripe/bug-1111454.t\ + tests/bugs/snapshot/bug-1140162-file-snapshot-features-encrypt-opts-validation.t\ + tests/bugs/write-behind/bug-1279730.t\ + tests/bugs/gfapi/bug-1093594.t\ + tests/bugs/replicate/bug-1473026.t\ + tests/bugs/replicate/bug-802417.t\ + tests/basic/inode-leak.t\ + tests/basic/distribute/force-migration.t\ + tests/basic/ec/heal-info.t\ + tests/basic/ec/ec-seek.t\ + tests/basic/jbr/jbr-volgen.t\ + tests/basic/jbr/jbr.t\ + tests/basic/afr/tarissue.t\ + tests/basic/tier/tierd_check.t\ + tests/basic/gfapi/bug1291259.t\ +" + +SMOKE_TESTS=$(echo $SMOKE_TESTS | tr -s ' ' ' ') +KNOWN_FLAKY_TESTS=$(echo $KNOWN_FLAKY_TESTS | tr -s ' ' ' ') +BROKEN_TESTS=$(echo $BROKEN_TESTS | tr -s ' ' ' ') diff --git a/extras/distributed-testing/distributed-test-runner.py b/extras/distributed-testing/distributed-test-runner.py new file mode 100755 index 00000000000..5a07e2feab1 --- /dev/null +++ b/extras/distributed-testing/distributed-test-runner.py @@ -0,0 +1,859 @@ +#!/usr/bin/python3 + +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +import re +import sys +import fcntl +import base64 +import threading +import socket +import os +import shlex +import argparse +import subprocess +import time +import SimpleXMLRPCServer +import xmlrpclib +import md5 +import httplib +import uuid + +DEFAULT_PORT = 9999 +TEST_TIMEOUT_S = 15 * 60 +CLIENT_CONNECT_TIMEOUT_S = 10 +CLIENT_TIMEOUT_S = 60 +PATCH_FILE_UID = str(uuid.uuid4()) +SSH_TIMEOUT_S = 10 +MAX_ATTEMPTS = 3 +ADDRESS_FAMILY = 'IPv4' + + +def socket_instance(address_family): + if address_family.upper() == 'ipv4'.upper(): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + elif address_family.upper() == 'ipv6'.upper(): + return socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + else: + Log.error("Invalid IP address family") + sys.exit(1) + + +def patch_file(): + return "/tmp/%s-patch.tar.gz" % PATCH_FILE_UID + +# .............................................................................. +# SimpleXMLRPCServer IPvX Wrapper +# .............................................................................. + + +class GeneralXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer): + def __init__(self, addr): + SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, addr) + + def server_bind(self): + if self.socket: + self.socket.close() + self.socket = socket_instance(args.address_family) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + SimpleXMLRPCServer.SimpleXMLRPCServer.server_bind(self) + + +class HTTPConnection(httplib.HTTPConnection): + def __init__(self, host): + self.host = host + httplib.HTTPConnection.__init__(self, host) + + def connect(self): + old_timeout = socket.getdefaulttimeout() + self.sock = socket.create_connection((self.host, self.port), + timeout=CLIENT_CONNECT_TIMEOUT_S) + self.sock.settimeout(old_timeout) + + +class IPTransport(xmlrpclib.Transport): + def __init__(self, *args, **kwargs): + xmlrpclib.Transport.__init__(self, *args, **kwargs) + + def make_connection(self, host): + return HTTPConnection(host) + + +# .............................................................................. +# Common +# .............................................................................. + + +class Timer: + def __init__(self): + self.start = time.time() + + def elapsed_s(self): + return int(time.time() - self.start) + + def reset(self): + ret = self.elapsed_s() + self.start = time.time() + return ret + + +def encode(buf): + return base64.b16encode(buf) + + +def decode(buf): + return base64.b16decode(buf) + + +def get_file_content(path): + with open(path, "r") as f: + return f.read() + + +def write_to_file(path, data): + with open(path, "w") as f: + f.write(data) + + +def failsafe(fn, args=()): + try: + return (True, fn(*args)) + except (xmlrpclib.Fault, xmlrpclib.ProtocolError, xmlrpclib.ResponseError, + Exception) as err: + Log.debug(str(err)) + return (False, None) + + +class LogLevel: + DEBUG = 2 + ERROR = 1 + CLI = 0 + + +class Log: + LOGLEVEL = LogLevel.ERROR + + @staticmethod + def _normalize(msg): + return msg[:100] + + @staticmethod + def debug(msg): + if Log.LOGLEVEL >= LogLevel.DEBUG: + sys.stdout.write("<debug> %s\n" % Log._normalize(msg)) + sys.stdout.flush() + + @staticmethod + def error(msg): + sys.stderr.write("<error> %s\n" % Log._normalize(msg)) + + @staticmethod + def header(msg): + sys.stderr.write("* %s *\n" % Log._normalize(msg)) + + @staticmethod + def cli(msg): + sys.stderr.write("%s\n" % msg) + + +class Shell: + def __init__(self, cwd=None, logpath=None): + self.cwd = cwd + self.shell = True + self.redirect = open(os.devnull if not logpath else logpath, "wr+") + + def __del__(self): + self.redirect.close() + + def cd(self, cwd): + Log.debug("cd %s" % cwd) + self.cwd = cwd + + def truncate(self): + self.redirect.truncate(0) + + def read_logs(self): + self.redirect.seek(0) + return self.redirect.read() + + def check_call(self, cmd): + status = self.call(cmd) + if status: + raise Exception("Error running command %s. status=%s" + % (cmd, status)) + + def call(self, cmd): + if isinstance(cmd, list): + return self._calls(cmd) + + return self._call(cmd) + + def ssh(self, hostname, cmd, id_rsa=None): + flags = "" if not id_rsa else "-i " + id_rsa + return self.call("timeout %s ssh %s root@%s \"%s\"" % + (SSH_TIMEOUT_S, flags, hostname, cmd)) + + def scp(self, hostname, src, dest, id_rsa=None): + flags = "" if not id_rsa else "-i " + id_rsa + return self.call("timeout %s scp %s %s root@%s:%s" % + (SSH_TIMEOUT_S, flags, src, hostname, dest)) + + def output(self, cmd, cwd=None): + Log.debug("%s> %s" % (cwd, cmd)) + return subprocess.check_output(shlex.split(cmd), cwd=self.cwd) + + def _calls(self, cmds): + Log.debug("Running commands. %s" % cmds) + for c in cmds: + status = self.call(c) + if status: + Log.error("Commands failed with %s" % status) + return status + return 0 + + def _call(self, cmd): + if not self.shell: + cmd = shlex.split(cmd) + + Log.debug("%s> %s" % (self.cwd, cmd)) + + status = subprocess.call(cmd, cwd=self.cwd, shell=self.shell, + stdout=self.redirect, stderr=self.redirect) + + Log.debug("return %s" % status) + return status + + +# .............................................................................. +# Server role +# .............................................................................. + +class TestServer: + def __init__(self, port, scratchdir): + self.port = port + self.scratchdir = scratchdir + self.shell = Shell() + self.rpc = None + self.pidf = None + + self.shell.check_call("mkdir -p %s" % self.scratchdir) + self._process_lock() + + def __del__(self): + if self.pidf: + self.pidf.close() + + def init(self): + Log.debug("Starting xmlrpc server on port %s" % self.port) + self.rpc = GeneralXMLRPCServer(("", self.port)) + self.rpc.register_instance(Handlers(self.scratchdir)) + + def serve(self): + (status, _) = failsafe(self.rpc.serve_forever) + Log.cli("== End ==") + + def _process_lock(self): + pid_filename = os.path.basename(__file__).replace("/", "-") + pid_filepath = "%s/%s.pid" % (self.scratchdir, pid_filename) + self.pidf = open(pid_filepath, "w") + try: + fcntl.lockf(self.pidf, fcntl.LOCK_EX | fcntl.LOCK_NB) + # We have the lock, kick anybody listening on this port + self.shell.call("kill $(lsof -t -i:%s)" % self.port) + except IOError: + Log.error("Another process instance is running") + sys.exit(0) + +# +# Server Handler +# + + +handler_lock = threading.Lock() +handler_serving_since = Timer() + + +def synchronized(func): + def decorator(*args, **kws): + handler_lock.acquire() + h = args[0] + try: + h.shell.truncate() + ret = func(*args, **kws) + return ret + except Exception() as err: + Log.error(str(err)) + Log.error(decode(h._log_content())) + raise + finally: + handler_lock.release() + handler_serving_since.reset() + + return decorator + + +class Handlers: + def __init__(self, scratchdir): + self.client_id = None + self.scratchdir = scratchdir + self.gluster_root = "%s/glusterfs" % self.scratchdir + self.shell = Shell(logpath="%s/test-handlers.log" % self.scratchdir) + + def hello(self, id): + if not handler_lock.acquire(False): + return False + try: + return self._hello_locked(id) + finally: + handler_lock.release() + + def _hello_locked(self, id): + if handler_serving_since.elapsed_s() > CLIENT_TIMEOUT_S: + Log.debug("Disconnected client %s" % self.client_id) + self.client_id = None + + if not self.client_id: + self.client_id = id + handler_serving_since.reset() + return True + + return (id == self.client_id) + + @synchronized + def ping(self, id=None): + if id: + return id == self.client_id + return True + + @synchronized + def bye(self, id): + assert id == self.client_id + self.client_id = None + handler_serving_since.reset() + return True + + @synchronized + def cleanup(self, id): + assert id == self.client_id + self.shell.cd(self.gluster_root) + self.shell.check_call("PATH=.:$PATH; sudo ./clean_gfs_devserver.sh") + return True + + @synchronized + def copy(self, id, name, content): + with open("%s/%s" % (self.scratchdir, name), "w+") as f: + f.write(decode(content)) + return True + + @synchronized + def copygzip(self, id, content): + assert id == self.client_id + gzipfile = "%s/tmp.tar.gz" % self.scratchdir + tarfile = "%s/tmp.tar" % self.scratchdir + self.shell.check_call("rm -f %s" % gzipfile) + self.shell.check_call("rm -f %s" % tarfile) + write_to_file(gzipfile, decode(content)) + + self.shell.cd(self.scratchdir) + self.shell.check_call("rm -r -f %s" % self.gluster_root) + self.shell.check_call("mkdir -p %s" % self.gluster_root) + + self.shell.cd(self.gluster_root) + cmds = [ + "gunzip -f -q %s" % gzipfile, + "tar -xvf %s" % tarfile + ] + return self.shell.call(cmds) == 0 + + @synchronized + def build(self, id, asan=False): + assert id == self.client_id + self.shell.cd(self.gluster_root) + self.shell.call("make clean") + env = "ASAN_ENABLED=1" if asan else "" + return self.shell.call( + "%s ./extras/distributed-testing/distributed-test-build.sh" % env) == 0 + + @synchronized + def install(self, id): + assert id == self.client_id + self.shell.cd(self.gluster_root) + return self.shell.call("make install") == 0 + + @synchronized + def prove(self, id, test, timeout, valgrind="no", asan_noleaks=True): + assert id == self.client_id + self.shell.cd(self.gluster_root) + env = "DEBUG=1 " + if valgrind == "memcheck" or valgrind == "yes": + cmd = "valgrind" + cmd += " --tool=memcheck --leak-check=full --track-origins=yes" + cmd += " --show-leak-kinds=all -v prove -v" + elif valgrind == "drd": + cmd = "valgrind" + cmd += " --tool=drd -v prove -v" + elif asan_noleaks: + cmd = "prove -v" + env += "ASAN_OPTIONS=detect_leaks=0 " + else: + cmd = "prove -v" + + status = self.shell.call( + "%s timeout %s %s %s" % (env, timeout, cmd, test)) + + if status != 0: + return (False, self._log_content()) + return (True, "") + + def _log_content(self): + return encode(self.shell.read_logs()) + +# .............................................................................. +# Cli role +# .............................................................................. + + +class RPCConnection((threading.Thread)): + def __init__(self, host, port, path, cb): + threading.Thread.__init__(self) + self.host = host + self.port = port + self.path = path + self.shell = Shell() + self.cb = cb + self.stop = False + self.proxy = None + self.logid = "%s:%s" % (self.host, self.port) + + def connect(self): + (status, ret) = failsafe(self._connect) + return (status and ret) + + def _connect(self): + url = "http://%s:%s" % (self.host, self.port) + self.proxy = xmlrpclib.ServerProxy(url, transport=IPTransport()) + return self.proxy.hello(self.cb.id) + + def disconnect(self): + self.stop = True + + def ping(self): + return self.proxy.ping() + + def init(self): + return self._copy() and self._compile_and_install() + + def run(self): + (status, ret) = failsafe(self.init) + if not status: + self.cb.note_lost_connection(self) + return + elif not ret: + self.cb.note_setup_failed(self) + return + + while not self.stop: + (status, ret) = failsafe(self._run) + if not status or not ret: + self.cb.note_lost_connection(self) + break + time.sleep(0) + + failsafe(self.proxy.bye, (self.cb.id,)) + Log.debug("%s connection thread stopped" % self.host) + + def _run(self): + test = self.cb.next_test() + (status, _) = failsafe(self._execute_next, (test,)) + if not status: + self.cb.note_retry(test) + return False + return True + + def _execute_next(self, test): + if not test: + time.sleep(1) + return + + (status, error) = self.proxy.prove(self.cb.id, test, + self.cb.test_timeout, + self.cb.valgrind, + self.cb.asan_noleaks) + if status: + self.cb.note_done(test) + else: + self.cb.note_error(test, error) + + def _compile_and_install(self): + Log.debug("<%s> Build " % self.logid) + asan = self.cb.asan or self.cb.asan_noleaks + return (self.proxy.build(self.cb.id, asan) and + self.proxy.install(self.cb.id)) + + def _copy(self): + return self._copy_gzip() + + def _copy_gzip(self): + Log.cli("<%s> copying and compiling %s to remote" % + (self.logid, self.path)) + data = encode(get_file_content(patch_file())) + Log.debug("GZIP size = %s B" % len(data)) + return self.proxy.copygzip(self.cb.id, data) + + +class RPCConnectionPool: + def __init__(self, gluster_path, hosts, n, id_rsa): + self.gluster_path = gluster_path + self.hosts = hosts + self.conns = [] + self.faulty = [] + self.n = int(len(hosts) / 2) + 1 if not n else n + self.id_rsa = id_rsa + self.stop = False + self.scanner = threading.Thread(target=self._scan_hosts_loop) + self.kicker = threading.Thread(target=self._kick_hosts_loop) + self.shell = Shell() + self.since_start = Timer() + + self.shell.check_call("rm -f %s" % patch_file()) + self.shell.check_call("tar -zcvf %s ." % patch_file()) + self.id = md5.new(get_file_content(patch_file())).hexdigest() + Log.cli("client UID %s" % self.id) + Log.cli("patch UID %s" % PATCH_FILE_UID) + + def __del__(self): + self.shell.check_call("rm -f %s" % patch_file()) + + def pool_status(self): + elapsed_m = int(self.since_start.elapsed_s() / 60) + return "%s/%s connected, %smin elapsed" % (len(self.conns), self.n, + elapsed_m) + + def connect(self): + Log.debug("Starting scanner") + self.scanner.start() + self.kicker.start() + + def disconnect(self): + self.stop = True + for conn in self.conns: + conn.disconnect() + + def note_lost_connection(self, conn): + Log.cli("lost connection to %s" % conn.host) + self.conns.remove(conn) + self.hosts.append((conn.host, conn.port)) + + def note_setup_failed(self, conn): + Log.error("Setup failed on %s:%s" % (conn.host, conn.port)) + self.conns.remove(conn) + self.faulty.append((conn.host, conn.port)) + + def _scan_hosts_loop(self): + Log.debug("Scanner thread started") + while not self.stop: + failsafe(self._scan_hosts) + time.sleep(5) + + def _scan_hosts(self): + if len(self.hosts) == 0 and len(self.conns) == 0: + Log.error("no more hosts available to loadbalance") + sys.exit(1) + + for (host, port) in self.hosts: + if (len(self.conns) >= self.n) or self.stop: + break + self._scan_host(host, port) + + def _scan_host(self, host, port): + Log.debug("scanning %s:%s" % (host, port)) + c = RPCConnection(host, port, self.gluster_path, self) + (status, result) = failsafe(c.connect) + if status and result: + self.hosts.remove((host, port)) + Log.debug("Connected to %s:%s" % (host, port)) + self.conns.append(c) + c.start() + Log.debug("%s / %s connected" % (len(self.conns), self.n)) + else: + Log.debug("Failed to connect to %s:%s" % (host, port)) + + def _kick_hosts_loop(self): + Log.debug("Kick thread started") + while not self.stop: + time.sleep(10) + failsafe(self._kick_hosts) + + Log.debug("Kick thread stopped") + + def _is_pingable(self, host, port): + c = RPCConnection(host, port, self.gluster_path, self) + failsafe(c.connect) + (status, result) = failsafe(c.ping) + return status and result + + def _kick_hosts(self): + # Do not kick hosts if we have the optimal number of connections + if (len(self.conns) >= self.n) or self.stop: + Log.debug("Skip kicking hosts") + return + + # Check and if dead kick all hosts + for (host, port) in self.hosts: + if self.stop: + Log.debug("Break kicking hosts") + break + + if self._is_pingable(host, port): + Log.debug("Host=%s is alive. Won't kick" % host) + continue + + Log.debug("Kicking %s" % host) + mypath = sys.argv[0] + myname = os.path.basename(mypath) + destpath = "/tmp/%s" % myname + sh = Shell() + sh.scp(host, mypath, destpath, self.id_rsa) + sh.ssh(host, "nohup %s --server &>> %s.log &" % + (destpath, destpath), self.id_rsa) + + def join(self): + self.scanner.join() + self.kicker.join() + for c in self.conns: + c.join() + + +# .............................................................................. +# test role +# .............................................................................. + +class TestRunner(RPCConnectionPool): + def __init__(self, gluster_path, hosts, n, tests, flaky_tests, valgrind, + asan, asan_noleaks, id_rsa, test_timeout): + RPCConnectionPool.__init__(self, gluster_path, self._parse_hosts(hosts), + n, id_rsa) + self.flaky_tests = flaky_tests.split(" ") + self.pending = [] + self.done = [] + self.error = [] + self.retry = {} + self.error_logs = [] + self.stats_timer = Timer() + self.valgrind = valgrind + self.asan = asan + self.asan_noleaks = asan_noleaks + self.test_timeout = test_timeout + + self.tests = self._get_tests(tests) + + Log.debug("tests: %s" % self.tests) + + def _get_tests(self, tests): + if not tests or tests == "all": + return self._not_flaky(self._all()) + elif tests == "flaky": + return self.flaky_tests + else: + return self._not_flaky(tests.strip().split(" ")) + + def run(self): + self.connect() + self.join() + return len(self.error) + + def _pretty_print(self, data): + if isinstance(data, list): + str = "" + for i in data: + str = "%s %s" % (str, i) + return str + return "%s" % data + + def print_result(self): + Log.cli("== RESULTS ==") + Log.cli("SUCCESS : %s" % len(self.done)) + Log.cli("ERRORS : %s" % len(self.error)) + Log.cli("== ERRORS ==") + Log.cli(self._pretty_print(self.error)) + Log.cli("== LOGS ==") + Log.cli(self._pretty_print(self.error_logs)) + Log.cli("== END ==") + + def next_test(self): + if len(self.tests): + test = self.tests.pop() + self.pending.append(test) + return test + + if not len(self.pending): + self.disconnect() + + return None + + def _pct_completed(self): + total = len(self.tests) + len(self.pending) + len(self.done) + total += len(self.error) + completed = len(self.done) + len(self.error) + return 0 if not total else int(completed / total * 100) + + def note_done(self, test): + Log.cli("%s PASS (%s%% done) (%s)" % (test, self._pct_completed(), + self.pool_status())) + self.pending.remove(test) + self.done.append(test) + if test in self.retry: + del self.retry[test] + + def note_error(self, test, errstr): + Log.cli("%s FAIL" % test) + self.pending.remove(test) + if test not in self.retry: + self.retry[test] = 1 + + if errstr: + path = "%s/%s-%s.log" % ("/tmp", test.replace("/", "-"), + self.retry[test]) + failsafe(write_to_file, (path, decode(errstr),)) + self.error_logs.append(path) + + if self.retry[test] < MAX_ATTEMPTS: + self.retry[test] += 1 + Log.debug("retry test %s attempt %s" % (test, self.retry[test])) + self.tests.append(test) + else: + Log.debug("giveup attempt test %s" % test) + del self.retry[test] + self.error.append(test) + + def note_retry(self, test): + Log.cli("retry %s on another host" % test) + self.pending.remove(test) + self.tests.append(test) + + # + # test classifications + # + def _all(self): + return self._list_tests(["tests"], recursive=True) + + def _not_flaky(self, tests): + for t in self.flaky_tests: + if t in tests: + tests.remove(t) + return tests + + def _list_tests(self, prefixes, recursive=False, ignore_ifnotexist=False): + tests = [] + for prefix in prefixes: + real_path = "%s/%s" % (self.gluster_path, prefix) + if not os.path.exists(real_path) and ignore_ifnotexist: + continue + for f in os.listdir(real_path): + if os.path.isdir(real_path + "/" + f): + if recursive: + tests += self._list_tests([prefix + "/" + f], recursive) + else: + if re.match(r".*\.t$", f): + tests += [prefix + "/" + f] + return tests + + def _parse_hosts(self, hosts): + ret = [] + for h in args.hosts.split(" "): + ret.append((h, DEFAULT_PORT)) + Log.debug(ret) + return ret + +# .............................................................................. +# Roles entry point +# .............................................................................. + + +def run_as_server(args): + if not args.server_path: + Log.error("please provide server path") + return 1 + + server = TestServer(args.port, args.server_path) + server.init() + server.serve() + return 0 + + +def run_as_tester(args): + Log.header("GLUSTER TEST CLI") + + Log.debug("args = %s" % args) + + tests = TestRunner(args.gluster_path, args.hosts, args.n, args.tests, + args.flaky_tests, valgrind=args.valgrind, + asan=args.asan, asan_noleaks=args.asan_noleaks, + id_rsa=args.id_rsa, test_timeout=args.test_timeout) + result = tests.run() + tests.print_result() + return result + +# .............................................................................. +# main +# .............................................................................. + + +def main(args): + if args.v: + Log.LOGLEVEL = LogLevel.DEBUG + + if args.server and args.tester: + Log.error("Invalid arguments. More than one role specified") + sys.exit(1) + + if args.server: + sys.exit(run_as_server(args)) + elif args.tester: + sys.exit(run_as_tester(args)) + else: + Log.error("please specify a mode for CI") + parser.print_help() + sys.exit(1) + + +parser = argparse.ArgumentParser(description="Gluster CI") + +# server role +parser.add_argument("--server", help="start server", action="store_true") +parser.add_argument("--server_path", help="server scratch space", + default="/tmp/gluster-test") +parser.add_argument("--host", help="server address to listen", default="") +parser.add_argument("--port", help="server port to listen", + type=int, default=DEFAULT_PORT) +# test role +parser.add_argument("--tester", help="start tester", action="store_true") +parser.add_argument("--valgrind[=memcheck,drd]", + help="run tests with valgrind tool 'memcheck' or 'drd'", + default="no") +parser.add_argument("--asan", help="test with asan enabled", + action="store_true") +parser.add_argument("--asan-noleaks", help="test with asan but no mem leaks", + action="store_true") +parser.add_argument("--tests", help="all/flaky/list of tests", default=None) +parser.add_argument("--flaky_tests", help="list of flaky tests", default=None) +parser.add_argument("--n", help="max number of machines to use", type=int, + default=0) +parser.add_argument("--hosts", help="list of worker machines") +parser.add_argument("--gluster_path", help="gluster path to test", + default=os.getcwd()) +parser.add_argument("--id-rsa", help="private key to use for ssh", + default=None) +parser.add_argument("--test-timeout", + help="test timeout in sec (default 15min)", + default=TEST_TIMEOUT_S) +# general +parser.add_argument("-v", help="verbose", action="store_true") +parser.add_argument("--address_family", help="IPv6 or IPv4 to use", + default=ADDRESS_FAMILY) + +args = parser.parse_args() + +main(args) diff --git a/extras/distributed-testing/distributed-test.sh b/extras/distributed-testing/distributed-test.sh new file mode 100755 index 00000000000..8f1e0310f33 --- /dev/null +++ b/extras/distributed-testing/distributed-test.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +source ./extras/distributed-testing/distributed-test-env + +N=0 +TESTS='all' +FLAKY=$KNOWN_FLAKY_TESTS +BROKEN=$BROKEN_TESTS +TEST_TIMEOUT_S=900 +ADDRESS_FAMILY='IPv4' + +FLAGS="" + +function print_env { + echo "Settings:" + echo "N=$N" + echo -e "-------\nHOSTS\n$HOSTS\n-------" + echo -e "TESTS\n$TESTS\n-------" + echo -e "SKIP\n$FLAKY $BROKEN\n-------" + echo -e "TEST_TIMEOUT_S=$TEST_TIMEOUT_S s\n" +} + +function cleanup { + rm -f /tmp/test*.log +} + +function usage { + echo "Usage: $0 [-h or --help] [-v or --verbose] + [--all] [--flaky] [--smoke] [--broken] + [--valgrind] [--asan] [--asan-noleaks] + [--hosts <hosts>] [-n <parallelism>] + [--tests <tests>] + [--id-rsa <ssh private key>] + [--address_family <IPv4 or IPv6>] + " +} + +function parse_args () { + args=`getopt \ + -o hvn: \ + --long help,verbose,address_family:,valgrind,asan,asan-noleaks,all,\ +smoke,flaky,broken,hosts:,tests:,id-rsa:,test-timeout: \ + -n 'fb-remote-test.sh' -- "$@"` + + if [ $? != 0 ]; then + echo "Error parsing getopt" + exit 1 + fi + + eval set -- "$args" + + while true; do + case "$1" in + -h | --help) usage ; exit 1 ;; + -v | --verbose) FLAGS="$FLAGS -v" ; shift ;; + --address_family) ADDRESS_FAMILY=$2; shift 2 ;; + --valgrind) FLAGS="$FLAGS --valgrind" ; shift ;; + --asan-noleaks) FLAGS="$FLAGS --asan-noleaks"; shift ;; + --asan) FLAGS="$FLAGS --asan" ; shift ;; + --hosts) HOSTS=$2; shift 2 ;; + --tests) TESTS=$2; FLAKY= ; BROKEN= ; shift 2 ;; + --test-timeout) TEST_TIMEOUT_S=$2; shift 2 ;; + --all) TESTS='all' ; shift 1 ;; + --flaky) TESTS=$FLAKY; FLAKY= ; shift 1 ;; + --smoke) TESTS=$SMOKE_TESTS; shift 1 ;; + --broken) TESTS=$BROKEN_TESTS; FLAKY= ; BROKEN= ; shift 1 ;; + --id-rsa) FLAGS="$FLAGS --id-rsa $2" ; shift 2 ;; + -n) N=$2; shift 2 ;; + *) break ;; + esac + done + run_tests_args="$@" +} + +function main { + parse_args "$@" + + if [ -z "$HOSTS" ]; then + echo "Please provide hosts to run the tests in" + exit -1 + fi + + print_env + + cleanup + + "extras/distributed-testing/distributed-test-runner.py" $FLAGS --tester \ + --n "$N" --hosts "$HOSTS" --tests "$TESTS" \ + --flaky_tests "$FLAKY $BROKEN" --test-timeout "$TEST_TIMEOUT_S" \ + --address_family "$ADDRESS_FAMILY" + + exit $? +} + +main "$@" diff --git a/extras/ec-heal-script/README.md b/extras/ec-heal-script/README.md new file mode 100644 index 00000000000..aaefd6681f6 --- /dev/null +++ b/extras/ec-heal-script/README.md @@ -0,0 +1,69 @@ +# gluster-heal-scripts +Scripts to correct extended attributes of fragments of files to make them healble. + +Following are the guidelines/suggestions to use these scripts. + +1 - Passwordless ssh should be setup for all the nodes of the cluster. + +2 - Scripts should be executed from one of these nodes. + +3 - Make sure NO "IO" is going on for the files for which we are running +these two scripts. + +4 - There should be no heal going on for the file for which xattrs are being +set by correct_pending_heals.sh. Disable the self heal while running this script. + +5 - All the bricks of the volume should be UP to identify good and bad fragments +and to decide if an entry is healble or not. + +6 - If correct_pending_heals.sh is stopped in the middle while it was processing +healble entries, it is suggested to re-run gfid_needing_heal_parallel.sh to create +latest list of healble and non healble entries and "potential_heal" "can_not_heal" files. + +7 - Based on the number of entries, these files might take time to get and set the +stats and xattrs of entries. + +8 - A backup of the fragments will be taken on <brick path>/.glusterfs/correct_pending_heals + directory with a file name same as gfid. + +9 - Once the correctness of the file gets verified by user, these backup should be removed. + +10 - Make sure we have enough space on bricks to take these backups. + +11 - At the end this will create two files - + 1 - modified_and_backedup_files - Contains list of files which have been modified and should be healed. + 2 - can_not_heal - Contains list of files which can not be healed. + +12 - It is suggested that the integrity of the data of files, which were modified and healed, + should be checked by the user. + + +Usage: + +Following are the sequence of steps to use these scripts - + +1 - ./gfid_needing_heal_parallel.sh <volume name> + + Execute gfid_needing_heal_parallel.sh with volume name to create list of files which could + be healed and can not be healed. It creates "potential_heal" and "can_not_heal" files. + During execution, it also displays the list of files on consol with the verdict. + +2 - ./correct_pending_heals.sh + + Execute correct_pending_heals.sh without any argument. This script processes entries present + in "heal" file. It asks user to enter how many files we want to process in one attempt. + Once the count is provided, this script will fetch the entries one by one from "potential_heal" file and takes necessary action. + If at this point also a file can not be healed, it will be pushed to "can_not_heal" file. + If a file can be healed, this script will modify the xattrs of that file fragments and create an entry in "modified_and_backedup_files" file + +3 - At the end, all the entries of "potential_heal" will be processed and based on the processing only two files will be left. + + 1 - modified_and_backedup_files - Contains list of files which have been modified and should be healed. + 2 - can_not_heal - Contains list of files which can not be healed. + +Logs and other files - + +1 - modified_and_backedup_files - It contains all the files which could be healed and the location of backup of each fragments. +2 - can_not_heal - It contains all the files which can not be healed. +3 - potential_heal - List of files which could be healed and should be processed by "correct_pending_heals.sh" +4 - /var/log/glusterfs/ec-heal-script.log - It contains logs of both the files. diff --git a/extras/ec-heal-script/correct_pending_heals.sh b/extras/ec-heal-script/correct_pending_heals.sh new file mode 100755 index 00000000000..c9f19dd7c89 --- /dev/null +++ b/extras/ec-heal-script/correct_pending_heals.sh @@ -0,0 +1,415 @@ +#!/bin/bash +# Copyright (c) 2019-2020 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +# This script finally resets the xattrs of all the fragments of a file +# which can be healed as per gfid_needing_heal_parallel.sh. +# gfid_needing_heal_parallel.sh will produce two files, potential_heal and can_not_heal. +# This script takes potential_heal as input and resets xattrs of all the fragments +# of those files present in this file and which could be healed as per +# trusted.ec.size xattar of the file else it will place the entry in can_not_heal +# file. Those entries which must be healed will be place in must_heal file +# after setting xattrs so that user can track those files. + + +MOD_BACKUP_FILES="modified_and_backedup_files" +CAN_NOT_HEAL="can_not_heal" +LOG_DIR="/var/log/glusterfs" +LOG_FILE="$LOG_DIR/ec-heal-script.log" +LINE_SEP="===================================================" + +function heal_log() +{ + echo "$1" >> "$LOG_FILE" +} + +function desc () +{ + echo "" + echo "This script finally resets the xattrs of all the fragments of a file +which can be healed as per gfid_needing_heal_parallel.sh. +gfid_needing_heal_parallel.sh will produce two files, potential_heal and can_not_heal. +This script takes potential_heal as input and resets xattrs of all the fragments +of those files present in this file and which could be healed as per +trusted.ec.size xattar of the file else it will place the entry in can_not_heal +file. Those entries which must be healed will be place in must_heal file +after setting xattrs so that user can track those files." +} + +function _init () +{ + if [ $# -ne 0 ] + then + echo "usage: $0" + desc + exit 2 + fi + + if [ ! -f "potential_heal" ] + then + echo "Nothing to correct. File "potential_heal" does not exist" + echo "" + desc + exit 2 + fi +} + +function total_file_size_in_hex() +{ + local frag_size=$1 + local size=0 + local hex_size="" + + size=$((frag_size * 4)) + hex_size=$(printf '0x%016x' $size) + echo "$hex_size" +} + +function backup_file_fragment() +{ + local file_host=$1 + local file_entry=$2 + local gfid_actual_paths=$3 + local brick_root="" + local temp="" + local backup_dir="" + local cmd="" + local gfid="" + + brick_root=$(echo "$file_entry" | cut -d "#" -f 1) + temp=$(echo "$(basename "$BASH_SOURCE")" | cut -d '.' -f 1) + backup_dir=$(echo "${brick_root}/.glusterfs/${temp}") + file_entry=${file_entry//#} + + gfid=$(echo "${gfid_actual_paths}" | cut -d '|' -f 1 | cut -d '/' -f 5) + echo "${file_host}:${backup_dir}/${gfid}" >> "$MOD_BACKUP_FILES" + + cmd="mkdir -p ${backup_dir} && yes | cp -af ${file_entry} ${backup_dir}/${gfid} 2>/dev/null" + ssh -n "${file_host}" "${cmd}" +} + +function set_frag_xattr () +{ + local file_host=$1 + local file_entry=$2 + local good=$3 + local cmd1="" + local cmd2="" + local cmd="" + local version="0x00000000000000010000000000000001" + local dirty="0x00000000000000010000000000000001" + + if [[ $good -eq 0 ]] + then + version="0x00000000000000000000000000000000" + fi + + cmd1=" setfattr -n trusted.ec.version -v ${version} ${file_entry} &&" + cmd2=" setfattr -n trusted.ec.dirty -v ${dirty} ${file_entry}" + cmd=${cmd1}${cmd2} + ssh -n "${file_host}" "${cmd}" +} + +function set_version_dirty_xattr () +{ + local file_paths=$1 + local good=$2 + local gfid_actual_paths=$3 + local file_entry="" + local file_host="" + local bpath="" + + for bpath in ${file_paths//,/ } + do + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + backup_file_fragment "$file_host" "$file_entry" "$gfid_actual_paths" + file_entry=${file_entry//#} + set_frag_xattr "$file_host" "$file_entry" "$good" + done +} + +function match_size_xattr_quorum () +{ + local file_paths=$1 + local file_entry="" + local file_host="" + local cmd="" + local size_xattr="" + local bpath="" + declare -A xattr_count + + for bpath in ${file_paths//,/ } + do + size_xattr="" + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + file_entry=${file_entry//#} + + cmd="getfattr -n trusted.ec.size -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.size" | cut -d '=' -f 2" + size_xattr=$(ssh -n "${file_host}" "${cmd}") + if [[ -n $size_xattr ]] + then + count=$((xattr_count["$size_xattr"] + 1)) + xattr_count["$size_xattr"]=${count} + if [[ $count -ge 4 ]] + then + echo "${size_xattr}" + return + fi + fi + done + echo "False" +} + +function match_version_xattr () +{ + local file_paths=$1 + local file_entry="" + local file_host="" + local cmd="" + local version="" + local bpath="" + declare -A ver_count + + for bpath in ${file_paths//,/ } + do + version="" + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + file_entry=${file_entry//#} + + cmd="getfattr -n trusted.ec.version -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.version" | cut -d '=' -f 2" + version=$(ssh -n "${file_host}" "${cmd}") + ver_count["$version"]=$((ver_count["$version"] + 1)) + done + for key in "${ver_count[@]}" + do + if [[ $key -ge 4 ]] + then + echo "True" + return + else + echo "False" + return + fi + done +} + +function match_stat_size_with_xattr () +{ + local bpath=$1 + local size=$2 + local file_stat=$3 + local xattr=$4 + local file_entry="" + local file_host="" + local cmd="" + local stat_output="" + local hex_size="" + + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + + file_entry=${file_entry//#} + cmd="stat --format=%F:%B:%s $file_entry 2>/dev/null" + stat_output=$(ssh -n "${file_host}" "${cmd}") + echo "$stat_output" | grep -w "${file_stat}" > /dev/null + + if [[ $? -eq 0 ]] + then + cmd="getfattr -n trusted.ec.size -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.size" | cut -d '=' -f 2" + hex_size=$(ssh -n "${file_host}" "${cmd}") + + if [[ -z $hex_size || "$hex_size" != "$xattr" ]] + then + echo "False" + return + fi + size_diff=$(printf '%d' $(( size - hex_size ))) + if [[ $size_diff -gt 2047 ]] + then + echo "False" + return + else + echo "True" + return + fi + else + echo "False" + return + fi +} + +function find_file_paths () +{ + local bpath=$1 + local file_entry="" + local file_host="" + local cmd="" + local brick_root="" + local gfid="" + local actual_path="" + local gfid_path="" + + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + brick_root=$(echo "$file_entry" | cut -d "#" -f 1) + + gfid=$(echo "${file_entry}" | grep ".glusterfs") + if [[ -n "$gfid" ]] + then + gfid_path=$(echo "$file_entry" | cut -d "#" -f 2) + file_entry=${file_entry//#} + cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep -v '.glusterfs' " + actual_path=$(ssh -n "${file_host}" "${cmd}") + #removing absolute path so that user can refer this from mount point + actual_path=${actual_path#"$brick_root"} + else + actual_path=$(echo "$file_entry" | cut -d "#" -f 2) + file_entry=${file_entry//#} + cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep '.glusterfs' " + gfid_path=$(ssh -n "${file_host}" "${cmd}") + gfid_path=${gfid_path#"$brick_root"} + fi + + echo "${gfid_path}|${actual_path}" +} + +function log_can_not_heal () +{ + local gfid_actual_paths=$1 + local file_paths=$2 + file_paths=${file_paths//#} + + echo "${LINE_SEP}" >> "$CAN_NOT_HEAL" + echo "Can Not Heal : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$CAN_NOT_HEAL" + for bpath in ${file_paths//,/ } + do + echo "${bpath}" >> "$CAN_NOT_HEAL" + done +} + +function check_all_frag_and_set_xattr () +{ + local file_paths=$1 + local total_size=$2 + local file_stat=$3 + local bpath="" + local healthy_count=0 + local match="False" + local matching_bricks="" + local bad_bricks="" + local gfid_actual_paths="" + + for bpath in ${file_paths//,/ } + do + if [[ -n "$gfid_actual_paths" ]] + then + break + fi + gfid_actual_paths=$(find_file_paths "$bpath") + done + + match=$(match_size_xattr_quorum "$file_paths") + +# echo "${match} : $bpath" >> "$MOD_BACKUP_FILES" + + if [[ "$match" != "False" ]] + then + xattr="$match" + for bpath in ${file_paths//,/ } + do + match="False" + match=$(match_stat_size_with_xattr "$bpath" "$total_size" "$file_stat" "$xattr") + if [[ "$match" == "True" ]] + then + matching_bricks="${bpath},${matching_bricks}" + healthy_count=$((healthy_count + 1)) + else + bad_bricks="${bpath},${bad_bricks}" + fi + done + fi + + if [[ $healthy_count -ge 4 ]] + then + match="True" + echo "${LINE_SEP}" >> "$MOD_BACKUP_FILES" + echo "Modified : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$MOD_BACKUP_FILES" + set_version_dirty_xattr "$matching_bricks" 1 "$gfid_actual_paths" + set_version_dirty_xattr "$bad_bricks" 0 "$gfid_actual_paths" + else + log_can_not_heal "$gfid_actual_paths" "${file_paths}" + fi + + echo "$match" +} +function set_xattr() +{ + local count=$1 + local heal_entry="" + local file_stat="" + local frag_size="" + local total_size="" + local file_paths="" + local num="" + local can_heal_count=0 + + heal_log "Started $(basename $BASH_SOURCE) on $(date) " + + while read -r heal_entry + do + heal_log "$LINE_SEP" + heal_log "${heal_entry}" + + file_stat=$(echo "$heal_entry" | cut -d "|" -f 1) + frag_size=$(echo "$file_stat" | rev | cut -d ":" -f 1 | rev) + total_size="$(total_file_size_in_hex "$frag_size")" + file_paths=$(echo "$heal_entry" | cut -d "|" -f 2) + match=$(check_all_frag_and_set_xattr "$file_paths" "$total_size" "$file_stat") + if [[ "$match" == "True" ]] + then + can_heal_count=$((can_heal_count + 1)) + fi + + sed -i '1d' potential_heal + count=$((count - 1)) + if [ $count == 0 ] + then + num=$(cat potential_heal | wc -l) + heal_log "$LINE_SEP" + heal_log "${1} : Processed" + heal_log "${can_heal_count} : Modified to Heal" + heal_log "$((${1} - can_heal_count)) : Moved to can_not_heal." + heal_log "${num} : Pending as Potential Heal" + exit 0 + fi + + done < potential_heal +} + +function main () +{ + local count=0 + + read -p "Number of files to correct: [choose between 1-1000] (0 for All):" count + if [[ $count -lt 0 || $count -gt 1000 ]] + then + echo "Provide correct value:" + exit 2 + fi + + if [[ $count -eq 0 ]] + then + count=$(cat potential_heal | wc -l) + fi + set_xattr "$count" +} + +_init "$@" && main "$@" diff --git a/extras/ec-heal-script/gfid_needing_heal_parallel.sh b/extras/ec-heal-script/gfid_needing_heal_parallel.sh new file mode 100755 index 00000000000..d7f53c97c33 --- /dev/null +++ b/extras/ec-heal-script/gfid_needing_heal_parallel.sh @@ -0,0 +1,278 @@ +#!/bin/bash +# Copyright (c) 2019-2020 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +# This script provides a list of all the files which can be healed or not healed. +# It also generates two files, potential_heal and can_not_heal, which contains the information +# of all theose files. These files could be used by correct_pending_heals.sh to correct +# the fragmnets so that files could be healed by shd. + +CAN_NOT_HEAL="can_not_heal" +CAN_HEAL="potential_heal" +LINE_SEP="===================================================" +LOG_DIR="/var/log/glusterfs" +LOG_FILE="$LOG_DIR/ec-heal-script.log" + +function heal_log() +{ + echo "$1" >> "$LOG_FILE" +} + +function _init () +{ + if [ $# -ne 1 ]; then + echo "usage: $0 <gluster volume name>"; + echo "This script provides a list of all the files which can be healed or not healed. +It also generates two files, potential_heal and can_not_heal, which contains the information +of all theose files. These files could be used by correct_pending_heals.sh to correct +the fragmnets so that files could be healed by shd." + exit 2; + fi + + volume=$1; +} + +function get_pending_entries () +{ + local volume_name=$1 + + gluster volume heal "$volume_name" info | grep -v ":/" | grep -v "Number of entries" | grep -v "Status:" | sort -u | sed '/^$/d' +} + +function get_entry_path_on_brick() +{ + local path="$1" + local gfid_string="" + if [[ "${path:0:1}" == "/" ]]; + then + echo "$path" + else + gfid_string="$(echo "$path" | cut -f2 -d':' | cut -f1 -d '>')" + echo "/.glusterfs/${gfid_string:0:2}/${gfid_string:2:2}/$gfid_string" + fi +} + +function run_command_on_server() +{ + local subvolume="$1" + local host="$2" + local cmd="$3" + local output + output=$(ssh -n "${host}" "${cmd}") + if [ -n "$output" ] + then + echo "$subvolume:$output" + fi +} + +function get_entry_path_all_bricks () +{ + local entry="$1" + local bricks="$2" + local cmd="" + for brick in $bricks + do + echo "${brick}#$(get_entry_path_on_brick "$entry")" + done | tr '\n' ',' +} + +function get_stat_for_entry_from_all_bricks () +{ + local entry="$1" + local bricks="$2" + local subvolume=0 + local host="" + local bpath="" + local cmd="" + + for brick in $bricks + do + if [[ "$((subvolume % 6))" == "0" ]] + then + subvolume=$((subvolume+1)) + fi + host=$(echo "$brick" | cut -f1 -d':') + bpath=$(echo "$brick" | cut -f2 -d':') + + cmd="stat --format=%F:%B:%s $bpath$(get_entry_path_on_brick "$entry") 2>/dev/null" + run_command_on_server "$subvolume" "${host}" "${cmd}" & + done | sort | uniq -c | sort -rnk1 +} + +function get_bricks_from_volume() +{ + local v=$1 + gluster volume info "$v" | grep -E "^Brick[0-9][0-9]*:" | cut -f2- -d':' +} + +function print_entry_gfid() +{ + local host="$1" + local dirpath="$2" + local entry="$3" + local gfid + gfid="$(ssh -n "${host}" "getfattr -d -m. -e hex $dirpath/$entry 2>/dev/null | grep trusted.gfid=|cut -f2 -d'='")" + echo "$entry" - "$gfid" +} + +function print_brick_directory_info() +{ + local h="$1" + local dirpath="$2" + while read -r e + do + print_entry_gfid "${h}" "${dirpath}" "${e}" + done < <(ssh -n "${h}" "ls $dirpath 2>/dev/null") +} + +function print_directory_info() +{ + local entry="$1" + local bricks="$2" + local h + local b + local gfid + for brick in $bricks; + do + h="$(echo "$brick" | cut -f1 -d':')" + b="$(echo "$brick" | cut -f2 -d':')" + dirpath="$b$(get_entry_path_on_brick "$entry")" + print_brick_directory_info "${h}" "${dirpath}" & + done | sort | uniq -c +} + +function print_entries_needing_heal() +{ + local quorum=0 + local entry="$1" + local bricks="$2" + while read -r line + do + quorum=$(echo "$line" | awk '{print $1}') + if [[ "$quorum" -lt 4 ]] + then + echo "$line - Not in Quorum" + else + echo "$line - In Quorum" + fi + done < <(print_directory_info "$entry" "$bricks") +} + +function find_file_paths () +{ + local bpath=$1 + local file_entry="" + local file_host="" + local cmd="" + local brick_root="" + local gfid="" + local actual_path="" + local gfid_path="" + + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + brick_root=$(echo "$file_entry" | cut -d "#" -f 1) + + gfid=$(echo "${file_entry}" | grep ".glusterfs") + + if [[ -n "$gfid" ]] + then + gfid_path=$(echo "$file_entry" | cut -d "#" -f 2) + file_entry=${file_entry//#} + cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep -v '.glusterfs' " + actual_path=$(ssh -n "${file_host}" "${cmd}") + #removing absolute path so that user can refer this from mount point + actual_path=${actual_path#"$brick_root"} + else + actual_path=$(echo "$file_entry" | cut -d "#" -f 2) + file_entry=${file_entry//#} + cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep '.glusterfs' " + gfid_path=$(ssh -n "${file_host}" "${cmd}") + gfid_path=${gfid_path#"$brick_root"} + fi + + echo "${gfid_path}|${actual_path}" +} + +function log_can_not_heal () +{ + local gfid_actual_paths=$1 + local file_paths=$2 + file_paths=${file_paths//#} + + echo "${LINE_SEP}" >> "$CAN_NOT_HEAL" + echo "Can Not Heal : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$CAN_NOT_HEAL" + for bpath in ${file_paths//,/ } + do + echo "${bpath}" >> "$CAN_NOT_HEAL" + done +} + +function main () +{ + local bricks="" + local quorum=0 + local stat_info="" + local file_type="" + local gfid_actual_paths="" + local bpath="" + local file_paths="" + local good=0 + local bad=0 + bricks=$(get_bricks_from_volume "$volume") + rm -f "$CAN_HEAL" + rm -f "$CAN_NOT_HEAL" + mkdir "$LOG_DIR" -p + + heal_log "Started $(basename "$BASH_SOURCE") on $(date) " + while read -r heal_entry + do + heal_log "------------------------------------------------------------------" + heal_log "$heal_entry" + + gfid_actual_paths="" + file_paths="$(get_entry_path_all_bricks "$heal_entry" "$bricks")" + stat_info="$(get_stat_for_entry_from_all_bricks "$heal_entry" "$bricks")" + heal_log "$stat_info" + + quorum=$(echo "$stat_info" | head -1 | awk '{print $1}') + good_stat=$(echo "$stat_info" | head -1 | awk '{print $3}') + file_type="$(echo "$stat_info" | head -1 | cut -f2 -d':')" + if [[ "$file_type" == "directory" ]] + then + print_entries_needing_heal "$heal_entry" "$bricks" + else + if [[ "$quorum" -ge 4 ]] + then + good=$((good + 1)) + heal_log "Verdict: Healable" + + echo "${good_stat}|$file_paths" >> "$CAN_HEAL" + else + bad=$((bad + 1)) + heal_log "Verdict: Not Healable" + for bpath in ${file_paths//,/ } + do + if [[ -z "$gfid_actual_paths" ]] + then + gfid_actual_paths=$(find_file_paths "$bpath") + else + break + fi + done + log_can_not_heal "$gfid_actual_paths" "${file_paths}" + fi + fi + done < <(get_pending_entries "$volume") + heal_log "=========================================" + heal_log "Total number of potential heal : ${good}" + heal_log "Total number of can not heal : ${bad}" + heal_log "=========================================" +} + +_init "$@" && main "$@" diff --git a/extras/failed-tests.py b/extras/failed-tests.py new file mode 100755 index 00000000000..f7f110246b5 --- /dev/null +++ b/extras/failed-tests.py @@ -0,0 +1,180 @@ +#!/usr/bin/python3 + +from __future__ import print_function +import blessings +import requests +from requests.packages.urllib3.exceptions import InsecureRequestWarning +import re +import argparse +from collections import defaultdict +from datetime import timedelta, datetime +from pystache import render + +# This tool goes though the Gluster regression links and checks for failures + +BASE = 'https://build.gluster.org' +TERM = blessings.Terminal() +MAX_BUILDS = 1000 +summary = defaultdict(list) +VERBOSE = None + + +def process_failure(url, node): + text = requests.get(url, verify=False).text + accum = [] + for t in text.split('\n'): + if t.find("Result: FAIL") != -1: + for t2 in accum: + if VERBOSE: + print(t2.encode('utf-8')) + if t2.find("Wstat") != -1: + test_case = re.search('\./tests/.*\.t', t2) + if test_case: + summary[test_case.group()].append((url, node)) + accum = [] + elif t.find("cur_cores=/") != -1: + summary["core"].append([t.split("/")[1]]) + summary["core"].append(url) + else: + accum.append(t) + + +def print_summary(failed_builds, total_builds, html=False): + # All the templates + count = [ + '{{failed}} of {{total}} regressions failed', + '<p><b>{{failed}}</b> of <b>{{total}}</b> regressions failed</p>' + ] + regression_link = [ + '\tRegression Link: {{link}}\n' + '\tNode: {{node}}', + '<p> Regression Link: {{link}}</p>' + '<p> Node: {{node}}</p>' + ] + component = [ + '\tComponent: {{comp}}', + '<p> Component: {{comp}}</p>' + ] + failure_count = [ + ''.join([ + TERM.red, + '{{test}} ; Failed {{count}} times', + TERM.normal + ]), + ( + '<p><font color="red"><b>{{test}};</b> Failed <b>{{count}}' + '</b> times</font></p>' + ) + ] + + template = 0 + if html: + template = 1 + print(render( + count[template], + {'failed': failed_builds, 'total': total_builds} + )) + for k, v in summary.items(): + if k == 'core': + print(''.join([TERM.red, "Found cores:", TERM.normal])) + for comp, link in zip(v[::2], v[1::2]): + print(render(component[template], {'comp': comp})) + print(render( + regression_link[template], + {'link': link[0], 'node': link[1]} + )) + else: + print(render(failure_count[template], {'test': k, 'count': len(v)})) + for link in v: + print(render( + regression_link[template], + {'link': link[0], 'node': link[1]} + )) + + +def get_summary(cut_off_date, reg_link): + ''' + Get links to the failed jobs + ''' + success_count = 0 + failure_count = 0 + for page in range(0, MAX_BUILDS, 100): + build_info = requests.get(''.join([ + BASE, + reg_link, + 'api/json?depth=1&tree=allBuilds' + '[url,result,timestamp,builtOn]', + '{{{0},{1}}}'.format(page, page+100) + ]), verify=False).json() + for build in build_info.get('allBuilds'): + if datetime.fromtimestamp(build['timestamp']/1000) < cut_off_date: + # stop when timestamp older than cut off date + return failure_count, failure_count + success_count + if build['result'] in [None, 'SUCCESS']: + # pass when build is a success or ongoing + success_count += 1 + continue + if VERBOSE: + print(''.join([ + TERM.red, + 'FAILURE on {0}'.format(build['url']), + TERM.normal + ])) + url = ''.join([build['url'], 'consoleText']) + failure_count += 1 + process_failure(url, build['builtOn']) + return failure_count, failure_count + success_count + + +def main(num_days, regression_link, html_report): + cut_off_date = datetime.today() - timedelta(days=num_days) + failure = 0 + total = 0 + for reg in regression_link: + if reg == 'centos': + reg_link = '/job/centos6-regression/' + elif reg == 'netbsd': + reg_link = '/job/netbsd7-regression/' + else: + reg_link = reg + counts = get_summary(cut_off_date, reg_link) + failure += counts[0] + total += counts[1] + print_summary(failure, total, html_report) + + +if __name__ == '__main__': + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + parser = argparse.ArgumentParser() + parser.add_argument("get-summary") + parser.add_argument( + "last_no_of_days", + default=1, + type=int, + help="Regression summary of last number of days" + ) + parser.add_argument( + "regression_link", + default="centos", + nargs='+', + help="\"centos\" | \"netbsd\" | any other regression link" + ) + parser.add_argument( + "--verbose", + default=False, + action="store_true", + help="Print a detailed report of each test case that is failed" + ) + parser.add_argument( + "--html-report", + default=False, + action="store_true", + help="Print a brief report of failed regressions in html format" + ) + args = parser.parse_args() + VERBOSE = args.verbose + main( + num_days=args.last_no_of_days, + regression_link=args.regression_link, + html_report=args.html_report + ) diff --git a/extras/file_size_contri.sh b/extras/file_size_contri.sh new file mode 100755 index 00000000000..4f52a9a89b4 --- /dev/null +++ b/extras/file_size_contri.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# This script checks whether the contribution and disk-usage of a file is same. + +CONTRIBUTION_HEX=`getfattr -h -e hex -d -m trusted.glusterfs.quota.*.contri $1 2>&1 | sed -e '/^#/d' | sed -e '/^getfattr/d' | sed -e '/^$/d' | cut -d'=' -f 2` + +BLOCKS=`stat -c %b $1` +SIZE=$(($BLOCKS * 512)) + +CONTRIBUTION=`printf "%d" $CONTRIBUTION_HEX` + +if [ $CONTRIBUTION -ne $SIZE ]; then + printf "contribution of %s:%d\n" $1 $CONTRIBUTION + echo "size of $1: $SIZE" + echo "===================================================" +fi + diff --git a/extras/firewalld/Makefile.am b/extras/firewalld/Makefile.am new file mode 100644 index 00000000000..530881fb8eb --- /dev/null +++ b/extras/firewalld/Makefile.am @@ -0,0 +1,8 @@ +EXTRA_DIST = glusterfs.xml + +if USE_FIREWALLD +if WITH_SERVER +staticdir = /usr/lib/firewalld/services/ +static_DATA = glusterfs.xml +endif +endif diff --git a/extras/firewalld/glusterfs.xml b/extras/firewalld/glusterfs.xml new file mode 100644 index 00000000000..7e176442f5b --- /dev/null +++ b/extras/firewalld/glusterfs.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="utf-8"?> +<service> +<short>glusterfs-static</short> +<description>Default ports for gluster-distributed storage</description> +<port protocol="tcp" port="24007"/> <!--For glusterd --> +<port protocol="tcp" port="24008"/> <!--For glusterd RDMA port management --> +<port protocol="tcp" port="24009"/> <!--For glustereventsd --> +<port protocol="tcp" port="38465"/> <!--Gluster NFS service --> +<port protocol="tcp" port="38466"/> <!--Gluster NFS service --> +<port protocol="tcp" port="38467"/> <!--Gluster NFS service --> +<port protocol="tcp" port="38468"/> <!--Gluster NFS service --> +<port protocol="tcp" port="38469"/> <!--Gluster NFS service --> +<port protocol="tcp" port="49152-49664"/> <!--512 ports for bricks --> +</service> diff --git a/extras/ganesha/Makefile.am b/extras/ganesha/Makefile.am new file mode 100644 index 00000000000..9eaa401b6c8 --- /dev/null +++ b/extras/ganesha/Makefile.am @@ -0,0 +1,2 @@ +SUBDIRS = scripts config ocf +CLEANFILES = diff --git a/extras/ganesha/config/Makefile.am b/extras/ganesha/config/Makefile.am new file mode 100644 index 00000000000..c729273096e --- /dev/null +++ b/extras/ganesha/config/Makefile.am @@ -0,0 +1,4 @@ +EXTRA_DIST= ganesha-ha.conf.sample + +confdir = $(sysconfdir)/ganesha +conf_DATA = ganesha-ha.conf.sample diff --git a/extras/ganesha/config/ganesha-ha.conf.sample b/extras/ganesha/config/ganesha-ha.conf.sample new file mode 100644 index 00000000000..c22892bde56 --- /dev/null +++ b/extras/ganesha/config/ganesha-ha.conf.sample @@ -0,0 +1,19 @@ +# Name of the HA cluster created. +# must be unique within the subnet +HA_NAME="ganesha-ha-360" +# +# N.B. you may use short names or long names; you may not use IP addrs. +# Once you select one, stay with it as it will be mildly unpleasant to +# clean up if you switch later on. Ensure that all names - short and/or +# long - are in DNS or /etc/hosts on all machines in the cluster. +# +# The subset of nodes of the Gluster Trusted Pool that form the ganesha +# HA cluster. Hostname is specified. +HA_CLUSTER_NODES="server1,server2,..." +#HA_CLUSTER_NODES="server1.lab.redhat.com,server2.lab.redhat.com,..." +# +# Virtual IPs for each of the nodes specified above. +VIP_server1="10.0.2.1" +VIP_server2="10.0.2.2" +#VIP_server1_lab_redhat_com="10.0.2.1" +#VIP_server2_lab_redhat_com="10.0.2.2" diff --git a/extras/ganesha/ocf/Makefile.am b/extras/ganesha/ocf/Makefile.am new file mode 100644 index 00000000000..990a609f254 --- /dev/null +++ b/extras/ganesha/ocf/Makefile.am @@ -0,0 +1,11 @@ +EXTRA_DIST= ganesha_grace ganesha_mon ganesha_nfsd + +# The root of the OCF resource agent hierarchy +# Per the OCF standard, it's always "lib", +# not "lib64" (even on 64-bit platforms). +ocfdir = $(prefix)/lib/ocf + +# The provider directory +radir = $(ocfdir)/resource.d/heartbeat + +ra_SCRIPTS = ganesha_grace ganesha_mon ganesha_nfsd diff --git a/extras/ganesha/ocf/ganesha_grace b/extras/ganesha/ocf/ganesha_grace new file mode 100644 index 00000000000..825f7164597 --- /dev/null +++ b/extras/ganesha/ocf/ganesha_grace @@ -0,0 +1,221 @@ +#!/bin/bash +# +# Copyright (c) 2014 Anand Subramanian anands@redhat.com +# Copyright (c) 2015 Red Hat Inc. +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# +# + +# Initialization: +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +if [ -n "$OCF_DEBUG_LIBRARY" ]; then + . $OCF_DEBUG_LIBRARY +else + : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} + . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs +fi + +OCF_RESKEY_grace_active_default="grace-active" +: ${OCF_RESKEY_grace_active=${OCF_RESKEY_grace_active_default}} + +ganesha_meta_data() { + cat <<END +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="ganesha_grace"> +<version>1.0</version> + +<longdesc lang="en"> +This Linux-specific resource agent acts as a dummy +resource agent for nfs-ganesha. +</longdesc> + +<shortdesc lang="en">Manages the user-space nfs-ganesha NFS server</shortdesc> + +<parameters> +<parameter name="grace_active"> +<longdesc lang="en">NFS-Ganesha grace active attribute</longdesc> +<shortdesc lang="en">NFS-Ganesha grace active attribute</shortdesc> +<content type="string" default="grace-active" /> +</parameter> +</parameters> + +<actions> +<action name="start" timeout="40s" /> +<action name="stop" timeout="40s" /> +<action name="status" timeout="20s" interval="60s" /> +<action name="monitor" depth="0" timeout="10s" interval="5s" /> +<action name="notify" timeout="10s" /> +<action name="meta-data" timeout="20s" /> +</actions> +</resource-agent> +END + +return ${OCF_SUCCESS} +} + +ganesha_grace_usage() { + echo "ganesha.nfsd USAGE" +} + +# Make sure meta-data and usage always succeed +case $__OCF_ACTION in + meta-data) ganesha_meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) ganesha_usage + exit ${OCF_SUCCESS} + ;; + *) + ;; +esac + +ganesha_grace_start() +{ + local rc=${OCF_ERR_GENERIC} + local host=$(hostname -s) + + ocf_log debug "ganesha_grace_start()" + # give ganesha_mon RA a chance to set the crm_attr first + # I mislike the sleep, but it's not clear that looping + # with a small sleep is necessarily better + # start has a 40sec timeout, so a 5sec sleep here is okay + sleep 5 + attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null) + if [ $? -ne 0 ]; then + host=$(hostname) + attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null ) + if [ $? -ne 0 ]; then + ocf_log info "grace start: crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} failed" + fi + fi + + # Three possibilities: + # 1. There is no attribute at all and attr_updater returns + # a zero length string. This happens when + # ganesha_mon::monitor hasn't run at least once to set + # the attribute. The assumption here is that the system + # is coming up. We pretend, for now, that the node is + # healthy, to allow the system to continue coming up. + # It will cure itself in a few seconds + # 2. There is an attribute, and it has the value "1"; this + # node is healthy. + # 3. There is an attribute, but it has no value or the value + # "0"; this node is not healthy. + + # case 1 + if [[ -z "${attr}" ]]; then + return ${OCF_SUCCESS} + fi + + # case 2 + if [[ "${attr}" = *"value=1" ]]; then + return ${OCF_SUCCESS} + fi + + # case 3 + return ${OCF_NOT_RUNNING} +} + +ganesha_grace_stop() +{ + + ocf_log debug "ganesha_grace_stop()" + return ${OCF_SUCCESS} +} + +ganesha_grace_notify() +{ + # since this is a clone RA we should only ever see pre-start + # or post-stop + mode="${OCF_RESKEY_CRM_meta_notify_type}-${OCF_RESKEY_CRM_meta_notify_operation}" + case "${mode}" in + pre-start | post-stop) + dbus-send --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.grace string:${OCF_RESKEY_CRM_meta_notify_stop_uname} + if [ $? -ne 0 ]; then + ocf_log info "dbus-send --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.grace string:${OCF_RESKEY_CRM_meta_notify_stop_uname} failed" + fi + ;; + esac + + return ${OCF_SUCCESS} +} + +ganesha_grace_monitor() +{ + local host=$(hostname -s) + + ocf_log debug "monitor" + + attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null) + if [ $? -ne 0 ]; then + host=$(hostname) + attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null) + if [ $? -ne 0 ]; then + ocf_log info "crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} failed" + fi + fi + + # if there is no attribute (yet), maybe it's because + # this RA started before ganesha_mon (nfs-mon) has had + # chance to create it. In which case we'll pretend + # everything is okay this time around + if [[ -z "${attr}" ]]; then + return ${OCF_SUCCESS} + fi + + if [[ "${attr}" = *"value=1" ]]; then + return ${OCF_SUCCESS} + fi + + return ${OCF_NOT_RUNNING} +} + +ganesha_grace_validate() +{ + return ${OCF_SUCCESS} +} + +ganesha_grace_validate + +# Translate each action into the appropriate function call +case $__OCF_ACTION in +start) ganesha_grace_start + ;; +stop) ganesha_grace_stop + ;; +status|monitor) ganesha_grace_monitor + ;; +notify) ganesha_grace_notify + ;; +*) ganesha_grace_usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac + +rc=$? + +# The resource agent may optionally log a debug message +ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc" +exit $rc diff --git a/extras/ganesha/ocf/ganesha_mon b/extras/ganesha/ocf/ganesha_mon new file mode 100644 index 00000000000..2b4a9d6da84 --- /dev/null +++ b/extras/ganesha/ocf/ganesha_mon @@ -0,0 +1,234 @@ +#!/bin/bash +# +# Copyright (c) 2014 Anand Subramanian anands@redhat.com +# Copyright (c) 2015 Red Hat Inc. +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# +# + +# Initialization: +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +if [ -n "${OCF_DEBUG_LIBRARY}" ]; then + . ${OCF_DEBUG_LIBRARY} +else + : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} + . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs +fi + +# Defaults +OCF_RESKEY_ganesha_active_default="ganesha-active" +OCF_RESKEY_grace_active_default="grace-active" +OCF_RESKEY_grace_delay_default="5" + +: ${OCF_RESKEY_ganesha_active=${OCF_RESKEY_ganesha_active_default}} +: ${OCF_RESKEY_grace_active=${OCF_RESKEY_grace_active_default}} +: ${OCF_RESKEY_grace_delay=${OCF_RESKEY_grace_delay_default}} + +ganesha_meta_data() { + cat <<END +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="ganesha_mon"> +<version>1.0</version> + +<longdesc lang="en"> +This Linux-specific resource agent acts as a dummy +resource agent for nfs-ganesha. +</longdesc> + +<shortdesc lang="en">Manages the user-space nfs-ganesha NFS server</shortdesc> + +<parameters> +<parameter name="ganesha_active"> +<longdesc lang="en">NFS-Ganesha daemon active attribute</longdesc> +<shortdesc lang="en">NFS-Ganesha daemon active attribute</shortdesc> +<content type="string" default="ganesha-active" /> +</parameter> +<parameter name="grace_active"> +<longdesc lang="en">NFS-Ganesha grace active attribute</longdesc> +<shortdesc lang="en">NFS-Ganesha grace active attribute</shortdesc> +<content type="string" default="grace-active" /> +</parameter> +<parameter name="grace_delay"> +<longdesc lang="en"> +NFS-Ganesha grace delay. +When changing this, adjust the ganesha_grace RA's monitor interval to match. +</longdesc> +<shortdesc lang="en">NFS-Ganesha grace delay</shortdesc> +<content type="string" default="5" /> +</parameter> +</parameters> + +<actions> +<action name="start" timeout="40s" /> +<action name="stop" timeout="40s" /> +<action name="status" timeout="20s" interval="60s" /> +<action name="monitor" depth="0" timeout="10s" interval="10s" /> +<action name="meta-data" timeout="20s" /> +</actions> +</resource-agent> +END + +return ${OCF_SUCCESS} +} + +ganesha_mon_usage() { + echo "ganesha.nfsd USAGE" +} + +# Make sure meta-data and usage always succeed +case ${__OCF_ACTION} in + meta-data) ganesha_meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) ganesha_usage + exit ${OCF_SUCCESS} + ;; + *) + ;; +esac + +ganesha_mon_start() +{ + ocf_log debug "ganesha_mon_start" + ganesha_mon_monitor + return $OCF_SUCCESS +} + +ganesha_mon_stop() +{ + ocf_log debug "ganesha_mon_stop" + return $OCF_SUCCESS +} + +ganesha_mon_monitor() +{ + local host=$(hostname -s) + local pid_file="/var/run/ganesha.pid" + local rhel6_pid_file="/var/run/ganesha.nfsd.pid" + local proc_pid="/proc/" + + # RHEL6 /etc/init.d/nfs-ganesha adds -p /var/run/ganesha.nfsd.pid + # RHEL7 systemd does not. Would be nice if all distros used the + # same pid file. + if [ -e ${rhel6_pid_file} ]; then + pid_file=${rhel6_pid_file} + fi + if [ -e ${pid_file} ]; then + proc_pid="${proc_pid}$(cat ${pid_file})" + fi + + if [ "x${proc_pid}" != "x/proc/" -a -d ${proc_pid} ]; then + + attrd_updater -n ${OCF_RESKEY_ganesha_active} -v 1 + if [ $? -ne 0 ]; then + ocf_log info "warning: attrd_updater -n ${OCF_RESKEY_ganesha_active} -v 1 failed" + fi + + # ganesha_grace (nfs-grace) RA follows grace-active attr + # w/ constraint location + attrd_updater -n ${OCF_RESKEY_grace_active} -v 1 + if [ $? -ne 0 ]; then + ocf_log info "warning: attrd_updater -n ${OCF_RESKEY_grace_active} -v 1 failed" + fi + + # ganesha_mon (nfs-mon) and ganesha_grace (nfs-grace) + # track grace-active crm_attr (attr != crm_attr) + # we can't just use the attr as there's no way to query + # its value in RHEL6 pacemaker + + crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 2> /dev/null + if [ $? -ne 0 ]; then + host=$(hostname) + crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 2> /dev/null + if [ $? -ne 0 ]; then + ocf_log info "mon monitor warning: crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 failed" + fi + fi + + return ${OCF_SUCCESS} + fi + + # VIP fail-over is triggered by clearing the + # ganesha-active node attribute on this node. + # + # Meanwhile the ganesha_grace notify() runs when its + # nfs-grace resource is disabled on a node; which + # is triggered by clearing the grace-active attribute + # on this node. + # + # We need to allow time for it to run and put + # the remaining ganesha.nfsds into grace before + # initiating the VIP fail-over. + + attrd_updater -D -n ${OCF_RESKEY_grace_active} + if [ $? -ne 0 ]; then + ocf_log info "warning: attrd_updater -D -n ${OCF_RESKEY_grace_active} failed" + fi + + host=$(hostname -s) + crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 2> /dev/null + if [ $? -ne 0 ]; then + host=$(hostname) + crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 2> /dev/null + if [ $? -ne 0 ]; then + ocf_log info "mon monitor warning: crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 failed" + fi + fi + + sleep ${OCF_RESKEY_grace_delay} + + attrd_updater -D -n ${OCF_RESKEY_ganesha_active} + if [ $? -ne 0 ]; then + ocf_log info "warning: attrd_updater -D -n ${OCF_RESKEY_ganesha_active} failed" + fi + + return ${OCF_SUCCESS} +} + +ganesha_mon_validate() +{ + return ${OCF_SUCCESS} +} + +ganesha_mon_validate + +# Translate each action into the appropriate function call +case ${__OCF_ACTION} in +start) ganesha_mon_start + ;; +stop) ganesha_mon_stop + ;; +status|monitor) ganesha_mon_monitor + ;; +*) ganesha_mon_usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac + +rc=$? + +# The resource agent may optionally log a debug message +ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc" +exit $rc diff --git a/extras/ganesha/ocf/ganesha_nfsd b/extras/ganesha/ocf/ganesha_nfsd new file mode 100644 index 00000000000..f91e8b6b8f7 --- /dev/null +++ b/extras/ganesha/ocf/ganesha_nfsd @@ -0,0 +1,167 @@ +#!/bin/bash +# +# Copyright (c) 2014 Anand Subramanian anands@redhat.com +# Copyright (c) 2015 Red Hat Inc. +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# +# + +# Initialization: +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +if [ -n "${OCF_DEBUG_LIBRARY}" ]; then + . ${OCF_DEBUG_LIBRARY} +else + : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} + . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs +fi + +OCF_RESKEY_ha_vol_mnt_default="/run/gluster/shared_storage" +: ${OCF_RESKEY_ha_vol_mnt=${OCF_RESKEY_ha_vol_mnt_default}} + +ganesha_meta_data() { + cat <<END +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="ganesha_nfsd"> +<version>1.0</version> + +<longdesc lang="en"> +This Linux-specific resource agent acts as a dummy +resource agent for nfs-ganesha. +</longdesc> + +<shortdesc lang="en">Manages the user-space nfs-ganesha NFS server</shortdesc> + +<parameters> +<parameter name="ha_vol_mnt"> +<longdesc lang="en">HA State Volume Mount Point</longdesc> +<shortdesc lang="en">HA_State Volume Mount Point</shortdesc> +<content type="string" default="" /> +</parameter> +</parameters> + +<actions> +<action name="start" timeout="5s" /> +<action name="stop" timeout="5s" /> +<action name="status" depth="0" timeout="5s" interval="0" /> +<action name="monitor" depth="0" timeout="5s" interval="0" /> +<action name="meta-data" timeout="20s" /> +</actions> +</resource-agent> +END + +return ${OCF_SUCCESS} +} + +ganesha_nfsd_usage() { + echo "ganesha.nfsd USAGE" +} + +# Make sure meta-data and usage always succeed +case $__OCF_ACTION in + meta-data) ganesha_meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) ganesha_usage + exit ${OCF_SUCCESS} + ;; + *) + ;; +esac + +ganesha_nfsd_start() +{ + local long_host=$(hostname) + + if [[ -d /var/lib/nfs ]]; then + mv /var/lib/nfs /var/lib/nfs.backup + if [ $? -ne 0 ]; then + ocf_log notice "mv /var/lib/nfs /var/lib/nfs.backup failed" + fi + ln -s ${OCF_RESKEY_ha_vol_mnt}/nfs-ganesha/${long_host}/nfs /var/lib/nfs + if [ $? -ne 0 ]; then + ocf_log notice "ln -s ${OCF_RESKEY_ha_vol_mnt}/nfs-ganesha/${long_host}/nfs /var/lib/nfs failed" + fi + fi + + return ${OCF_SUCCESS} +} + +ganesha_nfsd_stop() +{ + + if [ -L /var/lib/nfs -a -d /var/lib/nfs.backup ]; then + rm -f /var/lib/nfs + if [ $? -ne 0 ]; then + ocf_log notice "rm -f /var/lib/nfs failed" + fi + mv /var/lib/nfs.backup /var/lib/nfs + if [ $? -ne 0 ]; then + ocf_log notice "mv /var/lib/nfs.backup /var/lib/nfs failed" + fi + fi + + return ${OCF_SUCCESS} +} + +ganesha_nfsd_monitor() +{ + # pacemaker checks to see if RA is already running before starting it. + # if we return success, then it's presumed it's already running and + # doesn't need to be started, i.e. invoke the start action. + # return something other than success to make pacemaker invoke the + # start action + if [[ -L /var/lib/nfs ]]; then + return ${OCF_SUCCESS} + fi + return ${OCF_NOT_RUNNING} +} + +ganesha_nfsd_validate() +{ + return ${OCF_SUCCESS} +} + +ganesha_nfsd_validate + +# ocf_log notice "ganesha_nfsd ${OCF_RESOURCE_INSTANCE} $__OCF_ACTION" + +# Translate each action into the appropriate function call +case $__OCF_ACTION in +start) ganesha_nfsd_start + ;; +stop) ganesha_nfsd_stop + ;; +status|monitor) ganesha_nfsd_monitor + ;; +*) ganesha_nfsd_usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac + +rc=$? + +# The resource agent may optionally log a debug message +ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc" +exit $rc diff --git a/extras/ganesha/scripts/Makefile.am b/extras/ganesha/scripts/Makefile.am new file mode 100644 index 00000000000..7e345fd5f19 --- /dev/null +++ b/extras/ganesha/scripts/Makefile.am @@ -0,0 +1,6 @@ +EXTRA_DIST= create-export-ganesha.sh generate-epoch.py dbus-send.sh \ + ganesha-ha.sh + +scriptsdir = $(libexecdir)/ganesha +scripts_SCRIPTS = create-export-ganesha.sh dbus-send.sh generate-epoch.py \ + ganesha-ha.sh diff --git a/extras/ganesha/scripts/create-export-ganesha.sh b/extras/ganesha/scripts/create-export-ganesha.sh new file mode 100755 index 00000000000..3040e8138b0 --- /dev/null +++ b/extras/ganesha/scripts/create-export-ganesha.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +#This script is called by glusterd when the user +#tries to export a volume via NFS-Ganesha. +#An export file specific to a volume +#is created in GANESHA_DIR/exports. + +# Try loading the config from any of the distro +# specific configuration locations +if [ -f /etc/sysconfig/ganesha ] + then + . /etc/sysconfig/ganesha +fi +if [ -f /etc/conf.d/ganesha ] + then + . /etc/conf.d/ganesha +fi +if [ -f /etc/default/ganesha ] + then + . /etc/default/ganesha +fi + +GANESHA_DIR=${1%/} +OPTION=$2 +VOL=$3 +CONF=$GANESHA_DIR"/ganesha.conf" +declare -i EXPORT_ID + +function check_cmd_status() +{ + if [ "$1" != "0" ] + then + rm -rf $GANESHA_DIR/exports/export.$VOL.conf + sed -i /$VOL.conf/d $CONF + exit 1 + fi +} + + +if [ ! -d "$GANESHA_DIR/exports" ]; + then + mkdir $GANESHA_DIR/exports + check_cmd_status `echo $?` +fi + +function write_conf() +{ +echo -e "# WARNING : Using Gluster CLI will overwrite manual +# changes made to this file. To avoid it, edit the +# file and run ganesha-ha.sh --refresh-config." + +echo "EXPORT{" +echo " Export_Id = 2;" +echo " Path = \"/$VOL\";" +echo " FSAL {" +echo " name = "GLUSTER";" +echo " hostname=\"localhost\";" +echo " volume=\"$VOL\";" +echo " }" +echo " Access_type = RW;" +echo " Disable_ACL = true;" +echo ' Squash="No_root_squash";' +echo " Pseudo=\"/$VOL\";" +echo ' Protocols = "3", "4" ;' +echo ' Transports = "UDP","TCP";' +echo ' SecType = "sys";' +echo ' Security_Label = False;' +echo " }" +} +if [ "$OPTION" = "on" ]; +then + if ! (cat $CONF | grep $VOL.conf\"$ ) + then + write_conf $@ > $GANESHA_DIR/exports/export.$VOL.conf + echo "%include \"$GANESHA_DIR/exports/export.$VOL.conf\"" >> $CONF + count=`ls -l $GANESHA_DIR/exports/*.conf | wc -l` + if [ "$count" = "1" ] ; then + EXPORT_ID=2 + else + EXPORT_ID=`cat $GANESHA_DIR/.export_added` + check_cmd_status `echo $?` + EXPORT_ID=EXPORT_ID+1 + sed -i s/Export_Id.*/"Export_Id= $EXPORT_ID ;"/ \ + $GANESHA_DIR/exports/export.$VOL.conf + check_cmd_status `echo $?` + fi + echo $EXPORT_ID > $GANESHA_DIR/.export_added + fi +else + rm -rf $GANESHA_DIR/exports/export.$VOL.conf + sed -i /$VOL.conf/d $CONF +fi diff --git a/extras/ganesha/scripts/dbus-send.sh b/extras/ganesha/scripts/dbus-send.sh new file mode 100755 index 00000000000..9d613a0e7ad --- /dev/null +++ b/extras/ganesha/scripts/dbus-send.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Try loading the config from any of the distro +# specific configuration locations +if [ -f /etc/sysconfig/ganesha ] + then + . /etc/sysconfig/ganesha +fi +if [ -f /etc/conf.d/ganesha ] + then + . /etc/conf.d/ganesha +fi +if [ -f /etc/default/ganesha ] + then + . /etc/default/ganesha +fi + +GANESHA_DIR=${1%/} +OPTION=$2 +VOL=$3 +CONF=$GANESHA_DIR"/ganesha.conf" + +function check_cmd_status() +{ + if [ "$1" != "0" ] + then + logger "dynamic export failed on node :${hostname -s}" + fi +} + +#This function keeps track of export IDs and increments it with every new entry +function dynamic_export_add() +{ + dbus-send --system \ +--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ +org.ganesha.nfsd.exportmgr.AddExport string:$GANESHA_DIR/exports/export.$VOL.conf \ +string:"EXPORT(Path=/$VOL)" + check_cmd_status `echo $?` +} + +#This function removes an export dynamically(uses the export_id of the export) +function dynamic_export_remove() +{ + # Below bash fetch all the export from ShowExport command and search + # export entry based on path and then get its export entry. + # There are two possiblities for path, either entire volume will be + # exported or subdir. It handles both cases. But it remove only first + # entry from the list based on assumption that entry exported via cli + # has lowest export id value + removed_id=$(dbus-send --type=method_call --print-reply --system \ + --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ + org.ganesha.nfsd.exportmgr.ShowExports | grep -B 1 -we \ + "/"$VOL -e "/"$VOL"/" | grep uint16 | awk '{print $2}' \ + | head -1) + + dbus-send --print-reply --system \ +--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ +org.ganesha.nfsd.exportmgr.RemoveExport uint16:$removed_id + check_cmd_status `echo $?` +} + +if [ "$OPTION" = "on" ]; +then + dynamic_export_add $@ +fi + +if [ "$OPTION" = "off" ]; +then + dynamic_export_remove $@ +fi diff --git a/extras/ganesha/scripts/ganesha-ha.sh b/extras/ganesha/scripts/ganesha-ha.sh new file mode 100644 index 00000000000..9790a719e10 --- /dev/null +++ b/extras/ganesha/scripts/ganesha-ha.sh @@ -0,0 +1,1199 @@ +#!/bin/bash + +# Copyright 2015-2016 Red Hat Inc. All Rights Reserved +# +# Pacemaker+Corosync High Availability for NFS-Ganesha +# +# setup, teardown, add, delete, refresh-config, and status +# +# Each participating node in the cluster is assigned a virtual IP (VIP) +# which fails over to another node when its associated ganesha.nfsd dies +# for any reason. After the VIP is moved to another node all the +# ganesha.nfsds are send a signal using DBUS to put them into NFS GRACE. +# +# There are six resource agent types used: ganesha_mon, ganesha_grace, +# ganesha_nfsd, IPaddr, and Dummy. ganesha_mon is used to monitor the +# ganesha.nfsd. ganesha_grace is used to send the DBUS signal to put +# the remaining ganesha.nfsds into grace. ganesha_nfsd is used to start +# and stop the ganesha.nfsd during setup and teardown. IPaddr manages +# the VIP. A Dummy resource named $hostname-trigger_ip-1 is used to +# ensure that the NFS GRACE DBUS signal is sent after the VIP moves to +# the new host. + +GANESHA_HA_SH=$(realpath $0) +HA_NUM_SERVERS=0 +HA_SERVERS="" +HA_VOL_NAME="gluster_shared_storage" +HA_VOL_MNT="/run/gluster/shared_storage" +HA_CONFDIR=$HA_VOL_MNT"/nfs-ganesha" +SERVICE_MAN="DISTRO_NOT_FOUND" + +# rhel, fedora id, version +ID="" +VERSION_ID="" + +PCS9OR10_PCS_CNAME_OPTION="" +PCS9OR10_PCS_CLONE_OPTION="clone" +SECRET_PEM="/var/lib/glusterd/nfs/secret.pem" + +# UNBLOCK RA uses shared_storage which may become unavailable +# during any of the nodes reboot. Hence increase timeout value. +PORTBLOCK_UNBLOCK_TIMEOUT="60s" + +# Try loading the config from any of the distro +# specific configuration locations +if [ -f /etc/sysconfig/ganesha ] + then + . /etc/sysconfig/ganesha +fi +if [ -f /etc/conf.d/ganesha ] + then + . /etc/conf.d/ganesha +fi +if [ -f /etc/default/ganesha ] + then + . /etc/default/ganesha +fi + +GANESHA_CONF= + +function find_rhel7_conf +{ + while [[ $# > 0 ]] + do + key="$1" + case $key in + -f) + CONFFILE="$2" + break; + ;; + *) + ;; + esac + shift + done +} + +if [ -z ${CONFFILE} ] + then + find_rhel7_conf ${OPTIONS} + +fi + +GANESHA_CONF=${CONFFILE:-/etc/ganesha/ganesha.conf} + +usage() { + + echo "Usage : add|delete|refresh-config|status" + echo "Add-node : ganesha-ha.sh --add <HA_CONF_DIR> \ +<NODE-HOSTNAME> <NODE-VIP>" + echo "Delete-node: ganesha-ha.sh --delete <HA_CONF_DIR> \ +<NODE-HOSTNAME>" + echo "Refresh-config : ganesha-ha.sh --refresh-config <HA_CONFDIR> \ +<volume>" + echo "Status : ganesha-ha.sh --status <HA_CONFDIR>" +} + +determine_service_manager () { + + if [ -e "/bin/systemctl" ]; + then + SERVICE_MAN="/bin/systemctl" + elif [ -e "/sbin/invoke-rc.d" ]; + then + SERVICE_MAN="/sbin/invoke-rc.d" + elif [ -e "/sbin/service" ]; + then + SERVICE_MAN="/sbin/service" + fi + if [[ "${SERVICE_MAN}X" == "DISTRO_NOT_FOUNDX" ]] + then + logger "Service manager not recognized, exiting" + exit 1 + fi +} + +manage_service () +{ + local action=${1} + local new_node=${2} + local option= + + if [[ "${action}" == "start" ]]; then + option="yes" + else + option="no" + fi + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${new_node} "${GANESHA_HA_SH} --setup-ganesha-conf-files $HA_CONFDIR $option" + + if [[ "${SERVICE_MAN}" == "/bin/systemctl" ]] + then + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${new_node} "${SERVICE_MAN} ${action} nfs-ganesha" + else + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${new_node} "${SERVICE_MAN} nfs-ganesha ${action}" + fi +} + + +check_cluster_exists() +{ + local name=${1} + local cluster_name="" + + if [ -e /var/run/corosync.pid ]; then + cluster_name=$(pcs status | grep "Cluster name:" | cut -d ' ' -f 3) + if [[ "${cluster_name}X" == "${name}X" ]]; then + logger "$name already exists, exiting" + exit 0 + fi + fi +} + + +determine_servers() +{ + local cmd=${1} + local num_servers=0 + local tmp_ifs=${IFS} + local ha_servers="" + + if [ "${cmd}X" != "setupX" -a "${cmd}X" != "statusX" ]; then + ha_servers=$(pcs status | grep "Online:" | grep -o '\[.*\]' | sed -e 's/\[//' | sed -e 's/\]//') + IFS=$' ' + for server in ${ha_servers} ; do + num_servers=$(expr ${num_servers} + 1) + done + IFS=${tmp_ifs} + HA_NUM_SERVERS=${num_servers} + HA_SERVERS="${ha_servers}" + else + IFS=$',' + for server in ${HA_CLUSTER_NODES} ; do + num_servers=$(expr ${num_servers} + 1) + done + IFS=${tmp_ifs} + HA_NUM_SERVERS=${num_servers} + HA_SERVERS="${HA_CLUSTER_NODES//,/ }" + fi +} + +stop_ganesha_all() +{ + local serverlist=${1} + for node in ${serverlist} ; do + manage_service "stop" ${node} + done +} + +setup_cluster() +{ + local name=${1} + local num_servers=${2} + local servers=${3} + local unclean="" + local quorum_policy="stop" + + logger "setting up cluster ${name} with the following ${servers}" + + # pcs cluster setup --force ${PCS9OR10_PCS_CNAME_OPTION} ${name} ${servers} + pcs cluster setup --force ${PCS9OR10_PCS_CNAME_OPTION} ${name} --enable ${servers} + if [ $? -ne 0 ]; then + logger "pcs cluster setup ${PCS9OR10_PCS_CNAME_OPTION} ${name} --enable ${servers} failed, shutting down ganesha and bailing out" + #set up failed stop all ganesha process and clean up symlinks in cluster + stop_ganesha_all "${servers}" + exit 1; + fi + + # pcs cluster auth ${servers} + pcs cluster auth + if [ $? -ne 0 ]; then + logger "pcs cluster auth failed" + fi + + pcs cluster start --all + if [ $? -ne 0 ]; then + logger "pcs cluster start failed" + exit 1; + fi + + sleep 1 + # wait for the cluster to elect a DC before querying or writing + # to the CIB. BZ 1334092 + crmadmin --dc_lookup --timeout=5000 > /dev/null 2>&1 + while [ $? -ne 0 ]; do + crmadmin --dc_lookup --timeout=5000 > /dev/null 2>&1 + done + + unclean=$(pcs status | grep -u "UNCLEAN") + while [[ "${unclean}X" == "UNCLEANX" ]]; do + sleep 1 + unclean=$(pcs status | grep -u "UNCLEAN") + done + sleep 1 + + if [ ${num_servers} -lt 3 ]; then + quorum_policy="ignore" + fi + pcs property set no-quorum-policy=${quorum_policy} + if [ $? -ne 0 ]; then + logger "warning: pcs property set no-quorum-policy=${quorum_policy} failed" + fi + + pcs property set stonith-enabled=false + if [ $? -ne 0 ]; then + logger "warning: pcs property set stonith-enabled=false failed" + fi +} + + +setup_finalize_ha() +{ + local cibfile=${1} + local stopped="" + + stopped=$(pcs status | grep -u "Stopped") + while [[ "${stopped}X" == "StoppedX" ]]; do + sleep 1 + stopped=$(pcs status | grep -u "Stopped") + done +} + + +refresh_config () +{ + local short_host=$(hostname -s) + local VOL=${1} + local HA_CONFDIR=${2} + local short_host=$(hostname -s) + + local export_id=$(grep ^[[:space:]]*Export_Id $HA_CONFDIR/exports/export.$VOL.conf |\ + awk -F"[=,;]" '{print $2}' | tr -d '[[:space:]]') + + + if [ -e ${SECRET_PEM} ]; then + while [[ ${3} ]]; do + current_host=`echo ${3} | cut -d "." -f 1` + if [[ ${short_host} != ${current_host} ]]; then + output=$(ssh -oPasswordAuthentication=no \ +-oStrictHostKeyChecking=no -i ${SECRET_PEM} root@${current_host} \ +"dbus-send --print-reply --system --dest=org.ganesha.nfsd \ +/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.UpdateExport \ +string:$HA_CONFDIR/exports/export.$VOL.conf \ +string:\"EXPORT(Export_Id=$export_id)\" 2>&1") + ret=$? + logger <<< "${output}" + if [ ${ret} -ne 0 ]; then + echo "Refresh-config failed on ${current_host}. Please check logs on ${current_host}" + else + echo "Refresh-config completed on ${current_host}." + fi + + fi + shift + done + else + echo "Error: refresh-config failed. Passwordless ssh is not enabled." + exit 1 + fi + + # Run the same command on the localhost, + output=$(dbus-send --print-reply --system --dest=org.ganesha.nfsd \ +/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.UpdateExport \ +string:$HA_CONFDIR/exports/export.$VOL.conf \ +string:"EXPORT(Export_Id=$export_id)" 2>&1) + ret=$? + logger <<< "${output}" + if [ ${ret} -ne 0 ] ; then + echo "Refresh-config failed on localhost." + else + echo "Success: refresh-config completed." + fi +} + + +teardown_cluster() +{ + local name=${1} + + for server in ${HA_SERVERS} ; do + if [[ ${HA_CLUSTER_NODES} != *${server}* ]]; then + logger "info: ${server} is not in config, removing" + + pcs cluster stop ${server} --force + if [ $? -ne 0 ]; then + logger "warning: pcs cluster stop ${server} failed" + fi + + pcs cluster node remove ${server} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster node remove ${server} failed" + fi + fi + done + + # BZ 1193433 - pcs doesn't reload cluster.conf after modification + # after teardown completes, a subsequent setup will appear to have + # 'remembered' the deleted node. You can work around this by + # issuing another `pcs cluster node remove $node`, + # `crm_node -f -R $server`, or + # `cibadmin --delete --xml-text '<node id="$server" + # uname="$server"/>' + + pcs cluster stop --all + if [ $? -ne 0 ]; then + logger "warning pcs cluster stop --all failed" + fi + + pcs cluster destroy + if [ $? -ne 0 ]; then + logger "error pcs cluster destroy failed" + exit 1 + fi +} + + +cleanup_ganesha_config () +{ + rm -f /etc/corosync/corosync.conf + rm -rf /etc/cluster/cluster.conf* + rm -rf /var/lib/pacemaker/cib/* + sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $HA_CONFDIR/ganesha.conf +} + +do_create_virt_ip_constraints() +{ + local cibfile=${1}; shift + local primary=${1}; shift + local weight="1000" + + # first a constraint location rule that says the VIP must be where + # there's a ganesha.nfsd running + pcs -f ${cibfile} constraint location ${primary}-group rule score=-INFINITY ganesha-active ne 1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint location ${primary}-group rule score=-INFINITY ganesha-active ne 1 failed" + fi + + # then a set of constraint location prefers to set the prefered order + # for where a VIP should move + while [[ ${1} ]]; do + pcs -f ${cibfile} constraint location ${primary}-group prefers ${1}=${weight} + if [ $? -ne 0 ]; then + logger "warning: pcs constraint location ${primary}-group prefers ${1}=${weight} failed" + fi + weight=$(expr ${weight} + 1000) + shift + done + # and finally set the highest preference for the VIP to its home node + # default weight when created is/was 100. + # on Fedora setting appears to be additive, so to get the desired + # value we adjust the weight + # weight=$(expr ${weight} - 100) + pcs -f ${cibfile} constraint location ${primary}-group prefers ${primary}=${weight} + if [ $? -ne 0 ]; then + logger "warning: pcs constraint location ${primary}-group prefers ${primary}=${weight} failed" + fi +} + + +wrap_create_virt_ip_constraints() +{ + local cibfile=${1}; shift + local primary=${1}; shift + local head="" + local tail="" + + # build a list of peers, e.g. for a four node cluster, for node1, + # the result is "node2 node3 node4"; for node2, "node3 node4 node1" + # and so on. + while [[ ${1} ]]; do + if [[ ${1} == ${primary} ]]; then + shift + while [[ ${1} ]]; do + tail=${tail}" "${1} + shift + done + else + head=${head}" "${1} + fi + shift + done + do_create_virt_ip_constraints ${cibfile} ${primary} ${tail} ${head} +} + + +create_virt_ip_constraints() +{ + local cibfile=${1}; shift + + while [[ ${1} ]]; do + wrap_create_virt_ip_constraints ${cibfile} ${1} ${HA_SERVERS} + shift + done +} + + +setup_create_resources() +{ + local cibfile=$(mktemp -u) + + # fixup /var/lib/nfs + logger "pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION}" + pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION} + if [ $? -ne 0 ]; then + logger "warning: pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION} failed" + fi + + pcs resource create nfs-mon ocf:heartbeat:ganesha_mon ${PCS9OR10_PCS_CLONE_OPTION} + if [ $? -ne 0 ]; then + logger "warning: pcs resource create nfs-mon ocf:heartbeat:ganesha_mon ${PCS9OR10_PCS_CLONE_OPTION} failed" + fi + + # see comment in (/usr/lib/ocf/resource.d/heartbeat/ganesha_grace + # start method. Allow time for ganesha_mon to start and set the + # ganesha-active crm_attribute + sleep 5 + + pcs resource create nfs-grace ocf:heartbeat:ganesha_grace ${PCS9OR10_PCS_CLONE_OPTION} notify=true + if [ $? -ne 0 ]; then + logger "warning: pcs resource create nfs-grace ocf:heartbeat:ganesha_grace ${PCS9OR10_PCS_CLONE_OPTION} failed" + fi + + pcs constraint location nfs-grace-clone rule score=-INFINITY grace-active ne 1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint location nfs-grace-clone rule score=-INFINITY grace-active ne 1" + fi + + pcs cluster cib ${cibfile} + + while [[ ${1} ]]; do + + # this is variable indirection + # from a nvs like 'VIP_host1=10.7.6.5' or 'VIP_host1="10.7.6.5"' + # (or VIP_host-1=..., or VIP_host-1.my.domain.name=...) + # a variable 'clean_name' is created (e.g. w/ value 'VIP_host_1') + # and a clean nvs (e.g. w/ value 'VIP_host_1="10_7_6_5"') + # after the `eval ${clean_nvs}` there is a variable VIP_host_1 + # with the value '10_7_6_5', and the following \$$ magic to + # reference it, i.e. `eval tmp_ipaddr=\$${clean_name}` gives us + # ${tmp_ipaddr} with 10_7_6_5 and then convert the _s back to .s + # to give us ipaddr="10.7.6.5". whew! + name="VIP_${1}" + clean_name=${name//[-.]/_} + nvs=$(grep "^${name}=" ${HA_CONFDIR}/ganesha-ha.conf) + clean_nvs=${nvs//[-.]/_} + eval ${clean_nvs} + eval tmp_ipaddr=\$${clean_name} + ipaddr=${tmp_ipaddr//_/.} + + pcs -f ${cibfile} resource create ${1}-nfs_block ocf:heartbeat:portblock protocol=tcp \ + portno=2049 action=block ip=${ipaddr} --group ${1}-group + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-nfs_block failed" + fi + pcs -f ${cibfile} resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \ + cidr_netmask=32 op monitor interval=15s --group ${1}-group --after ${1}-nfs_block + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \ + cidr_netmask=32 op monitor interval=15s failed" + fi + + pcs -f ${cibfile} constraint order nfs-grace-clone then ${1}-cluster_ip-1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint order nfs-grace-clone then ${1}-cluster_ip-1 failed" + fi + + pcs -f ${cibfile} resource create ${1}-nfs_unblock ocf:heartbeat:portblock protocol=tcp \ + portno=2049 action=unblock ip=${ipaddr} reset_local_on_unblock_stop=true \ + tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${1}-group --after ${1}-cluster_ip-1 \ + op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} \ + op monitor interval=10s timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-nfs_unblock failed" + fi + + + shift + done + + create_virt_ip_constraints ${cibfile} ${HA_SERVERS} + + pcs cluster cib-push ${cibfile} + if [ $? -ne 0 ]; then + logger "warning pcs cluster cib-push ${cibfile} failed" + fi + rm -f ${cibfile} +} + + +teardown_resources() +{ + # local mntpt=$(grep ha-vol-mnt ${HA_CONFIG_FILE} | cut -d = -f 2) + + # restore /var/lib/nfs + logger "notice: pcs resource delete nfs_setup-clone" + pcs resource delete nfs_setup-clone + if [ $? -ne 0 ]; then + logger "warning: pcs resource delete nfs_setup-clone failed" + fi + + # delete -clone resource agents + # in particular delete the ganesha monitor so we don't try to + # trigger anything when we shut down ganesha next. + pcs resource delete nfs-mon-clone + if [ $? -ne 0 ]; then + logger "warning: pcs resource delete nfs-mon-clone failed" + fi + + pcs resource delete nfs-grace-clone + if [ $? -ne 0 ]; then + logger "warning: pcs resource delete nfs-grace-clone failed" + fi + + while [[ ${1} ]]; do + pcs resource delete ${1}-group + if [ $? -ne 0 ]; then + logger "warning: pcs resource delete ${1}-group failed" + fi + shift + done + +} + + +recreate_resources() +{ + local cibfile=${1}; shift + + while [[ ${1} ]]; do + # this is variable indirection + # see the comment on the same a few lines up + name="VIP_${1}" + clean_name=${name//[-.]/_} + nvs=$(grep "^${name}=" ${HA_CONFDIR}/ganesha-ha.conf) + clean_nvs=${nvs//[-.]/_} + eval ${clean_nvs} + eval tmp_ipaddr=\$${clean_name} + ipaddr=${tmp_ipaddr//_/.} + + pcs -f ${cibfile} resource create ${1}-nfs_block ocf:heartbeat:portblock protocol=tcp \ + portno=2049 action=block ip=${ipaddr} --group ${1}-group + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-nfs_block failed" + fi + pcs -f ${cibfile} resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \ + cidr_netmask=32 op monitor interval=15s --group ${1}-group --after ${1}-nfs_block + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \ + cidr_netmask=32 op monitor interval=15s failed" + fi + + pcs -f ${cibfile} constraint order nfs-grace-clone then ${1}-cluster_ip-1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint order nfs-grace-clone then ${1}-cluster_ip-1 failed" + fi + + pcs -f ${cibfile} resource create ${1}-nfs_unblock ocf:heartbeat:portblock protocol=tcp \ + portno=2049 action=unblock ip=${ipaddr} reset_local_on_unblock_stop=true \ + tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${1}-group --after ${1}-cluster_ip-1 \ + op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} \ + op monitor interval=10s timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-nfs_unblock failed" + fi + + shift + done +} + + +addnode_recreate_resources() +{ + local cibfile=${1}; shift + local add_node=${1}; shift + local add_vip=${1}; shift + + recreate_resources ${cibfile} ${HA_SERVERS} + + pcs -f ${cibfile} resource create ${add_node}-nfs_block ocf:heartbeat:portblock \ + protocol=tcp portno=2049 action=block ip=${add_vip} --group ${add_node}-group + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${add_node}-nfs_block failed" + fi + pcs -f ${cibfile} resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr \ + ip=${add_vip} cidr_netmask=32 op monitor interval=15s --group ${add_node}-group \ + --after ${add_node}-nfs_block + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr \ + ip=${add_vip} cidr_netmask=32 op monitor interval=15s failed" + fi + + pcs -f ${cibfile} constraint order nfs-grace-clone then ${add_node}-cluster_ip-1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint order nfs-grace-clone then ${add_node}-cluster_ip-1 failed" + fi + pcs -f ${cibfile} resource create ${add_node}-nfs_unblock ocf:heartbeat:portblock \ + protocol=tcp portno=2049 action=unblock ip=${add_vip} reset_local_on_unblock_stop=true \ + tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${add_node}-group --after \ + ${add_node}-cluster_ip-1 op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start \ + timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op monitor interval=10s \ + timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${add_node}-nfs_unblock failed" + fi +} + + +clear_resources() +{ + local cibfile=${1}; shift + + while [[ ${1} ]]; do + pcs -f ${cibfile} resource delete ${1}-group + if [ $? -ne 0 ]; then + logger "warning: pcs -f ${cibfile} resource delete ${1}-group" + fi + + shift + done +} + + +addnode_create_resources() +{ + local add_node=${1}; shift + local add_vip=${1}; shift + local cibfile=$(mktemp -u) + + # start HA on the new node + pcs cluster start ${add_node} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster start ${add_node} failed" + fi + + pcs cluster cib ${cibfile} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster cib ${cibfile} failed" + fi + + # delete all the -cluster_ip-1 resources, clearing + # their constraints, then create them again so we can + # recompute their constraints + clear_resources ${cibfile} ${HA_SERVERS} + addnode_recreate_resources ${cibfile} ${add_node} ${add_vip} + + HA_SERVERS="${HA_SERVERS} ${add_node}" + create_virt_ip_constraints ${cibfile} ${HA_SERVERS} + + pcs cluster cib-push ${cibfile} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster cib-push ${cibfile} failed" + fi + rm -f ${cibfile} +} + + +deletenode_delete_resources() +{ + local node=${1}; shift + local ha_servers=$(echo "${HA_SERVERS}" | sed s/${node}//) + local cibfile=$(mktemp -u) + + pcs cluster cib ${cibfile} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster cib ${cibfile} failed" + fi + + # delete all the -cluster_ip-1 and -trigger_ip-1 resources, + # clearing their constraints, then create them again so we can + # recompute their constraints + clear_resources ${cibfile} ${HA_SERVERS} + recreate_resources ${cibfile} ${ha_servers} + HA_SERVERS=$(echo "${ha_servers}" | sed -e "s/ / /") + + create_virt_ip_constraints ${cibfile} ${HA_SERVERS} + + pcs cluster cib-push ${cibfile} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster cib-push ${cibfile} failed" + fi + rm -f ${cibfile} + +} + + +deletenode_update_haconfig() +{ + local name="VIP_${1}" + local clean_name=${name//[-.]/_} + + ha_servers=$(echo ${HA_SERVERS} | sed -e "s/ /,/") + sed -i -e "s/^HA_CLUSTER_NODES=.*$/HA_CLUSTER_NODES=\"${ha_servers// /,}\"/" -e "s/^${name}=.*$//" -e "/^$/d" ${HA_CONFDIR}/ganesha-ha.conf +} + + +setup_state_volume() +{ + local mnt=${HA_VOL_MNT} + local longname="" + local shortname="" + local dname="" + local dirname="" + + longname=$(hostname) + dname=${longname#$(hostname -s)} + + while [[ ${1} ]]; do + + if [[ ${1} == *${dname} ]]; then + dirname=${1} + else + dirname=${1}${dname} + fi + + if [ ! -d ${mnt}/nfs-ganesha/tickle_dir ]; then + mkdir ${mnt}/nfs-ganesha/tickle_dir + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname} ]; then + mkdir ${mnt}/nfs-ganesha/${dirname} + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd + fi + if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/state ]; then + touch ${mnt}/nfs-ganesha/${dirname}/nfs/state + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/state + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak + fi + if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state ]; then + touch ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state + fi + for server in ${HA_SERVERS} ; do + if [[ ${server} != ${dirname} ]]; then + ln -s ${mnt}/nfs-ganesha/${server}/nfs/ganesha ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/${server} + ln -s ${mnt}/nfs-ganesha/${server}/nfs/statd ${mnt}/nfs-ganesha/${dirname}/nfs/statd/${server} + fi + done + shift + done + +} + + +enable_pacemaker() +{ + while [[ ${1} ]]; do + if [[ "${SERVICE_MAN}" == "/bin/systemctl" ]]; then + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${1} "${SERVICE_MAN} enable pacemaker" + else + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${1} "${SERVICE_MAN} pacemaker enable" + fi + shift + done +} + + +addnode_state_volume() +{ + local newnode=${1}; shift + local mnt=${HA_VOL_MNT} + local longname="" + local dname="" + local dirname="" + + longname=$(hostname) + dname=${longname#$(hostname -s)} + + if [[ ${newnode} == *${dname} ]]; then + dirname=${newnode} + else + dirname=${newnode}${dname} + fi + + if [ ! -d ${mnt}/nfs-ganesha/${dirname} ]; then + mkdir ${mnt}/nfs-ganesha/${dirname} + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd + fi + if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/state ]; then + touch ${mnt}/nfs-ganesha/${dirname}/nfs/state + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/state + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak + fi + if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state ]; then + touch ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state + fi + + for server in ${HA_SERVERS} ; do + + if [[ ${server} != ${dirname} ]]; then + ln -s ${mnt}/nfs-ganesha/${server}/nfs/ganesha ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/${server} + ln -s ${mnt}/nfs-ganesha/${server}/nfs/statd ${mnt}/nfs-ganesha/${dirname}/nfs/statd/${server} + + ln -s ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ${mnt}/nfs-ganesha/${server}/nfs/ganesha/${dirname} + ln -s ${mnt}/nfs-ganesha/${dirname}/nfs/statd ${mnt}/nfs-ganesha/${server}/nfs/statd/${dirname} + fi + done + +} + + +delnode_state_volume() +{ + local delnode=${1}; shift + local mnt=${HA_VOL_MNT} + local longname="" + local dname="" + local dirname="" + + longname=$(hostname) + dname=${longname#$(hostname -s)} + + if [[ ${delnode} == *${dname} ]]; then + dirname=${delnode} + else + dirname=${delnode}${dname} + fi + + rm -rf ${mnt}/nfs-ganesha/${dirname} + + for server in ${HA_SERVERS} ; do + if [[ ${server} != ${dirname} ]]; then + rm -f ${mnt}/nfs-ganesha/${server}/nfs/ganesha/${dirname} + rm -f ${mnt}/nfs-ganesha/${server}/nfs/statd/${dirname} + fi + done +} + + +status() +{ + local scratch=$(mktemp) + local regex_str="^${1}-cluster_ip-1" + local healthy=0 + local index=1 + local nodes + + # change tabs to spaces, strip leading spaces, including any + # new '*' at the beginning of a line introduced in pcs-0.10.x + pcs status | sed -e "s/\t/ /g" -e "s/^[ ]*\*//" -e "s/^[ ]*//" > ${scratch} + + nodes[0]=${1}; shift + + # make a regex of the configured nodes + # and initalize the nodes array for later + while [[ ${1} ]]; do + + regex_str="${regex_str}|^${1}-cluster_ip-1" + nodes[${index}]=${1} + ((index++)) + shift + done + + # print the nodes that are expected to be online + grep -E "Online:" ${scratch} + + echo + + # print the VIPs and which node they are on + grep -E "${regex_str}" < ${scratch} | cut -d ' ' -f 1,4 + + echo + + # check if the VIP and port block/unblock RAs are on the expected nodes + for n in ${nodes[*]}; do + + grep -E -x "${n}-nfs_block \(ocf::heartbeat:portblock\): Started ${n}" > /dev/null 2>&1 ${scratch} + result=$? + ((healthy+=${result})) + grep -E -x "${n}-cluster_ip-1 \(ocf::heartbeat:IPaddr\): Started ${n}" > /dev/null 2>&1 ${scratch} + result=$? + ((healthy+=${result})) + grep -E -x "${n}-nfs_unblock \(ocf::heartbeat:portblock\): Started ${n}" > /dev/null 2>&1 ${scratch} + result=$? + ((healthy+=${result})) + done + + grep -E "\):\ Stopped|FAILED" > /dev/null 2>&1 ${scratch} + result=$? + + if [ ${result} -eq 0 ]; then + echo "Cluster HA Status: BAD" + elif [ ${healthy} -eq 0 ]; then + echo "Cluster HA Status: HEALTHY" + else + echo "Cluster HA Status: FAILOVER" + fi + + rm -f ${scratch} +} + +create_ganesha_conf_file() +{ + if [[ "$1" == "yes" ]]; + then + if [ -e $GANESHA_CONF ]; + then + rm -rf $GANESHA_CONF + fi + # The symlink /etc/ganesha/ganesha.conf need to be + # created using ganesha conf file mentioned in the + # shared storage. Every node will only have this + # link and actual file will stored in shared storage, + # so that ganesha conf editing of ganesha conf will + # be easy as well as it become more consistent. + + ln -s $HA_CONFDIR/ganesha.conf $GANESHA_CONF + else + # Restoring previous file + rm -rf $GANESHA_CONF + cp $HA_CONFDIR/ganesha.conf $GANESHA_CONF + sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $GANESHA_CONF + fi +} + +set_quorum_policy() +{ + local quorum_policy="stop" + local num_servers=${1} + + if [ ${num_servers} -lt 3 ]; then + quorum_policy="ignore" + fi + pcs property set no-quorum-policy=${quorum_policy} + if [ $? -ne 0 ]; then + logger "warning: pcs property set no-quorum-policy=${quorum_policy} failed" + fi +} + +main() +{ + + local cmd=${1}; shift + if [[ ${cmd} == *help ]]; then + usage + exit 0 + fi + + if (selinuxenabled) ;then + semanage boolean -m gluster_use_execmem --on + fi + + local osid="" + + osid=$(grep ^ID= /etc/os-release) + eval $(echo ${osid} | grep -F ID=) + osid=$(grep ^VERSION_ID= /etc/os-release) + eval $(echo ${osid} | grep -F VERSION_ID=) + + HA_CONFDIR=${1%/}; shift + local ha_conf=${HA_CONFDIR}/ganesha-ha.conf + local node="" + local vip="" + + # ignore any comment lines + cfgline=$(grep ^HA_NAME= ${ha_conf}) + eval $(echo ${cfgline} | grep -F HA_NAME=) + cfgline=$(grep ^HA_CLUSTER_NODES= ${ha_conf}) + eval $(echo ${cfgline} | grep -F HA_CLUSTER_NODES=) + + case "${cmd}" in + + setup | --setup) + logger "setting up ${HA_NAME}" + + check_cluster_exists ${HA_NAME} + + determine_servers "setup" + + # Fedora 29+ and rhel/centos 8 has PCS-0.10.x + # default is pcs-0.10.x options but check for + # rhel/centos 7 (pcs-0.9.x) and adjust accordingly + if [[ ! ${ID} =~ {rhel,centos} ]]; then + if [[ ${VERSION_ID} == 7.* ]]; then + PCS9OR10_PCS_CNAME_OPTION="--name" + PCS9OR10_PCS_CLONE_OPTION="--clone" + fi + fi + + if [[ "${HA_NUM_SERVERS}X" != "1X" ]]; then + + determine_service_manager + + setup_cluster ${HA_NAME} ${HA_NUM_SERVERS} "${HA_SERVERS}" + + setup_create_resources ${HA_SERVERS} + + setup_finalize_ha + + setup_state_volume ${HA_SERVERS} + + enable_pacemaker ${HA_SERVERS} + + else + + logger "insufficient servers for HA, aborting" + fi + ;; + + teardown | --teardown) + logger "tearing down ${HA_NAME}" + + determine_servers "teardown" + + teardown_resources ${HA_SERVERS} + + teardown_cluster ${HA_NAME} + + cleanup_ganesha_config ${HA_CONFDIR} + ;; + + cleanup | --cleanup) + cleanup_ganesha_config ${HA_CONFDIR} + ;; + + add | --add) + node=${1}; shift + vip=${1}; shift + + logger "adding ${node} with ${vip} to ${HA_NAME}" + + determine_service_manager + + manage_service "start" ${node} + + determine_servers "add" + + pcs cluster node add ${node} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster node add ${node} failed" + fi + + addnode_create_resources ${node} ${vip} + # Subsequent add-node recreates resources for all the nodes + # that already exist in the cluster. The nodes are picked up + # from the entries in the ganesha-ha.conf file. Adding the + # newly added node to the file so that the resources specfic + # to this node is correctly recreated in the future. + clean_node=${node//[-.]/_} + echo "VIP_${node}=\"${vip}\"" >> ${HA_CONFDIR}/ganesha-ha.conf + + NEW_NODES="$HA_CLUSTER_NODES,${node}" + + sed -i s/HA_CLUSTER_NODES.*/"HA_CLUSTER_NODES=\"$NEW_NODES\""/ \ +$HA_CONFDIR/ganesha-ha.conf + + addnode_state_volume ${node} + + # addnode_create_resources() already appended ${node} to + # HA_SERVERS, so only need to increment HA_NUM_SERVERS + # and set quorum policy + HA_NUM_SERVERS=$(expr ${HA_NUM_SERVERS} + 1) + set_quorum_policy ${HA_NUM_SERVERS} + ;; + + delete | --delete) + node=${1}; shift + + logger "deleting ${node} from ${HA_NAME}" + + determine_servers "delete" + + deletenode_delete_resources ${node} + + pcs cluster node remove ${node} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster node remove ${node} failed" + fi + + deletenode_update_haconfig ${node} + + delnode_state_volume ${node} + + determine_service_manager + + manage_service "stop" ${node} + + HA_NUM_SERVERS=$(expr ${HA_NUM_SERVERS} - 1) + set_quorum_policy ${HA_NUM_SERVERS} + ;; + + status | --status) + determine_servers "status" + + status ${HA_SERVERS} + ;; + + refresh-config | --refresh-config) + VOL=${1} + + determine_servers "refresh-config" + + refresh_config ${VOL} ${HA_CONFDIR} ${HA_SERVERS} + ;; + + setup-ganesha-conf-files | --setup-ganesha-conf-files) + + create_ganesha_conf_file ${1} + ;; + + *) + # setup and teardown are not intended to be used by a + # casual user + usage + logger "Usage: ganesha-ha.sh add|delete|status" + ;; + + esac + + if (selinuxenabled) ;then + semanage boolean -m gluster_use_execmem --off + fi +} + +main $* diff --git a/extras/ganesha/scripts/generate-epoch.py b/extras/ganesha/scripts/generate-epoch.py new file mode 100755 index 00000000000..77af014bab9 --- /dev/null +++ b/extras/ganesha/scripts/generate-epoch.py @@ -0,0 +1,48 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. +# +# Generates unique epoch value on each gluster node to be used by +# nfs-ganesha service on that node. +# +# Configure 'EPOCH_EXEC' option to this script path in +# '/etc/sysconfig/ganesha' file used by nfs-ganesha service. +# +# Construct epoch as follows - +# first 32-bit contains the now() time +# rest 32-bit value contains the local glusterd node uuid + +import time +import binascii + +# Calculate the now() time into a 64-bit integer value +def epoch_now(): + epoch_time = int(time.mktime(time.localtime())) << 32 + return epoch_time + +# Read glusterd UUID and extract first 32-bit of it +def epoch_uuid(): + file_name = '/var/lib/glusterd/glusterd.info' + + for line in open(file_name): + if "UUID" in line: + glusterd_uuid = line.split('=')[1].strip() + + uuid_bin = binascii.unhexlify(glusterd_uuid.replace("-","")) + + epoch_uuid = int(binascii.hexlify(uuid_bin), 32) & 0xFFFF0000 + return epoch_uuid + +# Construct epoch as follows - +# first 32-bit contains the now() time +# rest 32-bit value contains the local glusterd node uuid +epoch = (epoch_now() | epoch_uuid()) +print((str(epoch))) + +exit(0) diff --git a/extras/geo-rep/Makefile.am b/extras/geo-rep/Makefile.am new file mode 100644 index 00000000000..09eff308ac4 --- /dev/null +++ b/extras/geo-rep/Makefile.am @@ -0,0 +1,16 @@ +scriptsdir = $(libexecdir)/glusterfs/scripts +scripts_SCRIPTS = gsync-upgrade.sh generate-gfid-file.sh get-gfid.sh \ + slave-upgrade.sh schedule_georep.py + +scripts_PROGRAMS = gsync-sync-gfid +gsync_sync_gfid_CFLAGS = $(GF_CFLAGS) -Wall -I$(top_srcdir)/libglusterfs/src +gsync_sync_gfid_LDFLAGS = $(GF_LDFLAGS) +gsync_sync_gfid_LDADD = $(GF_LDADD) $(top_builddir)/libglusterfs/src/libglusterfs.la +gsync_sync_gfid_SOURCES = gsync-sync-gfid.c +gsync_sync_gfid_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ + -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src + +EXTRA_DIST = gsync-sync-gfid.c gsync-upgrade.sh generate-gfid-file.sh \ + get-gfid.sh slave-upgrade.sh schedule_georep.py.in + +CLEANFILES = schedule_georep.py diff --git a/extras/geo-rep/generate-gfid-file.sh b/extras/geo-rep/generate-gfid-file.sh new file mode 100644 index 00000000000..14f104b986d --- /dev/null +++ b/extras/geo-rep/generate-gfid-file.sh @@ -0,0 +1,70 @@ +#!/bin/bash +#Usage: generate-gfid-file.sh <master-volfile-server:master-volume> <path-to-get-gfid.sh> <output-file> [dirs-list-file] + +function get_gfids() +{ + GET_GFID_CMD=$1 + OUTPUT_FILE=$2 + DIR_PATH=$3 + find "$DIR_PATH" -exec $GET_GFID_CMD {} \; >> $OUTPUT_FILE +} + +function mount_client() +{ + local T; # temporary mount + local i; # inode number + + VOLFILE_SERVER=$1; + VOLUME=$2; + GFID_CMD=$3; + OUTPUT=$4; + + T=$(mktemp -d -t ${0##*/}.XXXXXX); + + glusterfs -s $VOLFILE_SERVER --volfile-id $VOLUME $T; + + i=$(stat -c '%i' $T); + + [ "x$i" = "x1" ] || fatal "could not mount volume $MASTER on $T"; + + cd $T; + rm -f $OUTPUT; + touch $OUTPUT; + + if [ "$DIRS_FILE" = "." ] + then + get_gfids $GFID_CMD $OUTPUT "." + else + while read line + do + get_gfids $GFID_CMD $OUTPUT "$line" + done < $DIRS_FILE + fi; + + cd -; + + umount $T || fatal "could not umount $MASTER from $T"; + + rmdir $T || warn "rmdir of $T failed"; +} + + +function main() +{ + SLAVE=$1 + GET_GFID_CMD=$2 + OUTPUT=$3 + + VOLFILE_SERVER=`echo $SLAVE | sed -e 's/\(.*\):.*/\1/'` + VOLUME_NAME=`echo $SLAVE | sed -e 's/.*:\(.*\)/\1/'` + + if [ "$#" -lt 4 ] + then + DIRS_FILE="." + else + DIRS_FILE=$4 + fi + mount_client $VOLFILE_SERVER $VOLUME_NAME $GET_GFID_CMD $OUTPUT $DIRS_FILE +} + +main "$@"; diff --git a/extras/geo-rep/get-gfid.sh b/extras/geo-rep/get-gfid.sh new file mode 100755 index 00000000000..a4d609b0bc5 --- /dev/null +++ b/extras/geo-rep/get-gfid.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +ATTR_STR=`getfattr -h $1 -n glusterfs.gfid.string` +GLFS_PATH=`echo $ATTR_STR | sed -e 's/# file: \(.*\) glusterfs.gfid.string*/\1/g'` +GFID=`echo $ATTR_STR | sed -e 's/.*glusterfs.gfid.string="\(.*\)"/\1/g'` + +echo "$GFID $GLFS_PATH" diff --git a/extras/geo-rep/gsync-sync-gfid.c b/extras/geo-rep/gsync-sync-gfid.c new file mode 100644 index 00000000000..47dca0413e9 --- /dev/null +++ b/extras/geo-rep/gsync-sync-gfid.c @@ -0,0 +1,109 @@ + +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <limits.h> +#include <sys/types.h> +#include <libgen.h> +#include <ctype.h> +#include <stdlib.h> +#include <glusterfs/glusterfs.h> +#include <glusterfs/syscall.h> + +#ifndef UUID_CANONICAL_FORM_LEN +#define UUID_CANONICAL_FORM_LEN 36 +#endif + +#ifndef GF_FUSE_AUX_GFID_HEAL +#define GF_FUSE_AUX_GFID_HEAL "glusterfs.gfid.heal" +#endif + +#define GLFS_LINE_MAX (PATH_MAX + (2 * UUID_CANONICAL_FORM_LEN)) + +int +main(int argc, char *argv[]) +{ + char *file = NULL; + char *tmp = NULL; + char *tmp1 = NULL; + char *parent_dir = NULL; + char *gfid = NULL; + char *bname = NULL; + int ret = -1; + int len = 0; + FILE *fp = NULL; + char line[GLFS_LINE_MAX] = { + 0, + }; + char *path = NULL; + void *blob = NULL; + void *tmp_blob = NULL; + + if (argc != 2) { + /* each line in the file has the following format + * uuid-in-canonical-form path-relative-to-gluster-mount. + * Both uuid and relative path are from master mount. + */ + fprintf(stderr, "usage: %s <file-of-paths-to-be-synced>\n", argv[0]); + goto out; + } + + file = argv[1]; + + fp = fopen(file, "r"); + if (fp == NULL) { + fprintf(stderr, "cannot open %s for reading (%s)\n", file, + strerror(errno)); + goto out; + } + + while (fgets(line, GLFS_LINE_MAX, fp) != NULL) { + tmp = line; + path = gfid = line; + + path += UUID_CANONICAL_FORM_LEN + 1; + + while (isspace(*path)) + path++; + + len = strlen(line); + if ((len < GLFS_LINE_MAX) && (line[len - 1] == '\n')) + line[len - 1] = '\0'; + + line[UUID_CANONICAL_FORM_LEN] = '\0'; + + tmp = strdup(path); + tmp1 = strdup(path); + parent_dir = dirname(tmp); + bname = basename(tmp1); + + /* gfid + '\0' + bname + '\0' */ + len = UUID_CANONICAL_FORM_LEN + 1 + strlen(bname) + 1; + + blob = malloc(len); + + memcpy(blob, gfid, UUID_CANONICAL_FORM_LEN); + + tmp_blob = blob + UUID_CANONICAL_FORM_LEN + 1; + + memcpy(tmp_blob, bname, strlen(bname)); + + ret = sys_lsetxattr(parent_dir, GF_FUSE_AUX_GFID_HEAL, blob, len, 0); + if (ret < 0) { + fprintf(stderr, "setxattr on %s/%s failed (%s)\n", parent_dir, + bname, strerror(errno)); + } + memset(line, 0, GLFS_LINE_MAX); + + free(blob); + free(tmp); + free(tmp1); + blob = NULL; + } + + ret = 0; +out: + if (fp) + fclose(fp); + return ret; +} diff --git a/extras/geo-rep/gsync-upgrade.sh b/extras/geo-rep/gsync-upgrade.sh new file mode 100644 index 00000000000..0f73a33884b --- /dev/null +++ b/extras/geo-rep/gsync-upgrade.sh @@ -0,0 +1,123 @@ +#!/bin/bash +#usage: gsync-upgrade.sh <slave-volfile-server:slave-volume> <gfid-file> +# <path-to-gsync-sync-gfid> <ssh-identity-file> +#<slave-volfile-server>: a machine on which gluster cli can fetch slave volume info. +# slave-volfile-server defaults to localhost. +# +#<gfid-file>: a file containing paths and their associated gfids +# on master. The paths are relative to master mount point +# (not absolute). An example extract of <gfid-file> can be, +# +# <extract> +# 22114455-57c5-46e9-a783-c40f83a72b09 /dir +# 25772386-3eb8-4550-a802-c3fdc938ca80 /dir/file +# </extract> +# +#<ssh-identity-file>: file from which the identity (private key) for public key authentication is read. + +SLAVE_MOUNT='/tmp/glfs_slave' + +function SSH() +{ + HOST=$1 + SSHKEY=$2 + + shift 2 + + ssh -qi $SSHKEY \ + -oPasswordAuthentication=no \ + -oStrictHostKeyChecking=no \ + "$HOST" "$@"; +} + +function get_bricks() +{ + SSHKEY=$3 + + SSH $1 $SSHKEY "gluster volume info $2" | grep -E 'Brick[0-9]+' | sed -e 's/[^:]*:\(.*\)/\1/g' +} + +function cleanup_brick() +{ + HOST=$1 + BRICK=$2 + SSHKEY=$3 + + # TODO: write a C program to receive a list of files and does cleanup on + # them instead of spawning a new setfattr process for each file if + # performance is bad. + SSH -i $SSHKEY $HOST "rm -rf $BRICK/.glusterfs/* && find $BRICK -exec setfattr -x trusted.gfid {} \;" +} + +function cleanup_slave() +{ + SSHKEY=$2 + + VOLFILE_SERVER=`echo $1 | sed -e 's/\(.*\):.*/\1/'` + VOLUME_NAME=`echo $1 | sed -e 's/.*:\(.*\)/\1/'` + + BRICKS=`get_bricks $VOLFILE_SERVER $VOLUME_NAME $SSHKEY` + + for i in $BRICKS; do + HOST=`echo $i | sed -e 's/\(.*\):.*/\1/'` + BRICK=`echo $i | sed -e 's/.*:\(.*\)/\1/'` + cleanup_brick $HOST $BRICK $SSHKEY + done + + SSH -i $SSHKEY $VOLFILE_SERVER "gluster --mode=script volume stop $VOLUME_NAME; gluster volume start $VOLUME_NAME"; + +} + +function mount_client() +{ + local T; # temporary mount + local i; # inode number + GFID_FILE=$3 + SYNC_CMD=$4 + + T=$(mktemp -d -t ${0##*/}.XXXXXX); + + glusterfs --aux-gfid-mount -s $1 --volfile-id $2 $T; + + i=$(stat -c '%i' $T); + + [ "x$i" = "x1" ] || fatal "could not mount volume $MASTER on $T"; + + cd $T; + + $SYNC_CMD $GFID_FILE + + cd -; + + umount -l $T || fatal "could not umount $MASTER from $T"; + + rmdir $T || warn "rmdir of $T failed"; +} + +function sync_gfids() +{ + SLAVE=$1 + GFID_FILE=$2 + + SLAVE_VOLFILE_SERVER=`echo $SLAVE | sed -e 's/\(.*\):.*/\1/'` + SLAVE_VOLUME_NAME=`echo $SLAVE | sed -e 's/.*:\(.*\)/\1/'` + + if [ "x$SLAVE_VOLFILE_SERVER" = "x" ]; then + SLAVE_VOLFILE_SERVER="localhost" + fi + + mount_client $SLAVE_VOLFILE_SERVER $SLAVE_VOLUME_NAME $GFID_FILE $3 +} + +function upgrade() +{ + SLAVE=$1 + GFID_FILE=$2 + SYNC_CMD=$3 + SSHKEY=$4 + + cleanup_slave $SLAVE $SSHKEY + sync_gfids $SLAVE $GFID_FILE $SYNC_CMD +} + +upgrade "$@" diff --git a/extras/geo-rep/schedule_georep.py.in b/extras/geo-rep/schedule_georep.py.in new file mode 100644 index 00000000000..48b2b507060 --- /dev/null +++ b/extras/geo-rep/schedule_georep.py.in @@ -0,0 +1,492 @@ +#!/usr/bin/python3 +""" +Schedule Geo-replication +------------------------ +A tool to run Geo-replication when required. This can be used to +schedule the Geo-replication to run once in a day using + + # Run daily at 08:30pm + 30 20 * * * root python /usr/share/glusterfs/scripts/schedule_georep.py \\ + --no-color gv1 fvm1 gv2 >> /var/log/glusterfs/schedule_georep.log 2>&1 + +This tool does the following, + +1. Stop Geo-replication if Started +2. Start Geo-replication +3. Set Checkpoint +4. Check the Status and see Checkpoint is Complete.(LOOP) +5. If checkpoint complete, Stop Geo-replication + +Usage: + + python /usr/share/glusterfs/scripts/schedule_georep.py <MASTERVOL> \\ + <SLAVEHOST> <SLAVEVOL> + +For example, + + python /usr/share/glusterfs/scripts/schedule_georep.py gv1 fvm1 gv2 + +""" +import subprocess +import time +import xml.etree.cElementTree as etree +import sys +from contextlib import contextmanager +import tempfile +import os +from argparse import ArgumentParser, RawDescriptionHelpFormatter + +ParseError = etree.ParseError if hasattr(etree, 'ParseError') else SyntaxError +cache_data = {} + +SESSION_MOUNT_LOG_FILE = ("/var/log/glusterfs/geo-replication" + "/schedule_georep.mount.log") + +USE_CLI_COLOR = True +mnt_list = [] + +class GlusterBadXmlFormat(Exception): + """ + Exception class for XML Parse Errors + """ + pass + + +def output_notok(msg, err="", exitcode=1): + if USE_CLI_COLOR: + out = "\033[31m[NOT OK]\033[0m {0}\n{1}\n" + else: + out = "[NOT OK] {0}\n{1}\n" + sys.stderr.write(out.format(msg, err)) + sys.exit(exitcode) + + +def output_warning(msg): + if USE_CLI_COLOR: + out = "\033[33m[ WARN]\033[0m {0}\n" + else: + out = "[ WARN] {0}\n" + sys.stderr.write(out.format(msg)) + + +def output_ok(msg): + if USE_CLI_COLOR: + out = "\033[32m[ OK]\033[0m {0}\n" + else: + out = "[ OK] {0}\n" + sys.stderr.write(out.format(msg)) + + +def execute(cmd, success_msg="", failure_msg="", exitcode=-1): + """ + Generic wrapper to execute the CLI commands. Returns Output if success. + On success it can print message in stdout if specified. + On failure, exits after writing to stderr. + """ + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + out, err = p.communicate() + if p.returncode == 0: + if success_msg: + output_ok(success_msg) + return out + else: + if exitcode == 0: + return + err_msg = err if err else out + output_notok(failure_msg, err=err_msg, exitcode=exitcode) + + +def cache_output_with_args(func): + """ + Decorator function to remember the output of any function + """ + def wrapper(*args, **kwargs): + global cache_data + key = "_".join([func.func_name] + list(args)) + if cache_data.get(key, None) is None: + cache_data[key] = func(*args, **kwargs) + + return cache_data[key] + return wrapper + + +def cleanup(hostname, volname, mnt): + """ + Unmount the Volume and Remove the temporary directory + """ + execute(["umount", "-l", mnt], + failure_msg="Unable to Unmount Gluster Volume " + "{0}:{1}(Mounted at {2})".format(hostname, volname, mnt)) + execute(["rmdir", mnt], + failure_msg="Unable to Remove temp directory " + "{0}".format(mnt), exitcode=0) + + +@contextmanager +def glustermount(hostname, volname): + """ + Context manager for Mounting Gluster Volume + Use as + with glustermount(HOSTNAME, VOLNAME) as MNT: + # Do your stuff + Automatically unmounts it in case of Exceptions/out of context + """ + mnt = tempfile.mkdtemp(prefix="georepsetup_") + mnt_list.append(mnt) + execute(["@SBIN_DIR@/glusterfs", + "--volfile-server", hostname, + "--volfile-id", volname, + "-l", SESSION_MOUNT_LOG_FILE, + mnt], + failure_msg="Unable to Mount Gluster Volume " + "{0}:{1}".format(hostname, volname)) + if os.path.ismount(mnt): + yield mnt + else: + output_notok("Unable to Mount Gluster Volume " + "{0}:{1}".format(hostname, volname)) + cleanup(hostname, volname, mnt) + + +@cache_output_with_args +def get_bricks(volname): + """ + Returns Bricks list, caches the Bricks list for a volume once + parsed. + """ + value = [] + cmd = ["@SBIN_DIR@/gluster", "volume", "info", volname, "--xml"] + info = execute(cmd) + try: + tree = etree.fromstring(info) + volume_el = tree.find('volInfo/volumes/volume') + for b in volume_el.findall('bricks/brick'): + value.append({"name": b.find("name").text, + "hostUuid": b.find("hostUuid").text}) + except ParseError: + raise GlusterBadXmlFormat("Bad XML Format: %s" % " ".join(cmd)) + + return value + + +def get_georep_status(mastervol, slave): + session_keys = set() + out = {} + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication"] + if mastervol is not None: + cmd += [mastervol] + if slave: + cmd += [slave] + + cmd += ["status", "--xml"] + info = execute(cmd) + + try: + tree = etree.fromstring(info) + # Get All Sessions + for volume_el in tree.findall("geoRep/volume"): + sessions_el = volume_el.find("sessions") + # Master Volume name if multiple Volumes + mvol = volume_el.find("name").text + + # For each session, collect the details + for session in sessions_el.findall("session"): + session_slave = "{0}:{1}".format(mvol, session.find( + "session_slave").text) + session_keys.add(session_slave) + out[session_slave] = {} + + for pair in session.findall('pair'): + master_brick = "{0}:{1}".format( + pair.find("master_node").text, + pair.find("master_brick").text + ) + + out[session_slave][master_brick] = { + "mastervol": mvol, + "slavevol": pair.find("slave").text.split("::")[-1], + "master_node": pair.find("master_node").text, + "master_brick": pair.find("master_brick").text, + "slave_user": pair.find("slave_user").text, + "slave": pair.find("slave").text, + "slave_node": pair.find("slave_node").text, + "status": pair.find("status").text, + "crawl_status": pair.find("crawl_status").text, + "entry": pair.find("entry").text, + "data": pair.find("data").text, + "meta": pair.find("meta").text, + "failures": pair.find("failures").text, + "checkpoint_completed": pair.find( + "checkpoint_completed").text, + "master_node_uuid": pair.find("master_node_uuid").text, + "last_synced": pair.find("last_synced").text, + "checkpoint_time": pair.find("checkpoint_time").text, + "checkpoint_completion_time": + pair.find("checkpoint_completion_time").text + } + except ParseError: + raise GlusterBadXmlFormat("Bad XML Format: %s" % " ".join(cmd)) + + return session_keys, out + + +def get_offline_status(volname, brick, node_uuid, slave): + node, brick = brick.split(":") + if "@" not in slave: + slave_user = "root" + else: + slave_user, _ = slave.split("@") + + return { + "mastervol": volname, + "slavevol": slave.split("::")[-1], + "master_node": node, + "master_brick": brick, + "slave_user": slave_user, + "slave": slave, + "slave_node": "N/A", + "status": "Offline", + "crawl_status": "N/A", + "entry": "N/A", + "data": "N/A", + "meta": "N/A", + "failures": "N/A", + "checkpoint_completed": "N/A", + "master_node_uuid": node_uuid, + "last_synced": "N/A", + "checkpoint_time": "N/A", + "checkpoint_completion_time": "N/A" + } + + +def get(mastervol=None, slave=None): + """ + This function gets list of Bricks of Master Volume and collects + respective Geo-rep status. Output will be always ordered as the + bricks list in Master Volume. If Geo-rep status is not available + for any brick then it updates OFFLINE status. + """ + out = [] + session_keys, gstatus = get_georep_status(mastervol, slave) + + for session in session_keys: + mvol, _, slave = session.split(":", 2) + slave = slave.replace("ssh://", "") + master_bricks = get_bricks(mvol) + out.append([]) + for brick in master_bricks: + bname = brick["name"] + if gstatus.get(session) and gstatus[session].get(bname, None): + out[-1].append(gstatus[session][bname]) + else: + out[-1].append( + get_offline_status(mvol, bname, brick["hostUuid"], slave)) + + return out + + +def get_summary(mastervol, slave_url): + """ + Wrapper function around Geo-rep Status and Gluster Volume Info + This combines the output from Bricks list and Geo-rep Status. + If a Master Brick node is down or Status is faulty then increments + the faulty counter. It also collects the checkpoint status from all + workers and compares with Number of Bricks. + """ + down_rows = [] + faulty_rows = [] + out = [] + + status_data = get(mastervol, slave_url) + + for session in status_data: + session_name = "" + summary = { + "active": 0, + "passive": 0, + "faulty": 0, + "initializing": 0, + "stopped": 0, + "created": 0, + "offline": 0, + "paused": 0, + "workers": 0, + "completed_checkpoints": 0, + "checkpoint": False, + "checkpoints_ok": False, + "ok": False + } + + for row in session: + summary[row["status"].replace("...", "").lower()] += 1 + summary["workers"] += 1 + if row["checkpoint_completed"] == "Yes": + summary["completed_checkpoints"] += 1 + + session_name = "{0}=>{1}".format( + row["mastervol"], + row["slave"].replace("ssh://", "") + ) + + if row["status"] == "Faulty": + faulty_rows.append("{0}:{1}".format(row["master_node"], + row["master_brick"])) + + if row["status"] == "Offline": + down_rows.append("{0}:{1}".format(row["master_node"], + row["master_brick"])) + + if summary["active"] == summary["completed_checkpoints"] and \ + summary["faulty"] == 0 and summary["offline"] == 0: + summary["checkpoints_ok"] = True + + if summary["faulty"] == 0 and summary["offline"] == 0: + summary["ok"] = True + + if session_name != "": + out.append([session_name, summary, faulty_rows, down_rows]) + + return out + + +def touch_mount_root(mastervol): + # Create a Mount and Touch the Mount point root, + # Hack to make sure some event available after + # setting Checkpoint. Without this there is a chance of + # Checkpoint never completes. + with glustermount("localhost", mastervol) as mnt: + execute(["touch", mnt]) + + +def main(args): + turns = 1 + + # Stop Force + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", args.mastervol, + "%s::%s" % (args.slave, args.slavevol), "stop", "force"] + execute(cmd) + output_ok("Stopped Geo-replication") + + # Set Checkpoint to NOW + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", args.mastervol, + "%s::%s" % (args.slave, args.slavevol), "config", "checkpoint", + "now"] + execute(cmd) + output_ok("Set Checkpoint") + + # Start the Geo-replication + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", args.mastervol, + "%s::%s" % (args.slave, args.slavevol), "start"] + execute(cmd) + output_ok("Started Geo-replication and watching Status for " + "Checkpoint completion") + + start_time = int(time.time()) + duration = 0 + + # Sleep till Geo-rep initializes + time.sleep(60) + + touch_mount_root(args.mastervol) + + slave_url = "{0}::{1}".format(args.slave, args.slavevol) + + # Loop to Check the Geo-replication Status and Checkpoint + # If All Status OK and all Checkpoints complete, + # Stop the Geo-replication and Log the Completeness + while True: + session_summary = get_summary(args.mastervol, + slave_url) + if len(session_summary) == 0: + # If Status command fails with another transaction error + # or any other error. Gluster cmd still produces XML output + # with different message + output_warning("Unable to get Geo-replication Status") + else: + session_name, summary, faulty_rows, down_rows = session_summary[0] + chkpt_status = "COMPLETE" if summary["checkpoints_ok"] else \ + "NOT COMPLETE" + ok_status = "OK" if summary["ok"] else "NOT OK" + + if summary["ok"]: + output_ok("All Checkpoints {1}, " + "All status {2} (Turns {0:>3})".format( + turns, chkpt_status, ok_status)) + else: + output_warning("All Checkpoints {1}, " + "All status {2} (Turns {0:>3})".format( + turns, chkpt_status, ok_status)) + + output_warning("Geo-rep workers Faulty/Offline, " + "Faulty: {0} Offline: {1}".format( + repr(faulty_rows), + repr(down_rows))) + + if summary["checkpoints_ok"]: + output_ok("Stopping Geo-replication session now") + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", + args.mastervol, + "%s::%s" % (args.slave, args.slavevol), "stop"] + execute(cmd) + break + else: + # If Checkpoint is not complete after a iteration means brick + # was down and came online now. SETATTR on mount is not + # recorded, So again issue touch on mount root So that + # Stime will increase and Checkpoint will complete. + touch_mount_root(args.mastervol) + + # Increment the turns and Sleep for 10 sec + turns += 1 + duration = int(time.time()) - start_time + if args.timeout > 0 and duration > (args.timeout * 60): + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", + args.mastervol, + "%s::%s" % (args.slave, args.slavevol), "stop", "force"] + execute(cmd) + output_notok("Timed out, Stopping Geo-replication(" + "Duration: {0}sec)".format(duration)) + + time.sleep(args.interval) + + for mnt in mnt_list: + execute(["rmdir", mnt], + failure_msg="Unable to Remove temp directory " + "{0}".format(mnt), exitcode=0) + +if __name__ == "__main__": + parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter, + description=__doc__) + parser.add_argument("mastervol", help="Master Volume Name") + parser.add_argument("slave", + help="Slave hostname " + "(<username>@SLAVEHOST or SLAVEHOST)", + metavar="SLAVE") + parser.add_argument("slavevol", help="Slave Volume Name") + parser.add_argument("--interval", help="Interval in Seconds. " + "Wait time before each status check", + type=int, default=10) + parser.add_argument("--timeout", help="Timeout in minutes. Script will " + "stop Geo-replication if Checkpoint is not complete " + "in the specified timeout time", type=int, + default=0) + parser.add_argument("--no-color", help="Don't use Color in CLI output", + action="store_true") + args = parser.parse_args() + if args.no_color: + USE_CLI_COLOR = False + try: + # Check for session existence + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", + args.mastervol, "%s::%s" % (args.slave, args.slavevol), "status"] + execute(cmd) + main(args) + except KeyboardInterrupt: + for mnt in mnt_list: + execute(["umount", "-l", mnt], + failure_msg="Unable to Unmount Gluster Volume " + "Mounted at {0}".format(mnt), exitcode=0) + execute(["rmdir", mnt], + failure_msg="Unable to Remove temp directory " + "{0}".format(mnt), exitcode=0) + output_notok("Exiting...") diff --git a/extras/geo-rep/slave-upgrade.sh b/extras/geo-rep/slave-upgrade.sh new file mode 100644 index 00000000000..3a37f8e3579 --- /dev/null +++ b/extras/geo-rep/slave-upgrade.sh @@ -0,0 +1,102 @@ +#!/bin/bash +#usage: slave-upgrade.sh <volfile-server:volname> <gfid-file> +# <path-to-gsync-sync-gfid> +#<slave-volfile-server>: a machine on which gluster cli can fetch slave volume info. +# slave-volfile-server defaults to localhost. +# +#<gfid-file>: a file containing paths and their associated gfids +# on master. The paths are relative to master mount point +# (not absolute). An example extract of <gfid-file> can be, +# +# <extract> +# 22114455-57c5-46e9-a783-c40f83a72b09 /dir +# 25772386-3eb8-4550-a802-c3fdc938ca80 /dir/file +# </extract> + +function get_bricks() +{ + gluster volume info $1 | grep -E 'Brick[0-9]+' | sed -e 's/[^:]*:\(.*\)/\1/g' +} + +function cleanup_brick() +{ + HOST=$1 + BRICK=$2 + + # TODO: write a C program to receive a list of files and does cleanup on + # them instead of spawning a new setfattr process for each file if + # performance is bad. + ssh $HOST "rm -rf $BRICK/.glusterfs/* && find $BRICK -exec setfattr -x trusted.gfid {} \; 2>/dev/null" +} + +function cleanup_slave() +{ + VOLUME_NAME=`echo $1 | sed -e 's/.*:\(.*\)/\1/'` + + BRICKS=`get_bricks $VOLUME_NAME` + + for i in $BRICKS; do + HOST=`echo $i | sed -e 's/\(.*\):.*/\1/'` + BRICK=`echo $i | sed -e 's/.*:\(.*\)/\1/'` + cleanup_brick $HOST $BRICK + done + + # Now restart the volume + gluster --mode=script volume stop $VOLUME_NAME; + gluster volume start $VOLUME_NAME; +} + +function mount_client() +{ + local T; # temporary mount + local i; # inode number + + VOLUME_NAME=$2; + GFID_FILE=$3 + SYNC_CMD=$4 + + T=$(mktemp -d -t ${0##*/}.XXXXXX); + + glusterfs --aux-gfid-mount -s $1 --volfile-id $VOLUME_NAME $T; + + i=$(stat -c '%i' $T); + + cd $T; + + $SYNC_CMD $GFID_FILE + + cd -; + + umount $T || fatal "could not umount $MASTER from $T"; + + rmdir $T || warn "rmdir of $T failed"; +} + +function sync_gfids() +{ + SLAVE=$1 + GFID_FILE=$2 + SYNC_CMD=$3 + + SLAVE_VOLFILE_SERVER=`echo $SLAVE | sed -e 's/\(.*\):.*/\1/'` + SLAVE_VOLUME_NAME=`echo $SLAVE | sed -e 's/.*:\(.*\)/\1/'` + + if [ "x$SLAVE_VOLFILE_SERVER" = "x" ]; then + SLAVE_VOLFILE_SERVER="localhost" + fi + + mount_client $SLAVE_VOLFILE_SERVER $SLAVE_VOLUME_NAME $GFID_FILE $SYNC_CMD +} + +function upgrade() +{ + SLAVE=$1 + GFID_FILE=$2 + SYNC_CMD=$3 + + cleanup_slave $SLAVE + + sync_gfids $SLAVE $GFID_FILE $SYNC_CMD +} + +upgrade "$@" diff --git a/extras/gfid-to-dirname.sh b/extras/gfid-to-dirname.sh new file mode 100755 index 00000000000..fd359fab58a --- /dev/null +++ b/extras/gfid-to-dirname.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +function read_symlink() +{ + DOT_GLUSTERFS_PATH=$BRICK_PATH/.glusterfs + gfid_string=$1 + symlink_path="$DOT_GLUSTERFS_PATH/${gfid_string:0:2}/${gfid_string:2:2}/$gfid_string" + #remove trailing '/' + symlink_path=${symlink_path%/} + linkname=$(readlink $symlink_path) + if [ $? -ne 0 ]; then + echo "readlink of $symlink_path returned an error." >&2 + exit -1 + fi + echo $linkname +} + +main() +{ + if [ $# -lt 2 ] ;then + echo "Usage: $0 <brick-path> <gfid-string-of-directory>" + echo "Example: $0 /bricks/brick1 1b835012-1ae5-4f0d-9db4-64de574d891c" + exit -1 + fi + + BRICK_PATH=$1 + name=$(read_symlink $2) + if [ $? -ne 0 ]; then + exit -1 + fi + + while [ ${name:12:36} != "00000000-0000-0000-0000-000000000001" ] + do + LOCATION=`basename $name`/$LOCATION + GFID_STRING=${name:12:36} + name=$(read_symlink $GFID_STRING) + if [ $? -ne 0 ]; then + exit -1 + fi + done + + LOCATION=`basename $name`/$LOCATION + echo "Location of the directory corresponding to gfid:$2 is $BRICK_PATH/$LOCATION" +} + +main "$@" diff --git a/extras/git-branch-diff.py b/extras/git-branch-diff.py new file mode 100755 index 00000000000..382513e069e --- /dev/null +++ b/extras/git-branch-diff.py @@ -0,0 +1,285 @@ +#!/bin/python2 + +""" + Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +""" + +""" + ABOUT: + This script helps in visualizing backported and missed commits between two + different branches, tags or commit ranges. In the list of missed commits, + it will help you identify patches which are posted for reviews on gerrit server. + + USAGE: + $ ./extras/git-branch-diff.py --help + usage: git-branch-diff.py [-h] [-s SOURCE] -t TARGET [-a AUTHOR] [-p PATH] + [-o OPTIONS] + + git wrapper to diff local or remote branches/tags/commit-ranges + + optional arguments: + -h, --help show this help message and exit + -s SOURCE, --source SOURCE + source pattern, it could be a branch, tag or a commit + range + -t TARGET, --target TARGET + target pattern, it could be a branch, tag or a commit + range + -a AUTHOR, --author AUTHOR + default: git config name/email, to provide multiple + specify comma separated values + -p PATH, --path PATH show source and target diff w.r.t given path, to + provide multiple specify space in between them + -o OPTIONS, --options OPTIONS + add other git options such as --after=<>, --before=<> + etc. experts use; + + SAMPLE EXECUTIONS: + $ ./extras/git-branch-diff.py -t origin/release-3.8 + + $ ./extras/git-branch-diff.py -s local_branch -t origin/release-3.7 + + $ ./extras/git-branch-diff.py -s 4517bf8..e66add8 -t origin/release-3.7 + $ ./extras/git-branch-diff.py -s HEAD..c4efd39 -t origin/release-3.7 + + $ ./extras/git-branch-diff.py -t v3.7.11 --author="author@redhat.com" + $ ./extras/git-branch-diff.py -t v3.7.11 --author="authorX, authorY, authorZ" + + $ ./extras/git-branch-diff.py -t origin/release-3.8 --path="xlators/" + $ ./extras/git-branch-diff.py -t origin/release-3.8 --path="./xlators ./rpc" + + $ ./extras/git-branch-diff.py -t origin/release-3.6 --author="*" + $ ./extras/git-branch-diff.py -t origin/release-3.6 --author="All" + $ ./extras/git-branch-diff.py -t origin/release-3.6 --author="Null" + + $ ./extras/git-branch-diff.py -t v3.7.11 --options="--after=2015-03-01 \ + --before=2016-01-30" + + DECLARATION: + While backporting commit to another branch only subject of the patch may + remain unchanged, all others such as commit message, commit Id, change Id, + bug Id, may be changed. This script works by taking commit subject as the + key value for comparing two git branches, which can be local or remote. + + Note: This script may ignore commits which have altered their commit subjects + while backporting patches. Also this script doesn't have any intelligence to + detect squashed commits. + + AUTHOR: + Prasanna Kumar Kalever <prasanna.kalever@redhat.com> +""" + +from __future__ import print_function +import os +import sys +import argparse +import commands +import subprocess +import requests + +class GitBranchDiff: + def __init__ (self): + " color symbols" + self.tick = u'\033[1;32m[ \u2714 ]\033[0m' + self.cross = u'\033[1;31m[ \u2716 ]\033[0m' + self.green_set = u'\033[1;34m' + self.yello_set = u'\033[4;33m' + self.color_unset = '\033[0m' + + self.parse_cmd_args() + + " replace default values with actual values from command args" + self.g_author = self.argsdict['author'] + self.s_pattern = self.argsdict['source'] + self.t_pattern = self.argsdict['target'] + self.r_path = self.argsdict['path'] + self.options = ' '.join(self.argsdict['options']) + + self.gerrit_server = "http://review.gluster.org" + + def check_dir_exist (self, os_path): + " checks whether given path exist" + path_list = os_path.split() + for path in path_list: + if not os.path.exists(path): + raise argparse.ArgumentTypeError("'%s' path %s is not valid" + %(os_path, path)) + return os_path + + def check_pattern_exist (self): + " defend to check given branch[s] exit" + status_sbr, op = commands.getstatusoutput('git log ' + + self.s_pattern) + status_tbr, op = commands.getstatusoutput('git log ' + + self.t_pattern) + if status_sbr != 0: + print("Error: --source=" + self.s_pattern + " doesn't exit\n") + self.parser.print_help() + exit(status_sbr) + elif status_tbr != 0: + print("Error: --target=" + self.t_pattern + " doesn't exit\n") + self.parser.print_help() + exit(status_tbr) + + def check_author_exist (self): + " defend to check given author exist, format in case of multiple" + contrib_list = ['', '*', 'all', 'All', 'ALL', 'null', 'Null', 'NULL'] + if self.g_author in contrib_list: + self.g_author = "" + else: + ide_list = self.g_author.split(',') + for ide in ide_list: + cmd4 = 'git log ' + self.s_pattern + ' --author=' + ide + c_list = subprocess.check_output(cmd4, shell = True) + if len(c_list) is 0: + print("Error: --author=%s doesn't exit" %self.g_author) + print("see '%s --help'" %__file__) + exit(1) + if len(ide_list) > 1: + self.g_author = "\|".join(ide_list) + + def connected_to_gerrit (self): + "check if gerrit server is reachable" + try: + r = requests.get(self.gerrit_server, timeout=3) + return True + except requests.Timeout as err: + " request timed out" + print("Warning: failed to get list of open review commits on " \ + "gerrit.\n" \ + "hint: Request timed out! gerrit server could possibly " \ + "slow ...\n") + return False + except requests.RequestException as err: + " handle other errors" + print("Warning: failed to get list of open review commits on " \ + "gerrit\n" \ + "hint: check with internet connection ...\n") + return False + + def parse_cmd_args (self): + " command line parser" + author = subprocess.check_output('git config user.email', + shell = True).rstrip('\n') + source = "remotes/origin/master" + options = [' --pretty=format:"%h %s" '] + path = subprocess.check_output('git rev-parse --show-toplevel', + shell = True).rstrip('\n') + self.parser = argparse.ArgumentParser(description = 'git wrapper to ' + 'diff local or remote branches/' + 'tags/commit-ranges') + self.parser.add_argument('-s', + '--source', + help = 'source pattern, it could be a branch,' + ' tag or a commit range', + default = source, + dest = 'source') + self.parser.add_argument('-t', + '--target', + help = 'target pattern, it could be a branch,' + ' tag or a commit range', + required = True, + dest = 'target') + self.parser.add_argument('-a', + '--author', + help = 'default: git config name/email, ' + 'to provide multiple specify comma' + ' separated values', + default = author, + dest = 'author') + self.parser.add_argument('-p', + '--path', + type = self.check_dir_exist, + help = 'show source and target diff w.r.t ' + 'given path, to provide multiple ' + 'specify space in between them', + default = path, + dest = 'path') + self.parser.add_argument('-o', + '--options', + help = 'add other git options such as ' + '--after=<>, --before=<> etc. ' + 'experts use;', + default = options, + dest = 'options', + action='append') + self.argsdict = vars(self.parser.parse_args()) + + def print_output (self): + " display the result list" + print("\n------------------------------------------------------------\n") + print(self.tick + " Successfully Backported changes:") + print(' {' + 'from: ' + self.s_pattern + \ + ' to: '+ self.t_pattern + '}\n') + for key, value in self.s_dict.items(): + if value in self.t_dict.itervalues(): + print("[%s%s%s] %s" %(self.yello_set, + key, + self.color_unset, + value)) + print("\n------------------------------------------------------------\n") + print(self.cross + " Missing patches in " + self.t_pattern + ':\n') + if self.connected_to_gerrit(): + cmd3 = "git review -r origin -l" + review_list = subprocess.check_output(cmd3, shell = True).split('\n') + else: + review_list = [] + + for key, value in self.s_dict.items(): + if value not in self.t_dict.itervalues(): + if any(value in s for s in review_list): + print("[%s%s%s] %s %s(under review)%s" %(self.yello_set, + key, + self.color_unset, + value, + self.green_set, + self.color_unset)) + else: + print("[%s%s%s] %s" %(self.yello_set, + key, + self.color_unset, + value)) + print("\n------------------------------------------------------------\n") + + def main (self): + self.check_pattern_exist() + self.check_author_exist() + + " actual git commands" + cmd1 = 'git log' + self.options + ' ' + self.s_pattern + \ + ' --author=\'' + self.g_author + '\' ' + self.r_path + + " could be backported by anybody so --author doesn't apply here" + cmd2 = 'git log' + self.options + ' ' + self.t_pattern + \ + ' ' + self.r_path + + s_list = subprocess.check_output(cmd1, shell = True).split('\n') + t_list = subprocess.check_output(cmd2, shell = True) + + if len(t_list) is 0: + print("No commits in the target: %s" %self.t_pattern) + print("see '%s --help'" %__file__) + exit() + else: + t_list = t_list.split('\n') + + self.s_dict = dict() + self.t_dict = dict() + + for item in s_list: + self.s_dict.update(dict([item.split(' ', 1)])) + for item in t_list: + self.t_dict.update(dict([item.split(' ', 1)])) + + self.print_output() + + +if __name__ == '__main__': + run = GitBranchDiff() + run.main() diff --git a/extras/gluster-rsyslog-5.8.conf b/extras/gluster-rsyslog-5.8.conf new file mode 100644 index 00000000000..2519999bcac --- /dev/null +++ b/extras/gluster-rsyslog-5.8.conf @@ -0,0 +1,51 @@ +##### gluster.conf ##### + +# +## If you want to log every message to the log file instead of +## intelligently suppressing repeated messages, set off to +## RepeatedMsgReduction. This change requires rsyslog restart +## (eg. run 'service rsyslog restart') +# +#$RepeatedMsgReduction off +$RepeatedMsgReduction on + +# +## The mmcount module provides the capability to count log messages by +## severity or json property of given app-name. The count value is added +## into the log message as json property named '$msgid' +# +$ModLoad mmcount +$mmcountKey gf_code # start counting value of gf_code + +$template Glusterfsd_dynLogFile,"/var/log/glusterfs/bricks/%app-name%.log" +$template Gluster_dynLogFile,"/var/log/glusterfs/%app-name%.log" + +$template GLFS_Template,"%msgid%/%syslogfacility-text:::uppercase%/%syslogseverity-text:::uppercase% [%TIMESTAMP:::date-rfc3339%] %msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" + +# +## Pass logs to mmcount if app-name is 'gluster' +# +if $app-name contains 'gluster' then :mmcount: + +if $app-name contains 'glusterfsd' then ?Glusterfsd_dynLogFile;GLFS_Template +if $app-name contains 'gluster' and not ( $app-name contains 'glusterfsd' ) then ?Gluster_dynLogFile;GLFS_Template + +# +## Sample configuration to send a email alert for every 50th mmcount +# +#$ModLoad ommail +#$ActionMailSMTPServer smtp.example.com +#$ActionMailFrom rsyslog@example.com +#$ActionMailTo glusteradmin@example.com +#$template mailSubject,"50th message of gf_code=9999 on %hostname%" +#$template mailBody,"RSYSLOG Alert\r\nmsg='%msg%'" +#$ActionMailSubject mailSubject +#$ActionExecOnlyOnceEveryInterval 30 +#if $app-name == 'glusterfsd' and $msgid != 0 and $msgid % 50 == 0 \ +#then :ommail:;RSYSLOG_SyslogProtocol23Format +# + +# +## discard logs where app-name is 'gluster' as we processed already +# +if $app-name contains 'gluster' then ~ diff --git a/extras/gluster-rsyslog-7.2.conf b/extras/gluster-rsyslog-7.2.conf new file mode 100644 index 00000000000..8b2841543f4 --- /dev/null +++ b/extras/gluster-rsyslog-7.2.conf @@ -0,0 +1,76 @@ +##### gluster.conf ##### +# +## If you want to log every message to the log file instead of +## intelligently suppressing repeated messages, set off to +## RepeatedMsgReduction. This change requires rsyslog restart +## (eg. run 'service rsyslog restart') +# +#$RepeatedMsgReduction off +$RepeatedMsgReduction on + +$ModLoad mmjsonparse +*.* :mmjsonparse: + +# +## The mmcount module provides the capability to count log messages by +## severity or json property of given app-name. The count value is added +## into the log message as json property named 'mmcount' +## +## More info at http://www.rsyslog.com/doc/mmcount.html +# +#module(load="mmcount") +#action(type="mmcount" appname="glusterd" key="!gf_code") # count each value of gf_code of appname glusterd +#action(type="mmcount" appname="glusterfsd" key="!gf_code") # count each value of gf_code of appname glusterfsd +#action(type="mmcount" appname="glusterfs" key="!gf_code") # count each value of gf_code of appname glusterfs + +template (name="Glusterfsd_dynLogFile" type="string" string="/var/log/glusterfs/bricks/%app-name%.log") +template (name="Gluster_dynLogFile" type="string" string="/var/log/glusterfs/%app-name%.log") + +template(name="GLFS_template" type="list") { + property(name="$!mmcount") + constant(value="/") + property(name="syslogfacility-text" caseConversion="upper") + constant(value="/") + property(name="syslogseverity-text" caseConversion="upper") + constant(value=" ") + constant(value="[") + property(name="timereported" dateFormat="rfc3339") + constant(value="] ") + constant(value="[") + property(name="$!gf_code") + constant(value="] ") + constant(value="[") + property(name="$!gf_message") + constant(value="] ") + property(name="$!msg") + constant(value="\n") +} + +if $app-name contains 'glusterfsd' then { + action(type="omfile" + DynaFile="Glusterfsd_dynLogFile" + Template="GLFS_template") + stop +} + +if $app-name contains 'gluster' then { + action(type="omfile" + DynaFile="Gluster_dynLogFile" + Template="GLFS_template") + stop +} + +# +## send email for every 50th mmcount +#$ModLoad ommail +#if $app-name == 'glusterfsd' and $!mmcount <> 0 and $!mmcount % 50 == 0 then { +# $ActionMailSMTPServer smtp.example.com +# $ActionMailFrom rsyslog@example.com +# $ActionMailTo glusteradmin@example.com +# $template mailSubject,"50th message of gf_code=9999 on %hostname%" +# $template mailBody,"RSYSLOG Alert\r\nmsg='%msg%'" +# $ActionMailSubject mailSubject +# $ActionExecOnlyOnceEveryInterval 30 +# :ommail:;RSYSLOG_SyslogProtocol23Format +#} +# diff --git a/extras/glusterd-sysconfig b/extras/glusterd-sysconfig new file mode 100644 index 00000000000..8237c571104 --- /dev/null +++ b/extras/glusterd-sysconfig @@ -0,0 +1,6 @@ +## Set custom log file and log level (bellow are defaults) +# LOG_FILE='/var/log/glusterfs/glusterd.log' +# LOG_LEVEL='INFO' + +## Set custom options for glusterd +# GLUSTERD_OPTIONS='' diff --git a/extras/glusterd.vol.in b/extras/glusterd.vol.in new file mode 100644 index 00000000000..5d7bad0e4c8 --- /dev/null +++ b/extras/glusterd.vol.in @@ -0,0 +1,15 @@ +volume management + type mgmt/glusterd + option working-directory @GLUSTERD_WORKDIR@ + option transport-type socket + option transport.socket.keepalive-time 10 + option transport.socket.keepalive-interval 2 + option transport.socket.read-fail-log off + option transport.socket.listen-port 24007 + option ping-timeout 0 + option event-threads 1 +# option lock-timer 180 +# option transport.address-family inet6 +# option base-port 49152 + option max-port 60999 +end-volume diff --git a/extras/glusterfs-georep-logrotate b/extras/glusterfs-georep-logrotate new file mode 100644 index 00000000000..3e7ecf373a1 --- /dev/null +++ b/extras/glusterfs-georep-logrotate @@ -0,0 +1,61 @@ +/var/log/glusterfs/geo-replication/*/*.log { + sharedscripts + weekly + maxsize 10M + minsize 100k + + # 6 months of logs are good enough + rotate 26 + + missingok + compress + delaycompress + notifempty + postrotate + for pid in `ps -aef | grep glusterfs | egrep "\-\-aux-gfid-mount" | awk '{print $2}'`; do + /usr/bin/kill -HUP $pid > /dev/null 2>&1 || true + done + endscript +} + + +/var/log/glusterfs/geo-replication-slaves/*.log { + sharedscripts + weekly + maxsize 10M + minsize 100k + + # 6 months of logs are good enough + rotate 26 + + missingok + compress + delaycompress + notifempty + postrotate + for pid in `ps -aef | grep glusterfs | egrep "\-\-aux-gfid-mount" | awk '{print $2}'`; do + /usr/bin/kill -HUP $pid > /dev/null 2>&1 || true + done + endscript +} + + +/var/log/glusterfs/geo-replication-slaves/*/*.log { + sharedscripts + weekly + maxsize 10M + minsize 100k + + # 6 months of logs are good enough + rotate 26 + + missingok + compress + delaycompress + notifempty + postrotate + for pid in `ps -aef | grep glusterfs | egrep "\-\-aux-gfid-mount" | awk '{print $2}'`; do + /usr/bin/kill -HUP $pid > /dev/null 2>&1 || true + done + endscript +} diff --git a/extras/glusterfs-georep-upgrade.py b/extras/glusterfs-georep-upgrade.py new file mode 100755 index 00000000000..634576058d6 --- /dev/null +++ b/extras/glusterfs-georep-upgrade.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 +""" + +Copyright (c) 2020 Red Hat, Inc. <http://www.redhat.com> +This file is part of GlusterFS. + +This file is licensed to you under your choice of the GNU Lesser +General Public License, version 3 or any later version (LGPLv3 or +later), or the GNU General Public License, version 2 (GPLv2), in all +cases as published by the Free Software Foundation. + +""" + +import argparse +import errno +import os, sys +import shutil +from datetime import datetime + +def find_htime_path(brick_path): + dirs = [] + htime_dir = os.path.join(brick_path, '.glusterfs/changelogs/htime') + for file in os.listdir(htime_dir): + if os.path.isfile(os.path.join(htime_dir,file)) and file.startswith("HTIME"): + dirs.append(os.path.join(htime_dir, file)) + else: + raise FileNotFoundError("%s unavailable" % (os.path.join(htime_dir, file))) + return dirs + +def modify_htime_file(brick_path): + htime_file_path_list = find_htime_path(brick_path) + + for htime_file_path in htime_file_path_list: + changelog_path = os.path.join(brick_path, '.glusterfs/changelogs') + temp_htime_path = os.path.join(changelog_path, 'htime/temp_htime_file') + with open(htime_file_path, 'r') as htime_file, open(temp_htime_path, 'w') as temp_htime_file: + #extract epoch times from htime file + paths = htime_file.read().split("\x00") + + for pth in paths: + epoch_no = pth.split(".")[-1] + changelog = os.path.basename(pth) + #convert epoch time to year, month and day + if epoch_no != '': + date=(datetime.fromtimestamp(float(int(epoch_no))).strftime("%Y/%m/%d")) + #update paths in temp htime file + temp_htime_file.write("%s/%s/%s\x00" % (changelog_path, date, changelog)) + #create directory in the format year/month/days + path = os.path.join(changelog_path, date) + + if changelog.startswith("CHANGELOG."): + try: + os.makedirs(path, mode = 0o600); + except OSError as exc: + if exc.errno == errno.EEXIST: + pass + else: + raise + + #copy existing changelogs to new directory structure, delete old changelog files + shutil.copyfile(pth, os.path.join(path, changelog)) + os.remove(pth) + + #rename temp_htime_file with htime file + os.rename(htime_file_path, os.path.join('%s.bak'%htime_file_path)) + os.rename(temp_htime_path, htime_file_path) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('brick_path', help="This upgrade script, which is to be run on\ + server side, takes brick path as the argument, \ + updates paths inside htime file and alters the directory structure \ + above the changelog files inorder to support new optimised format \ + of the directory structure as per \ + https://review.gluster.org/#/c/glusterfs/+/23733/") + args = parser.parse_args() + modify_htime_file(args.brick_path) diff --git a/extras/glusterfs-logrotate b/extras/glusterfs-logrotate new file mode 100644 index 00000000000..6ba6ef18e9f --- /dev/null +++ b/extras/glusterfs-logrotate @@ -0,0 +1,68 @@ +# Rotate client logs +/var/log/glusterfs/*.log { + sharedscripts + weekly + maxsize 10M + minsize 100k + +# 6 months of logs are good enough + rotate 26 + + missingok + compress + delaycompress + notifempty + postrotate + /usr/bin/killall -HUP glusterfs > /dev/null 2>&1 || true + /usr/bin/killall -HUP glusterd > /dev/null 2>&1 || true + endscript +} + +# Rotate server logs +/var/log/glusterfs/bricks/*.log { + sharedscripts + weekly + maxsize 10M + minsize 100k + +# 6 months of logs are good enough + rotate 26 + + missingok + compress + delaycompress + notifempty + postrotate + /usr/bin/killall -HUP glusterfsd > /dev/null 2>&1 || true + endscript +} + +/var/log/glusterfs/samples/*.samp { + daily + rotate 3 + sharedscripts + missingok + compress + delaycompress +} + +# Rotate snapd log +/var/log/glusterfs/snaps/*/*.log { + sharedscripts + weekly + maxsize 10M + minsize 100k + + # 6 months of logs are good enough + rotate 26 + + missingok + compress + delaycompress + notifempty + postrotate + for pid in `ps -aef | grep glusterfs | egrep "snapd" | awk '{print $2}'`; do + /usr/bin/kill -HUP $pid > /dev/null 2>&1 || true + done + endscript +} diff --git a/extras/glusterfs-mode.el b/extras/glusterfs-mode.el index 2f4b40c4e01..a9ed2335ab3 100644 --- a/extras/glusterfs-mode.el +++ b/extras/glusterfs-mode.el @@ -1,112 +1,113 @@ -;;; Copyright (C) 2007-2009 Gluster Inc. <http://www.gluster.com>
-;;;
-;;; This program is free software; you can redistribute it and/or modify
-;;; it under the terms of the GNU General Public License as published by
-;;; the Free Software Foundation; either version 2 of the License, or
-;;; (at your option) any later version.
-;;;
-;;; This program is distributed in the hope that it will be useful,
-;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
-;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-;;; GNU General Public License for more details.
-;;;
-;;; You should have received a copy of the GNU General Public License
-;;; along with this program; if not, write to the Free Software
-;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-;;;
-
-(defvar glusterfs-mode-hook nil)
-
-;; (defvar glusterfs-mode-map
-;; (let ((glusterfs-mode-map (make-keymap)))
-;; (define-key glusterfs-mode-map "\C-j" 'newline-and-indent)
-;; glusterfs-mode-map)
-;; "Keymap for WPDL major mode")
-
-(add-to-list 'auto-mode-alist '("\\.vol\\'" . glusterfs-mode))
-
-(defconst glusterfs-font-lock-keywords-1
- (list
- ; "cluster/{unify,afr,stripe}"
- ; "performance/{io-cache,io-threads,write-behind,read-ahead,stat-prefetch}"
- ; "protocol/{client/server}"
- ; "features/{trash,posix-locks,fixed-id,filter}"
- ; "stroage/posix"
- ; "encryption/rot-13"
- ; "debug/trace"
- '("\\<\\(cluster/\\(unify\\|afr\\|replicate\\|stripe\\|ha\\|dht\\|distribute\\)\\|\\performance/\\(io-\\(cache\\|threads\\)\\|write-behind\\|read-ahead\\|symlink-cache\\)\\|protocol/\\(server\\|client\\)\\|features/\\(trash\\|posix-locks\\|locks\\|path-converter\\|filter\\)\\|storage/\\(posix\\|bdb\\)\\|encryption/rot-13\\|debug/trace\\)\\>" . font-lock-keyword-face))
-"Additional Keywords to highlight in GlusterFS mode.")
-
-(defconst glusterfs-font-lock-keywords-2
- (append glusterfs-font-lock-keywords-1
- (list
- ; "replicate" "namespace" "scheduler" "remote-subvolume" "remote-host"
- ; "auth.addr" "block-size" "remote-port" "listen-port" "transport-type"
- ; "limits.min-free-disk" "directory"
- ; TODO: add all the keys here.
- '("\\<\\(inode-lru-limit\\|replicate\\|namespace\\|scheduler\\|username\\|password\\|allow\\|reject\\|block-size\\|listen-port\\|transport-type\\|transport-timeout\\|directory\\|page-size\\|page-count\\|aggregate-size\\|non-blocking-io\\|client-volume-filename\\|bind-address\\|self-heal\\|read-only-subvolumes\\|read-subvolume\\|thread-count\\|cache-size\\|window-size\\|force-revalidate-timeout\\|priority\\|include\\|exclude\\|remote-\\(host\\|subvolume\\|port\\)\\|auth.\\(addr\\|login\\)\\|limits.\\(min-disk-free\\|transaction-size\\|ib-verbs-\\(work-request-\\(send-\\|recv-\\(count\\|size\\)\\)\\|port\\|mtu\\|device-name\\)\\)\\)\ \\>" . font-lock-constant-face)))
- "option keys in GlusterFS mode.")
-
-(defconst glusterfs-font-lock-keywords-3
- (append glusterfs-font-lock-keywords-2
- (list
- ; "option" "volume" "end-volume" "subvolumes" "type"
- '("\\<\\(option\ \\|volume\ \\|subvolumes\ \\|type\ \\|end-volume\\)\\>" . font-lock-builtin-face)))
- ;'((regexp-opt (" option " "^volume " "^end-volume" "subvolumes " " type ") t) . font-lock-builtin-face))
- "Minimal highlighting expressions for GlusterFS mode.")
-
-
-(defvar glusterfs-font-lock-keywords glusterfs-font-lock-keywords-3
- "Default highlighting expressions for GlusterFS mode.")
-
-(defvar glusterfs-mode-syntax-table
- (let ((glusterfs-mode-syntax-table (make-syntax-table)))
- (modify-syntax-entry ?\# "<" glusterfs-mode-syntax-table)
- (modify-syntax-entry ?* ". 23" glusterfs-mode-syntax-table)
- (modify-syntax-entry ?\n ">#" glusterfs-mode-syntax-table)
- glusterfs-mode-syntax-table)
- "Syntax table for glusterfs-mode")
-
-;; TODO: add an indentation table
-
-(defun glusterfs-indent-line ()
- "Indent current line as GlusterFS code"
- (interactive)
- (beginning-of-line)
- (if (bobp)
- (indent-line-to 0) ; First line is always non-indented
- (let ((not-indented t) cur-indent)
- (if (looking-at "^[ \t]*volume\ ")
- (progn
- (save-excursion
- (forward-line -1)
- (setq not-indented nil)
- (setq cur-indent 0))))
- (if (looking-at "^[ \t]*end-volume")
- (progn
- (save-excursion
- (forward-line -1)
- (setq cur-indent 0))
- (if (< cur-indent 0) ; We can't indent past the left margin
- (setq cur-indent 0)))
- (save-excursion
- (while not-indented ; Iterate backwards until we find an indentation hint
- (progn
- (setq cur-indent 2) ; Do the actual indenting
- (setq not-indented nil)))))
- (if cur-indent
- (indent-line-to cur-indent)
- (indent-line-to 0)))))
-
-(defun glusterfs-mode ()
- (interactive)
- (kill-all-local-variables)
- ;; (use-local-map glusterfs-mode-map)
- (set-syntax-table glusterfs-mode-syntax-table)
- (set (make-local-variable 'indent-line-function) 'glusterfs-indent-line)
- (set (make-local-variable 'font-lock-defaults) '(glusterfs-font-lock-keywords))
- (setq major-mode 'glusterfs-mode)
- (setq mode-name "GlusterFS")
- (run-hooks 'glusterfs-mode-hook))
-
-(provide 'glusterfs-mode)
+;;; Copyright (C) 2007-2017 Red Hat, Inc. <http://www.redhat.com> +;;; Copyright (C) 2007-2011 Gluster Inc. <http://www.gluster.com> +;;; +;;; This program is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation; either version 2 +;;; of the License, or (at your option) any later version. +;;; +;;; This program is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with this program; if not, write to the Free Software +;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +;;; + +(defvar glusterfs-mode-hook nil) + +;; (defvar glusterfs-mode-map +;; (let ((glusterfs-mode-map (make-keymap))) +;; (define-key glusterfs-mode-map "\C-j" 'newline-and-indent) +;; glusterfs-mode-map) +;; "Keymap for WPDL major mode") + +(add-to-list 'auto-mode-alist '("\\.vol\\'" . glusterfs-mode)) + +(defconst glusterfs-font-lock-keywords-1 + (list + ; "cluster/{unify,afr,stripe}" + ; "performance/{io-cache,io-threads,write-behind,read-ahead,stat-prefetch}" + ; "protocol/{client/server}" + ; "features/{trash,posix-locks,fixed-id,filter}" + ; "storage/posix" + ; "encryption/rot-13" + ; "debug/trace" + '("\\<\\(cluster/\\(unify\\|afr\\|replicate\\|stripe\\|ha\\|dht\\|distribute\\)\\|\\performance/\\(io-\\(cache\\|threads\\)\\|write-behind\\|read-ahead\\|symlink-cache\\)\\|protocol/\\(server\\|client\\)\\|features/\\(trash\\|posix-locks\\|locks\\|path-converter\\|filter\\)\\|storage/\\(posix\\|bdb\\)\\|encryption/rot-13\\|debug/trace\\)\\>" . font-lock-keyword-face)) +"Additional Keywords to highlight in GlusterFS mode.") + +(defconst glusterfs-font-lock-keywords-2 + (append glusterfs-font-lock-keywords-1 + (list + ; "replicate" "namespace" "scheduler" "remote-subvolume" "remote-host" + ; "auth.addr" "block-size" "remote-port" "listen-port" "transport-type" + ; "limits.min-free-disk" "directory" + ; TODO: add all the keys here. + '("\\<\\(inode-lru-limit\\|replicate\\|namespace\\|scheduler\\|username\\|password\\|allow\\|reject\\|block-size\\|listen-port\\|transport-type\\|transport-timeout\\|directory\\|page-size\\|page-count\\|aggregate-size\\|non-blocking-io\\|client-volume-filename\\|bind-address\\|self-heal\\|read-only-subvolumes\\|read-subvolume\\|thread-count\\|cache-size\\|window-size\\|force-revalidate-timeout\\|priority\\|include\\|exclude\\|remote-\\(host\\|subvolume\\|port\\)\\|auth.\\(addr\\|login\\)\\|limits.\\(min-disk-free\\|transaction-size\\|ib-verbs-\\(work-request-\\(send-\\|recv-\\(count\\|size\\)\\)\\|port\\|mtu\\|device-name\\)\\)\\)\ \\>" . font-lock-constant-face))) + "option keys in GlusterFS mode.") + +(defconst glusterfs-font-lock-keywords-3 + (append glusterfs-font-lock-keywords-2 + (list + ; "option" "volume" "end-volume" "subvolumes" "type" + '("\\<\\(option\ \\|volume\ \\|subvolumes\ \\|type\ \\|end-volume\\)\\>" . font-lock-builtin-face))) + ;'((regexp-opt (" option " "^volume " "^end-volume" "subvolumes " " type ") t) . font-lock-builtin-face)) + "Minimal highlighting expressions for GlusterFS mode.") + + +(defvar glusterfs-font-lock-keywords glusterfs-font-lock-keywords-3 + "Default highlighting expressions for GlusterFS mode.") + +(defvar glusterfs-mode-syntax-table + (let ((glusterfs-mode-syntax-table (make-syntax-table))) + (modify-syntax-entry ?\# "<" glusterfs-mode-syntax-table) + (modify-syntax-entry ?* ". 23" glusterfs-mode-syntax-table) + (modify-syntax-entry ?\n ">#" glusterfs-mode-syntax-table) + glusterfs-mode-syntax-table) + "Syntax table for glusterfs-mode") + +;; TODO: add an indentation table + +(defun glusterfs-indent-line () + "Indent current line as GlusterFS code" + (interactive) + (beginning-of-line) + (if (bobp) + (indent-line-to 0) ; First line is always non-indented + (let ((not-indented t) cur-indent) + (if (looking-at "^[ \t]*volume\ ") + (progn + (save-excursion + (forward-line -1) + (setq not-indented nil) + (setq cur-indent 0)))) + (if (looking-at "^[ \t]*end-volume") + (progn + (save-excursion + (forward-line -1) + (setq cur-indent 0)) + (if (< cur-indent 0) ; We can't indent past the left margin + (setq cur-indent 0))) + (save-excursion + (while not-indented ; Iterate backwards until we find an indentation hint + (progn + (setq cur-indent 2) ; Do the actual indenting + (setq not-indented nil))))) + (if cur-indent + (indent-line-to cur-indent) + (indent-line-to 0))))) + +(defun glusterfs-mode () + (interactive) + (kill-all-local-variables) + ;; (use-local-map glusterfs-mode-map) + (set-syntax-table glusterfs-mode-syntax-table) + (set (make-local-variable 'indent-line-function) 'glusterfs-indent-line) + (set (make-local-variable 'font-lock-defaults) '(glusterfs-font-lock-keywords)) + (setq major-mode 'glusterfs-mode) + (setq mode-name "GlusterFS") + (run-hooks 'glusterfs-mode-hook)) + +(provide 'glusterfs-mode) diff --git a/extras/glusterfs.vim b/extras/glusterfs.vim index 6bdf2a78d58..899cc65511c 100644 --- a/extras/glusterfs.vim +++ b/extras/glusterfs.vim @@ -1,20 +1,11 @@ " glusterfs.vim: GNU Vim Syntax file for GlusterFS .vol specification -" Copyright (C) 2007-2009 Gluster, Inc. <http://www.gluster.com> -" This file is part of GlusterFS. +" Copyright (c) 2007777777Red Hat, Inc. <http://www.redhat.com> +" This file is part of GlusterFS. " -" GlusterFS is free software; you can redistribute it and/or modify -" it under the terms of the GNU General Public License as published -" by the Free Software Foundation; either version 3 of the License, -" or (at your option) any later version. -" -" GlusterFS is distributed in the hope that it will be useful, but -" WITHOUT ANY WARRANTY; without even the implied warranty of -" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -" General Public License for more details. -" -" You should have received a copy of the GNU General Public License -" along with this program. If not, see -" <http://www.gnu.org/licenses/>. +" This file is licensed to you under your choice of the GNU Lesser +" General Public License, version 3 or any later version (LGPLv3 or +" later), or the GNU General Public License, version 2 (GPLv2), in all +" cases as published by the Free Software Foundation. " " Last Modified: Wed Aug 1 00:47:10 IST 2007 " Version: 0.8 diff --git a/extras/gnfs-loganalyse.py b/extras/gnfs-loganalyse.py new file mode 100755 index 00000000000..6341d007188 --- /dev/null +++ b/extras/gnfs-loganalyse.py @@ -0,0 +1,261 @@ +#!/bin/python +""" + Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. + +""" + +from __future__ import print_function +import os +import string +import sys + + +class NFSRequest: + def requestIsEntryOp (self): + op = self.op + if op == "CREATE" or op == "LOOKUP" or op == "REMOVE" or op == "LINK" or op == "RENAME" or op == "MKDIR" or op == "RMDIR" or op == "SYMLINK" or op == "MKNOD": + return 1 + else: + return 0 + + def __init__ (self, logline, linecount): + self.calllinecount = 0 + self.xid = "" + self.op = "" + self.opdata = "" + self.replydata = "" + self.replylinecount = 0 + self.timestamp = "" + self.entryname = "" + self.gfid = "" + self.replygfid = "" + + tokens = logline.strip ().split (" ") + self.timestamp = tokens[0] + " " + tokens[1] + if "XID:" not in tokens: + return None + + if "args:" not in tokens: + return None + + self.calllinecount = linecount + + xididx = tokens.index ("XID:") + self.xid = tokens [xididx+1].strip(",") + + opidx = tokens.index ("args:") + self.op = tokens [opidx-1].strip (":") + self.opdata = " ".join(tokens [opidx+1:]) + if self.requestIsEntryOp (): + nameidx = tokens.index ("name:") + self.entryname = tokens[nameidx + 1].strip (",") + gfididx = tokens.index ("gfid") + self.gfid = tokens[gfididx +1].strip(",") + + + def getXID (self): + return self.xid + + def setReply (self, logline, linecount): + tokens = logline.strip ().split (" ") + timestamp = tokens[0] + " " + tokens[1] + statidx = tokens.index ("NFS:") + self.replydata = " TimeStamp: " + timestamp + " " + " ".join (tokens [statidx+1:]) + self.replylinecount = linecount + if "gfid" in tokens: + gfididx = tokens.index ("gfid") + self.replygfid = tokens [gfididx + 1].strip(",") + + def dump (self): + print("ReqLine: " + str(self.calllinecount) + " TimeStamp: " + self.timestamp + ", XID: " + self.xid + " " + self.op + " ARGS: " + self.opdata + " RepLine: " + str(self.replylinecount) + " " + self.replydata) + +class NFSLogAnalyzer: + + def __init__ (self, optn, trackfilename, tracknamefh, stats): + self.stats = stats + self.xid_request_map = {} + self.orphan_replies = {} + self.rqlist = [] + self.CALL = 1 + self.REPLY = 2 + self.optn = optn + self.trackfilename = trackfilename + self.tracknamefh = tracknamefh + self.trackedfilehandles = [] + + def handle_call_line (self, logline, linecount): + newreq = NFSRequest (logline, linecount) + xid = newreq.getXID () + if (self.optn == SYNTHESIZE): + self.xid_request_map [xid] = newreq + self.rqlist.append(newreq) + elif self.optn == TRACKFILENAME: + if newreq.requestIsEntryOp(): + if newreq.entryname == self.trackfilename: + self.xid_request_map [xid] = newreq + self.rqlist.append(newreq) + else: + del newreq + elif self.tracknamefh == ENABLE_TRACKNAME_FH: + if len (self.trackedfilehandles) > 0: + if newreq.gfid in self.trackedfilehandles: + self.xid_request_map [xid] = newreq + self.rqlist.append(newreq) + else: + del newreq + else: + del newreq + else: + del newreq + + + def handle_reply_line (self, logline, linecount): + tokens = logline.strip ().split (" ") + + xididx = tokens.index ("XID:") + xid = tokens [xididx + 1].strip(",") + if xid not in self.xid_request_map.keys (): + self.orphan_replies [xid] = logline + else: + rq = self.xid_request_map [xid] + rq.setReply (logline, linecount) + if rq.requestIsEntryOp() and rq.entryname == self.trackfilename: + self.trackedfilehandles.append (rq.replygfid) + + def analyzeLine (self, logline, linecount): + tokens = logline.strip ().split (" ") + msgtype = 0 + + if "XID:" not in tokens: + return + + if "args:" in tokens: + msgtype = self.CALL + elif "NFS:" in tokens: + msgtype = self.REPLY + + if msgtype == self.CALL: + self.handle_call_line (logline, linecount) + elif msgtype == self.REPLY: + self.handle_reply_line (logline, linecount) + + def getStats (self): + if self.stats == 0: + return + rcount = len (self.xid_request_map.keys ()) + orphancount = len (self.orphan_replies.keys ()) + print("Requests: " + str(rcount) + ", Orphans: " + str(orphancount)) + + def dump (self): + self.getStats () + for rq in self.rqlist: + rq.dump () + del rq + + self.rqlist = [] + self.orphan_replies = {} + self.xid_request_map = {} + + +linecount = 0 + +SYNTHESIZE = 1 +TRACKFILENAME = 2 + +ENABLESTATS = 1 +DISABLESTATS = 0 + +ENABLE_TRACKNAME_FH = 1 +DISABLE_TRACKNAME_FH = 0 + +progmsgcount = 1000 +dumpinterval = 200000 +operation = SYNTHESIZE +stats = ENABLESTATS +tracknamefh = DISABLE_TRACKNAME_FH +trackfilename = "" + +""" +Print the progress of the analysing operations every X number of lines read from +the logs, where X is the argument provided to this option. + +Use this to print a status message every say 10000 lines processed or 100000 +lines processed to know how much longer the processing will go on for. + + +USAGE: --progress <NUMLINES> +""" +if "--progress" in sys.argv: + idx = sys.argv.index ("--progress") + progmsgcount = int(sys.argv[idx+1]) + +""" +The replies for a NFS request can be separated by hundreds and even thousands +of other NFS requests and replies. These can be spread over many hundreds and +thousands of log lines. This script maintains a memory dict to map each request +to its reply using the XID. Because this is in-core, there is a limit to the +number of entries in the dict. At regular intervals, it dumps the mapped +requests and the replies into the stdout. The requests whose replies were not +found at the point of dumping are left as orphans, i.e. without info about the +replies. Use this option to tune the number of lines to maximize the number of +requests whose replies are found while balancing the dict size with memory +on the machine. The default works fine for most cases. + +USAGE: --dump <NUMLINES> +""" +if "--dump" in sys.argv: + idx = sys.argv.index ("--dump") + dumpinterval = int(sys.argv[idx+1]) + +""" +The default operation of the script is to output all the requests mapped to +their replies in a single line. This operation mode can be changed by this +argument. It is used to print only those operations that were performed on the +filename given as the argument to this option. Only those entry operations are +printed which contain this filename. + +USAGE: --trackfilename <filename> +""" +if "--trackfilename" in sys.argv: + idx = sys.argv.index ("--trackfilename") + trackfilename = sys.argv[idx + 1] + operation = TRACKFILENAME + +""" +At every dump interval, some stats are printed about the dumped lines. +Use this option to disable printing that to avoid cluttering the +output. +""" +if "--nostats" in sys.argv: + stats = DISABLESTATS + +""" +While tracking a file using --trackfilename, we're only given those +operations which contain the filename. This excludes a large number +of operations which operate on that file using its filehandle instead of +the filename. This option enables outputting those operations also. It +tracks every single file handle that was ever seen in the log for a given +filename. + +USAGE: --trackfilename +""" +if "--tracknamefh" in sys.argv: + tracknamefh = ENABLE_TRACKNAME_FH + +la = NFSLogAnalyzer (operation, trackfilename, tracknamefh, stats) + +for line in sys.stdin: + linecount = linecount + 1 + if linecount % dumpinterval == 0: + sys.stderr.write ("Dumping data..\n") + la.dump () + + if linecount % progmsgcount == 0: + sys.stderr.write ("Integrating line: "+ str(linecount) + "\n") + la.analyzeLine (line, linecount) diff --git a/extras/group-db-workload b/extras/group-db-workload new file mode 100644 index 00000000000..9334d6fb942 --- /dev/null +++ b/extras/group-db-workload @@ -0,0 +1,12 @@ +performance.open-behind=on +performance.write-behind=off +performance.stat-prefetch=off +performance.quick-read=off +performance.strict-o-direct=on +performance.read-ahead=off +performance.io-cache=off +performance.readdir-ahead=off +performance.client-io-threads=on +server.event-threads=4 +client.event-threads=4 +performance.read-after-open=yes diff --git a/extras/group-distributed-virt b/extras/group-distributed-virt new file mode 100644 index 00000000000..a960b76c694 --- /dev/null +++ b/extras/group-distributed-virt @@ -0,0 +1,10 @@ +performance.quick-read=off +performance.read-ahead=off +performance.io-cache=off +performance.low-prio-threads=32 +network.remote-dio=enable +features.shard=on +user.cifs=off +client.event-threads=4 +server.event-threads=4 +performance.client-io-threads=on diff --git a/extras/group-gluster-block b/extras/group-gluster-block new file mode 100644 index 00000000000..1e398019e6b --- /dev/null +++ b/extras/group-gluster-block @@ -0,0 +1,27 @@ +performance.quick-read=off +performance.read-ahead=off +performance.io-cache=off +performance.stat-prefetch=off +performance.open-behind=off +performance.readdir-ahead=off +performance.strict-o-direct=on +performance.client-io-threads=on +performance.io-thread-count=32 +performance.high-prio-threads=32 +performance.normal-prio-threads=32 +performance.low-prio-threads=32 +performance.least-prio-threads=4 +client.event-threads=8 +server.event-threads=8 +network.remote-dio=disable +cluster.eager-lock=enable +cluster.quorum-type=auto +cluster.data-self-heal-algorithm=full +cluster.locking-scheme=granular +cluster.shd-max-threads=8 +cluster.shd-wait-qlength=10000 +features.shard=on +features.shard-block-size=64MB +user.cifs=off +server.allow-insecure=on +cluster.choose-local=off diff --git a/extras/group-metadata-cache b/extras/group-metadata-cache new file mode 100644 index 00000000000..b890b288fc7 --- /dev/null +++ b/extras/group-metadata-cache @@ -0,0 +1,6 @@ +features.cache-invalidation=on +features.cache-invalidation-timeout=600 +performance.stat-prefetch=on +performance.cache-invalidation=on +performance.md-cache-timeout=600 +network.inode-lru-limit=200000 diff --git a/extras/group-nl-cache b/extras/group-nl-cache new file mode 100644 index 00000000000..897807e8933 --- /dev/null +++ b/extras/group-nl-cache @@ -0,0 +1,5 @@ +features.cache-invalidation=on +features.cache-invalidation-timeout=600 +performance.nl-cache=on +performance.nl-cache-timeout=600 +network.inode-lru-limit=200000 diff --git a/extras/group-samba b/extras/group-samba new file mode 100644 index 00000000000..eeee6e06031 --- /dev/null +++ b/extras/group-samba @@ -0,0 +1,11 @@ +features.cache-invalidation=on +features.cache-invalidation-timeout=600 +performance.cache-samba-metadata=on +performance.stat-prefetch=on +performance.cache-invalidation=on +performance.md-cache-timeout=600 +network.inode-lru-limit=200000 +performance.nl-cache=on +performance.nl-cache-timeout=600 +performance.readdir-ahead=on +performance.parallel-readdir=on diff --git a/extras/group-virt.example b/extras/group-virt.example new file mode 100644 index 00000000000..cc37c98a25c --- /dev/null +++ b/extras/group-virt.example @@ -0,0 +1,24 @@ +performance.quick-read=off +performance.read-ahead=off +performance.io-cache=off +performance.low-prio-threads=32 +network.remote-dio=disable +performance.strict-o-direct=on +cluster.eager-lock=enable +cluster.quorum-type=auto +cluster.server-quorum-type=server +cluster.data-self-heal-algorithm=full +cluster.locking-scheme=granular +cluster.shd-max-threads=8 +cluster.shd-wait-qlength=10000 +features.shard=on +user.cifs=off +cluster.choose-local=off +client.event-threads=4 +server.event-threads=4 +performance.client-io-threads=on +network.ping-timeout=20 +server.tcp-user-timeout=20 +server.keepalive-time=10 +server.keepalive-interval=2 +server.keepalive-count=5 diff --git a/extras/hook-scripts/Makefile.am b/extras/hook-scripts/Makefile.am new file mode 100644 index 00000000000..26059d7dbb9 --- /dev/null +++ b/extras/hook-scripts/Makefile.am @@ -0,0 +1,7 @@ +EXTRA_DIST = S40ufo-stop.py S56glusterd-geo-rep-create-post.sh +SUBDIRS = add-brick create delete set start stop reset + +scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/gsync-create/post/ +if USE_GEOREP +scripts_SCRIPTS = S56glusterd-geo-rep-create-post.sh +endif diff --git a/extras/hook-scripts/S40ufo-stop.py b/extras/hook-scripts/S40ufo-stop.py new file mode 100755 index 00000000000..2c79eb1d54a --- /dev/null +++ b/extras/hook-scripts/S40ufo-stop.py @@ -0,0 +1,24 @@ +#!/usr/bin/python3 + +import os +from optparse import OptionParser + +if __name__ == '__main__': + # check if swift is installed + try: + from gluster.swift.common.Glusterfs import get_mnt_point, unmount + except ImportError: + import sys + sys.exit("Openstack Swift does not appear to be installed properly"); + + op = OptionParser(usage="%prog [options...]") + op.add_option('--volname', dest='vol', type=str) + op.add_option('--last', dest='last', type=str) + (opts, args) = op.parse_args() + + + mnt_point = get_mnt_point(opts.vol) + if mnt_point: + unmount(mnt_point) + else: + sys.exit("get_mnt_point returned none for mount point") diff --git a/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh new file mode 100755 index 00000000000..7d6052315bb --- /dev/null +++ b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +#key_val_pair is the arguments passed to the script in the form of +#key value pair + +key_val_pair1=`echo $2 | cut -d ',' -f 1` +key_val_pair2=`echo $2 | cut -d ',' -f 2` +key_val_pair3=`echo $2 | cut -d ',' -f 3` +key_val_pair4=`echo $2 | cut -d ',' -f 4` +key_val_pair5=`echo $2 | cut -d ',' -f 5` +key_val_pair6=`echo $2 | cut -d ',' -f 6` + +mastervol=`echo $1 | cut -d '=' -f 2` +if [ "$mastervol" == "" ]; then + exit; +fi + +key=`echo $key_val_pair1 | cut -d '=' -f 1` +val=`echo $key_val_pair1 | cut -d '=' -f 2` +if [ "$key" != "is_push_pem" ]; then + exit; +fi +if [ "$val" != '1' ]; then + exit; +fi + +key=`echo $key_val_pair2 | cut -d '=' -f 1` +val=`echo $key_val_pair2 | cut -d '=' -f 2` +if [ "$key" != "pub_file" ]; then + exit; +fi +if [ "$val" == "" ]; then + exit; +fi + +pub_file=`echo $val` +pub_file_bname="$(basename $pub_file)" +pub_file_dname="$(dirname $pub_file)" +pub_file_tmp=`echo $val`_tmp + +key=`echo $key_val_pair3 | cut -d '=' -f 1` +val=`echo $key_val_pair3 | cut -d '=' -f 2` +if [ "$key" != "slave_user" ]; then + exit; +fi +if [ "$val" == "" ]; then + exit; +fi +slave_user=`echo $val` + +key=`echo $key_val_pair4 | cut -d '=' -f 1` +val=`echo $key_val_pair4 | cut -d '=' -f 2` +if [ "$key" != "slave_ip" ]; then + exit; +fi +if [ "$val" == "" ]; then + exit; +fi +slave_ip=`echo $val` + +key=`echo $key_val_pair5 | cut -d '=' -f 1` +val=`echo $key_val_pair5 | cut -d '=' -f 2` +if [ "$key" != "slave_vol" ]; then + exit; +fi +if [ "$val" == "" ]; then + exit; +fi +slavevol=`echo $val` + +key=`echo $key_val_pair6 | cut -d '=' -f 1` +val=`echo $key_val_pair6 | cut -d '=' -f 2` +if [ "$key" != "ssh_port" ]; then + exit; +fi +if [ "$val" == "" ]; then + exit; +fi +SSH_PORT=`echo $val` +SSH_OPT="-oPasswordAuthentication=no -oStrictHostKeyChecking=no" + +if [ -f $pub_file ]; then + # For a non-root user copy the pub file to the user's home directory + # For a root user copy the pub files to priv_dir->geo-rep. + if [ "$slave_user" != "root" ]; then + slave_user_home_dir=`ssh -p ${SSH_PORT} ${SSH_OPT} $slave_user@$slave_ip "getent passwd $slave_user | cut -d ':' -f 6"` + scp -P ${SSH_PORT} ${SSH_OPT} $pub_file $slave_user@$slave_ip:$slave_user_home_dir/common_secret.pem.pub_tmp + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_user@$slave_ip "mv $slave_user_home_dir/common_secret.pem.pub_tmp $slave_user_home_dir/${mastervol}_${slavevol}_common_secret.pem.pub" + else + if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then + scp -P ${SSH_PORT} ${SSH_OPT} $pub_file $slave_ip:$pub_file_tmp + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}" + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster vol set ${slavevol} features.read-only on" + else + scp -P ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $pub_file $slave_ip:$pub_file_tmp + ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}" + ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster vol set ${slavevol} features.read-only on" + fi + fi +fi diff --git a/extras/hook-scripts/add-brick/Makefile.am b/extras/hook-scripts/add-brick/Makefile.am new file mode 100644 index 00000000000..6e2701e909e --- /dev/null +++ b/extras/hook-scripts/add-brick/Makefile.am @@ -0,0 +1,2 @@ +SUBDIRS = post pre +CLEANFILES = diff --git a/extras/hook-scripts/add-brick/post/Makefile.am b/extras/hook-scripts/add-brick/post/Makefile.am new file mode 100644 index 00000000000..9b236df096d --- /dev/null +++ b/extras/hook-scripts/add-brick/post/Makefile.am @@ -0,0 +1,6 @@ +EXTRA_DIST = disabled-quota-root-xattr-heal.sh S10selinux-label-brick.sh S13create-subdir-mounts.sh + +hookdir = $(GLUSTERD_WORKDIR)/hooks/1/add-brick/post/ +if WITH_SERVER +hook_SCRIPTS = disabled-quota-root-xattr-heal.sh S10selinux-label-brick.sh S13create-subdir-mounts.sh +endif diff --git a/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh b/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh new file mode 100755 index 00000000000..4a17c993a77 --- /dev/null +++ b/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# +# Install to hooks/<HOOKS_VER>/add-brick/post +# +# Add an SELinux file context for each brick using the glusterd_brick_t type. +# This ensures that the brick is relabeled correctly on an SELinux restart or +# restore. Subsequently, run a restore on the brick path to set the selinux +# labels. +# +### + +PROGNAME="Sselinux" +OPTSPEC="volname:,version:,gd-workdir:,volume-op:" +VOL= + +parse_args () { + ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@") + eval set -- "${ARGS}" + + while true; do + case ${1} in + --volname) + shift + VOL=${1} + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --version) + shift + ;; + --volume-op) + shift + ;; + *) + shift + break + ;; + esac + shift + done +} + +set_brick_labels() +{ + local volname="${1}" + local fctx + local list=() + + fctx="$(semanage fcontext --list -C)" + + # wait for new brick path to be updated under + # ${GLUSTERD_WORKDIR}/vols/${volname}/bricks/ + sleep 5 + + # grab the path for each local brick + brickpath="${GLUSTERD_WORKDIR}/vols/${volname}/bricks/" + brickdirs=$( + find "${brickpath}" -type f -exec grep '^path=' {} \; | \ + cut -d= -f 2 | \ + sort -u + ) + + # create a list of bricks for which custom SELinux + # label doesn't exist + for b in ${brickdirs}; do + pattern="${b}(/.*)?" + echo "${fctx}" | grep "^${pattern}\s" >/dev/null + if [[ $? -ne 0 ]]; then + list+=("${pattern}") + fi + done + + # Add a file context for each brick path in the list and associate with the + # glusterd_brick_t SELinux type. + for p in ${list[@]} + do + semanage fcontext --add -t glusterd_brick_t -r s0 "${p}" + done + + # Set the labels for which SELinux label was added above + for b in ${brickdirs} + do + echo "${list[@]}" | grep "${b}" >/dev/null + if [[ $? -eq 0 ]]; then + restorecon -R "${b}" + fi + done +} + +SELINUX_STATE=$(which getenforce && getenforce) +[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0 + +parse_args "$@" +[ -z "${VOL}" ] && exit 1 + +set_brick_labels "${VOL}" + +exit 0 diff --git a/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh b/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh new file mode 100755 index 00000000000..1a6923ee7aa --- /dev/null +++ b/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +##--------------------------------------------------------------------------- +## This script runs the self-heal of the directories which are expected to +## be present as they are mounted as subdirectory mounts. +##--------------------------------------------------------------------------- + +MOUNT_DIR=`mktemp -d -t ${0##*/}.XXXXXX`; +OPTSPEC="volname:,version:,gd-workdir:,volume-op:" +PROGNAME="add-brick-create-subdir" +VOL_NAME=test +GLUSTERD_WORKDIR="/var/lib/glusterd" + +cleanup_mountpoint () +{ + umount -f $MOUNT_DIR; + if [ 0 -ne $? ] + then + return $? + fi + + rmdir $MOUNT_DIR; + if [ 0 -ne $? ] + then + return $? + fi +} + +##------------------------------------------ +## Parse the arguments +##------------------------------------------ +ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) +eval set -- "$ARGS" + +while true; +do + case $1 in + --volname) + shift + VOL_NAME=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --version) + shift + ;; + --volume-op) + shift + ;; + *) + shift + break + ;; + esac + shift +done + +## See if we have any subdirs to be healed before going further +subdirs=$(grep 'auth.allow' ${GLUSTERD_WORKDIR}/vols/${VOL_NAME}/info | cut -f2 -d'=' | tr ',' '\n' | cut -f1 -d'('); + +if [ -z ${subdirs} ]; then + rmdir $MOUNT_DIR; + exit 0; +fi + +##---------------------------------------- +## Mount the volume in temp directory. +## ----------------------------------- +glusterfs -s localhost --volfile-id=$VOL_NAME --client-pid=-50 $MOUNT_DIR; +if [ 0 -ne $? ] +then + exit $?; +fi + +## ----------------------------------- +# Do the 'stat' on all the directory for now. Ideal fix is to look at subdir +# list from 'auth.allow' option and only stat them. +for subdir in ${subdirs} +do + stat ${MOUNT_DIR}/${subdir} > /dev/null; +done + +## Clean up and exit +cleanup_mountpoint; diff --git a/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh b/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh new file mode 100755 index 00000000000..ca17a903549 --- /dev/null +++ b/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh @@ -0,0 +1,145 @@ +#!/bin/sh + +##--------------------------------------------------------------------------- +## This script updates the 'limit-set' xattr on the newly added node. Please +## refer hook-scripts/add-brick/pre/S28Quota-root-xattr-heal.sh for the complete +## description. +## Do the following only if limit configured on root. +## 1. Do an auxiliary mount. +## 2. Get 'limit-set' xattr on root +## 3. Set xattrs with the same value on the root. +## 4. Disable itself +##--------------------------------------------------------------------------- + +QUOTA_LIMIT_XATTR="trusted.glusterfs.quota.limit-set" +QUOTA_OBJECT_LIMIT_XATTR="trusted.glusterfs.quota.limit-objects" +MOUNT_DIR=$(mktemp -d -t "${0##*/}.XXXXXX"); +OPTSPEC="volname:,version:,gd-workdir:,volume-op:" +PROGNAME="Quota-xattr-heal-add-brick" +VOL_NAME= +VERSION= +VOLUME_OP= +GLUSTERD_WORKDIR= +ENABLED_NAME_PREFIX="S28" +ENABLED_NAME="Quota-root-xattr-heal.sh" + +THIS_SCRIPT=$(echo "${0}" | awk -F'/' '{print $NF}') + +cleanup_mountpoint () +{ + + if umount -f "${MOUNT_DIR}"; then + return $? + fi + + if rmdir "${MOUNT_DIR}"; then + return $? + fi +} + +disable_and_exit () +{ + if [ -e "${ENABLED_STATE}" ] + then + unlink "${ENABLED_STATE}"; + exit $? + fi + + exit 0 +} + +get_and_set_xattr () +{ + XATTR=$1 + + VALUE=$(getfattr -n "${XATTR}" -e hex --absolute-names "${MOUNT_DIR}" 2>&1) + RET=$? + if [ 0 -eq ${RET} ]; then + VALUE=$(echo "${VALUE}" | grep "${XATTR}" | awk -F'=' '{print $NF}') + setfattr -n "${XATTR}" -v "${VALUE}" "${MOUNT_DIR}"; + RET=$? + else + if echo "${VALUE}" | grep -iq "No such attribute" ; then + RET=0 + fi + fi + + return ${RET}; +} + +##------------------------------------------ +## Parse the arguments +##------------------------------------------ +ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@") +eval set -- "$ARGS" + +while true; +do + case $1 in + --volname) + shift + VOL_NAME=$1 + ;; + --version) + shift + VERSION=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --volume-op) + shift + VOLUME_OP=$1 + ;; + *) + shift + break + ;; + esac + shift +done +##---------------------------------------- + +# Avoid long lines +ENABLED_STATE_1="${GLUSTERD_WORKDIR}/hooks/${VERSION}/${VOLUME_OP}/" +ENABLED_STATE_2="post/${ENABLED_NAME_PREFIX}${VOL_NAME}-${ENABLED_NAME}" +ENABLED_STATE="${ENABLED_STATE_1}${ENABLED_STATE_2}" + +if [ "${THIS_SCRIPT}" != *"${VOL_NAME}"* ]; then + exit 0 +fi + +## Is quota enabled? +FLAG=$(grep "^features.quota=" "${GLUSTERD_WORKDIR}/vols/${VOL_NAME}/info" \ +| awk -F'=' '{print $NF}'); +if [ "${FLAG}" != "on" ] +then + disable_and_exit +fi + +## ----------------------------------- +## Mount the volume in temp directory. +## ----------------------------------- +# Avoid long lines +CMD_1="glusterfs -s localhost" +CMD_2="--volfile-id=${VOL_NAME} client-pid=-42 ${MOUNT_DIR}" +CMD="${CMD_1}${CMD_2}" + +if ${CMD} +then + exit $?; +fi +## ----------------------------------- + +RET1=$(get_and_set_xattr "${QUOTA_LIMIT_XATTR}") +RET2=$(get_and_set_xattr "${QUOTA_OBJECT_LIMIT_XATTR}") + +## Clean up and exit +cleanup_mountpoint; + +if [ "${RET1}" -ne 0 ] || [ "${RET2}" -ne 0 ]; then + exit 1 +fi + +disable_and_exit; diff --git a/extras/hook-scripts/add-brick/pre/Makefile.am b/extras/hook-scripts/add-brick/pre/Makefile.am new file mode 100644 index 00000000000..3288581aa57 --- /dev/null +++ b/extras/hook-scripts/add-brick/pre/Makefile.am @@ -0,0 +1,6 @@ +EXTRA_DIST = S28Quota-enable-root-xattr-heal.sh + +hookdir = $(GLUSTERD_WORKDIR)/hooks/1/add-brick/pre/ +if WITH_SERVER +hook_SCRIPTS = S28Quota-enable-root-xattr-heal.sh +endif diff --git a/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh b/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh new file mode 100755 index 00000000000..27e85231f45 --- /dev/null +++ b/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh @@ -0,0 +1,101 @@ +#!/bin/sh + +############################################################################### +## ---------------------------------------------------------------------------- +## The scripts +## I. add-brick/pre/S28Quota-root-xattr-heal.sh (itself) +## II. add-brick/post/disabled-root-xattr-heal.sh AND +## collectively archieves the job of healing the 'limit-set' xattr upon +## add-brick to the gluster volume. +## +## This script is the 'controlling' script. Upon add-brick this script enables +## the corresponding script based on the status of the volume. +## If volume is started - enable add-brick/post script +## else - enable start/post script. +## +## The enabling and disabling of a script is based on the glusterd's logic, +## that it only runs the scripts which starts its name with 'S'. So, +## Enable - symlink the file to 'S'*. +## Disable- unlink symlink +## ---------------------------------------------------------------------------- +############################################################################### + +OPTSPEC="volname:,version:,gd-workdir:,volume-op:" +PROGNAME="Quota-xattr-heal-add-brick-pre" +VOL_NAME= +GLUSTERD_WORKDIR= +VOLUME_OP= +VERSION= +ENABLED_NAME_PREFIX="S28" +ENABLED_NAME="Quota-root-xattr-heal.sh" +DISABLED_NAME="disabled-quota-root-xattr-heal.sh" + +activate () +{ + ln -sf $DISABLED_STATE $1; +} + +##------------------------------------------ +## Parse the arguments +##------------------------------------------ +ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") +eval set -- "$ARGS" + +while true; +do + case $1 in + --volname) + shift + VOL_NAME=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --volume-op) + shift + VOLUME_OP=$1 + ;; + --version) + shift + VERSION=$1 + ;; + *) + shift + break + ;; + esac + shift +done +##---------------------------------------- + +DISABLED_STATE="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/$DISABLED_NAME" +ENABLED_STATE_START="$GLUSTERD_WORKDIR/hooks/$VERSION/start/post/""$ENABLED_NAME_PREFIX$VOL_NAME""-""$ENABLED_NAME" +ENABLED_STATE_ADD_BRICK="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/""$ENABLED_NAME_PREFIX""$VOL_NAME""-""$ENABLED_NAME"; + +## Why to proceed if the required script itself is not present? +ls $DISABLED_STATE; +if [ 0 -ne $? ] +then + exit $?; +fi + +## Is quota enabled? +FLAG=`cat $GLUSTERD_WORKDIR/vols/$VOL_NAME/info | grep "^features.quota=" \ + | awk -F'=' '{print $NF}'`; +if [ "$FLAG" != "on" ] +then + exit $EXIT_SUCCESS; +fi + +## Is volume started? +FLAG=`cat $GLUSTERD_WORKDIR/vols/$VOL_NAME/info | grep "^status=" \ + | awk -F'=' '{print $NF}'`; +if [ "$FLAG" != "1" ] +then + activate $ENABLED_STATE_START; + exit $? +fi + +activate $ENABLED_STATE_ADD_BRICK; +exit $? diff --git a/extras/hook-scripts/create/Makefile.am b/extras/hook-scripts/create/Makefile.am new file mode 100644 index 00000000000..b083a9145d6 --- /dev/null +++ b/extras/hook-scripts/create/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = post diff --git a/extras/hook-scripts/create/post/Makefile.am b/extras/hook-scripts/create/post/Makefile.am new file mode 100644 index 00000000000..fd1892e9589 --- /dev/null +++ b/extras/hook-scripts/create/post/Makefile.am @@ -0,0 +1,8 @@ +EXTRA_DIST = S10selinux-label-brick.sh + +scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/create/post/ +if WITH_SERVER +if USE_SELINUX +scripts_SCRIPTS = S10selinux-label-brick.sh +endif +endif diff --git a/extras/hook-scripts/create/post/S10selinux-label-brick.sh b/extras/hook-scripts/create/post/S10selinux-label-brick.sh new file mode 100755 index 00000000000..f9b4b1a57e3 --- /dev/null +++ b/extras/hook-scripts/create/post/S10selinux-label-brick.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Install to hooks/<HOOKS_VER>/create/post +# +# Add an SELinux file context for each brick using the glusterd_brick_t type. +# This ensures that the brick is relabeled correctly on an SELinux restart or +# restore. Subsequently, run a restore on the brick path to set the selinux +# labels. +# +### + +PROGNAME="Sselinux" +OPTSPEC="volname:" +VOL= + +parse_args () { + ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@") + eval set -- "${ARGS}" + + while true; do + case ${1} in + --volname) + shift + VOL=${1} + ;; + *) + shift + break + ;; + esac + shift + done +} + +set_brick_labels() +{ + volname="${1}" + + # grab the path for each local brick + brickpath="/var/lib/glusterd/vols/${volname}/bricks/" + brickdirs=$( + find "${brickpath}" -type f -exec grep '^path=' {} \; | \ + cut -d= -f 2 | \ + sort -u + ) + + for b in ${brickdirs}; do + # Add a file context for each brick path and associate with the + # glusterd_brick_t SELinux type. + pattern="${b}(/.*)?" + semanage fcontext --add -t glusterd_brick_t -r s0 "${pattern}" + # Set the labels on the new brick path. + restorecon -R "${b}" + done +} + +SELINUX_STATE=$(which getenforce && getenforce) +[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0 + +parse_args "$@" +[ -z "${VOL}" ] && exit 1 + +set_brick_labels "${VOL}" + +exit 0 diff --git a/extras/hook-scripts/delete/Makefile.am b/extras/hook-scripts/delete/Makefile.am new file mode 100644 index 00000000000..c98a05d9205 --- /dev/null +++ b/extras/hook-scripts/delete/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = pre diff --git a/extras/hook-scripts/delete/pre/Makefile.am b/extras/hook-scripts/delete/pre/Makefile.am new file mode 100644 index 00000000000..4fbfbe7311f --- /dev/null +++ b/extras/hook-scripts/delete/pre/Makefile.am @@ -0,0 +1,8 @@ +EXTRA_DIST = S10selinux-del-fcontext.sh + +scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/delete/pre/ +if WITH_SERVER +if USE_SELINUX +scripts_SCRIPTS = S10selinux-del-fcontext.sh +endif +endif diff --git a/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh b/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh new file mode 100755 index 00000000000..056b52afe76 --- /dev/null +++ b/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# +# Install to hooks/<HOOKS_VER>/delete/pre +# +# Delete the file context associated with the brick path on volume deletion. The +# associated file context was added during volume creation. +# +# We do not explicitly relabel the brick, as this could be time consuming and +# unnecessary. +# +### + +PROGNAME="Sselinux" +OPTSPEC="volname:" +VOL= + +function parse_args () { + ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@") + eval set -- "${ARGS}" + + while true; do + case ${1} in + --volname) + shift + VOL=${1} + ;; + *) + shift + break + ;; + esac + shift + done +} + +function delete_brick_fcontext() +{ + local volname=$1 + local fctx + local list=() + + fctx="$(semanage fcontext --list -C)" + # grab the path for each local brick + brickpath="/var/lib/glusterd/vols/${volname}/bricks/" + brickdirs=$(find "${brickpath}" -type f -exec grep '^path=' {} \; | \ + cut -d= -f 2 | sort -u) + for b in ${brickdirs} + do + pattern="${b}(/.*)?" + echo "${fctx}" | grep "^${pattern}\s" >/dev/null + if [[ $? -eq 0 ]]; then + list+=("${pattern}") + fi + done + if [[ ${#list[@]} -gt 0 ]]; then + printf 'fcontext --delete %s\n' "${list[@]}" | semanage -i - + fi + for b in ${brickdirs} + do + restorecon -R "${b}" + done +} + +SELINUX_STATE=$(which getenforce && getenforce) +[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0 + +parse_args "$@" +[ -z "${VOL}" ] && exit 1 + +delete_brick_fcontext "${VOL}" + +# failure to delete the fcontext is not fatal +exit 0 diff --git a/extras/hook-scripts/reset/Makefile.am b/extras/hook-scripts/reset/Makefile.am new file mode 100644 index 00000000000..6e2701e909e --- /dev/null +++ b/extras/hook-scripts/reset/Makefile.am @@ -0,0 +1,2 @@ +SUBDIRS = post pre +CLEANFILES = diff --git a/extras/hook-scripts/reset/post/Makefile.am b/extras/hook-scripts/reset/post/Makefile.am new file mode 100644 index 00000000000..1b336ac1a85 --- /dev/null +++ b/extras/hook-scripts/reset/post/Makefile.am @@ -0,0 +1 @@ +EXTRA_DIST = diff --git a/extras/hook-scripts/reset/pre/Makefile.am b/extras/hook-scripts/reset/pre/Makefile.am new file mode 100644 index 00000000000..1b336ac1a85 --- /dev/null +++ b/extras/hook-scripts/reset/pre/Makefile.am @@ -0,0 +1 @@ +EXTRA_DIST = diff --git a/extras/hook-scripts/set/Makefile.am b/extras/hook-scripts/set/Makefile.am new file mode 100644 index 00000000000..1fcade4b07f --- /dev/null +++ b/extras/hook-scripts/set/Makefile.am @@ -0,0 +1,2 @@ +SUBDIRS = post +CLEANFILES = diff --git a/extras/hook-scripts/set/post/Makefile.am b/extras/hook-scripts/set/post/Makefile.am new file mode 100644 index 00000000000..506a25a8666 --- /dev/null +++ b/extras/hook-scripts/set/post/Makefile.am @@ -0,0 +1,6 @@ +EXTRA_DIST = S30samba-set.sh S32gluster_enable_shared_storage.sh + +hookdir = $(GLUSTERD_WORKDIR)/hooks/1/set/post/ +if WITH_SERVER +hook_SCRIPTS = $(EXTRA_DIST) +endif diff --git a/extras/hook-scripts/set/post/S30samba-set.sh b/extras/hook-scripts/set/post/S30samba-set.sh new file mode 100755 index 00000000000..854f131f6c8 --- /dev/null +++ b/extras/hook-scripts/set/post/S30samba-set.sh @@ -0,0 +1,161 @@ +#!/bin/bash + +#Need to be copied to hooks/<HOOKS_VER>/set/post/ + +#TODO: All gluster and samba paths are assumed for fedora like systems. +#Some efforts are required to make it work on other distros. + +#The preferred way of creating a smb share of a gluster volume has changed. +#The old method was to create a fuse mount of the volume and share the mount +#point through samba. +# +#New method eliminates the requirement of fuse mount and changes in fstab. +#glusterfs_vfs plugin for samba makes call to libgfapi to access the volume. +# +#This hook script enables user to enable or disable smb share by volume set +#option. Keys "user.cifs" and "user.smb" both are valid, but user.smb is +#preferred. + + +PROGNAME="Ssamba-set" +OPTSPEC="volname:,gd-workdir:" +VOL= +CONFIGFILE= +LOGFILEBASE= +PIDDIR= +GLUSTERD_WORKDIR= +USERSMB_SET="" +USERCIFS_SET="" + +function parse_args () { + ARGS=$(getopt -o 'o:' -l $OPTSPEC -n $PROGNAME -- "$@") + eval set -- "$ARGS" + + while true; do + case $1 in + --volname) + shift + VOL=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --) + shift + break + ;; + -o) + shift + read key value < <(echo "$1" | tr "=" " ") + case "$key" in + "user.cifs") + USERCIFS_SET="YES" + ;; + "user.smb") + USERSMB_SET="YES" + ;; + *) + ;; + esac + ;; + *) + shift + break + ;; + esac + shift + done +} + +function find_config_info () { + cmdout=`smbd -b | grep smb.conf` + if [ $? -ne 0 ]; then + echo "Samba is not installed" + exit 1 + fi + CONFIGFILE=`echo $cmdout | awk '{print $2}'` + PIDDIR=`smbd -b | grep PIDDIR | awk '{print $2}'` + LOGFILEBASE=`smbd -b | grep 'LOGFILEBASE' | awk '{print $2}'` +} + +function add_samba_share () { + volname=$1 + STRING="\n[gluster-$volname]\n" + STRING+="comment = For samba share of volume $volname\n" + STRING+="vfs objects = glusterfs\n" + STRING+="glusterfs:volume = $volname\n" + STRING+="glusterfs:logfile = $LOGFILEBASE/glusterfs-$volname.%%M.log\n" + STRING+="glusterfs:loglevel = 7\n" + STRING+="path = /\n" + STRING+="read only = no\n" + STRING+="kernel share modes = no\n" + printf "$STRING" >> ${CONFIGFILE} +} + +function sighup_samba () { + pid=`cat ${PIDDIR}/smbd.pid` + if [ "x$pid" != "x" ] + then + kill -HUP "$pid"; + else + service smb condrestart + fi +} + +function deactivate_samba_share () { + volname=$1 + sed -i -e '/^\[gluster-'"$volname"'\]/{ :a' -e 'n; /available = no/H; /^$/!{$!ba;}; x; /./!{ s/^/available = no/; $!{G;x}; $H; }; s/.*//; x; };' ${CONFIGFILE} +} + +function is_volume_started () { + volname=$1 + echo "$(grep status $GLUSTERD_WORKDIR/vols/"$volname"/info |\ + cut -d"=" -f2)" +} + +function get_smb () { + volname=$1 + uservalue= + + usercifsvalue=$(grep user.cifs $GLUSTERD_WORKDIR/vols/"$volname"/info |\ + cut -d"=" -f2) + usersmbvalue=$(grep user.smb $GLUSTERD_WORKDIR/vols/"$volname"/info |\ + cut -d"=" -f2) + + if [ -n "$usercifsvalue" ]; then + if [ "$usercifsvalue" = "disable" ] || [ "$usercifsvalue" = "off" ]; then + uservalue="disable" + fi + fi + + if [ -n "$usersmbvalue" ]; then + if [ "$usersmbvalue" = "disable" ] || [ "$usersmbvalue" = "off" ]; then + uservalue="disable" + fi + fi + + echo "$uservalue" +} + +parse_args "$@" +if [ "0" = "$(is_volume_started "$VOL")" ]; then + exit 0 +fi + + +if [ "$USERCIFS_SET" = "YES" ] || [ "$USERSMB_SET" = "YES" ]; then + #Find smb.conf, smbd pid directory and smbd logfile path + find_config_info + + if [ "$(get_smb "$VOL")" = "disable" ]; then + deactivate_samba_share $VOL + else + if ! grep --quiet "\[gluster-$VOL\]" ${CONFIGFILE} ; then + add_samba_share $VOL + else + sed -i '/\[gluster-'"$VOL"'\]/,/^$/!b;/available = no/d' ${CONFIGFILE} + fi + fi + sighup_samba +fi diff --git a/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh b/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh new file mode 100755 index 00000000000..1f2564b44ff --- /dev/null +++ b/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +key=`echo $3 | cut -d '=' -f 1` +val=`echo $3 | cut -d '=' -f 2` +if [ "$key" != "cluster.enable-shared-storage" ] && [ "$key" != "enable-shared-storage" ]; then + exit; +fi +if [ "$val" != 'enable' ]; then + if [ "$val" != 'disable' ]; then + exit; + fi +fi + +option=$val + +key_val_pair1=`echo $4 | cut -d ',' -f 1` +key_val_pair2=`echo $4 | cut -d ',' -f 2` + +key=`echo $key_val_pair1 | cut -d '=' -f 1` +val=`echo $key_val_pair1 | cut -d '=' -f 2` +if [ "$key" != "is_originator" ]; then + exit; +fi +is_originator=$val; + +key=`echo $key_val_pair2 | cut -d '=' -f 1` +val=`echo $key_val_pair2 | cut -d '=' -f 2` +if [ "$key" != "local_node_hostname" ]; then + exit; +fi +local_node_hostname=$val; + +# Read gluster peer status to find the peers +# which are in 'Peer in Cluster' mode and +# are connected. + +number_of_connected_peers=0 +while read -r line +do + # Already got two connected peers. Including the current node + # we have 3 peers which is enough to create a shared storage + # with replica 3 + if [ "$number_of_connected_peers" == "2" ]; then + break; + fi + + key=`echo $line | cut -d ':' -f 1` + if [ "$key" == "Hostname" ]; then + hostname=`echo $line | cut -d ':' -f 2 | xargs` + fi + + if [ "$key" == "State" ]; then + peer_state=`echo $line | cut -d ':' -f 2 | cut -d '(' -f 1 | xargs` + conn_state=`echo $line | cut -d '(' -f 2 | cut -d ')' -f 1 | xargs` + + if [ "$peer_state" == "Peer in Cluster" ]; then + if [ "$conn_state" == "Connected" ]; then + ((number_of_connected_peers++)) + connected_peer[$number_of_connected_peers]=$hostname + fi + fi + fi + +done < <(gluster peer status) + +# Include current node in connected peer list +((number_of_connected_peers++)) +connected_peer[$number_of_connected_peers]=$local_node_hostname + +# forming the create vol command +create_cmd="gluster --mode=script --wignore volume create \ + gluster_shared_storage replica $number_of_connected_peers" + +# Adding the brick names in the command +for i in "${connected_peer[@]}" +do + create_cmd=$create_cmd" "$i:"$GLUSTERD_WORKDIR"/ss_brick +done + +if [ "$option" == "disable" ]; then + # Unmount the volume on all the nodes + umount /run/gluster/shared_storage + cat /etc/fstab | grep -v "gluster_shared_storage /run/gluster/shared_storage/" > /run/gluster/fstab.tmp + mv /run/gluster/fstab.tmp /etc/fstab +fi + +if [ "$is_originator" == 1 ]; then + if [ "$option" == "enable" ]; then + # Create and start the volume + $create_cmd + gluster --mode=script --wignore volume start gluster_shared_storage + fi + + if [ "$option" == "disable" ]; then + # Stop and delete the volume + gluster --mode=script --wignore volume stop gluster_shared_storage + gluster --mode=script --wignore volume delete gluster_shared_storage + fi +fi + +function check_volume_status() +{ + status=`gluster volume info gluster_shared_storage | grep Status | cut -d ':' -f 2 | xargs` + echo $status +} + +key=`echo $5 | cut -d '=' -f 1` +val=`echo $5 | cut -d '=' -f 2` +if [ "$key" == "transport.address-family" ]; then + mount_cmd="mount -t glusterfs -o xlator-option=transport.address-family=inet6 \ + $local_node_hostname:/gluster_shared_storage /run/gluster/shared_storage" +else + mount_cmd="mount -t glusterfs $local_node_hostname:/gluster_shared_storage \ + /run/gluster/shared_storage" +fi + +if [ "$option" == "enable" ]; then + retry=0; + # Wait for volume to start before mounting + status=$(check_volume_status) + while [ "$status" != "Started" ]; do + sleep 5; + ((retry++)) + if [ "$retry" == 3 ]; then + break; + fi + status=$(check_volume_status) + done + # Mount the volume on all the nodes + umount /run/gluster/shared_storage + mkdir -p /run/gluster/shared_storage + $mount_cmd + cp /etc/fstab /run/gluster/fstab.tmp + echo "$local_node_hostname:/gluster_shared_storage /run/gluster/shared_storage/ glusterfs defaults 0 0" >> /run/gluster/fstab.tmp + mv /run/gluster/fstab.tmp /etc/fstab +fi diff --git a/extras/hook-scripts/start/Makefile.am b/extras/hook-scripts/start/Makefile.am new file mode 100644 index 00000000000..1fcade4b07f --- /dev/null +++ b/extras/hook-scripts/start/Makefile.am @@ -0,0 +1,2 @@ +SUBDIRS = post +CLEANFILES = diff --git a/extras/hook-scripts/start/post/Makefile.am b/extras/hook-scripts/start/post/Makefile.am new file mode 100644 index 00000000000..792019d3c9f --- /dev/null +++ b/extras/hook-scripts/start/post/Makefile.am @@ -0,0 +1,6 @@ +EXTRA_DIST = S29CTDBsetup.sh S30samba-start.sh S31ganesha-start.sh + +hookdir = $(GLUSTERD_WORKDIR)/hooks/1/start/post/ +if WITH_SERVER +hook_SCRIPTS = $(EXTRA_DIST) +endif diff --git a/extras/hook-scripts/start/post/S29CTDBsetup.sh b/extras/hook-scripts/start/post/S29CTDBsetup.sh new file mode 100755 index 00000000000..69a0d89a3eb --- /dev/null +++ b/extras/hook-scripts/start/post/S29CTDBsetup.sh @@ -0,0 +1,84 @@ +#! /bin/bash +# +# - The script mounts the 'meta-vol' on start 'event' on a known +# directory (eg. /gluster/lock) +# - P.S: There are other 'tasks' that need to be done outside this script +# to get CTDB based failover up and running. + +CTDB_MNT=/gluster/lock +# Make sure ping-timeout is not default for CTDB volume +PING_TIMEOUT_SECS=10 +PROGNAME="ctdb" +OPTSPEC="volname:,gd-workdir:,version:,volume-op:,first:" +HOSTNAME=`hostname` +MNTOPTS="_netdev,transport=tcp,xlator-option=*client*.ping-timeout=${PING_TIMEOUT_SECS}" +VOL= +GLUSTERD_WORKDIR= +VERSION= +VOLUME_OP= +FIRST= +# $META is the volume that will be used by CTDB as a shared filesystem. +# It is not desirable to use this volume for storing 'data' as well. +# META is set to 'all' (viz. a keyword and hence not a legal volume name) +# to prevent the script from running for volumes it was not intended. +# User needs to set META to the volume that serves CTDB lockfile. +META="all" + +function parse_args () { + ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") + eval set -- "$ARGS" + + while true; do + case $1 in + --volname) + shift + VOL=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --version) + shift + VERSION=$1 + ;; + --volume-op) + shift + VOLUME_OP=$1 + ;; + --first) + shift + FIRST=$1 + ;; + *) + shift + break + ;; + esac + + shift + done +} + +function add_fstab_entry () { + volname=$1 + mntpt=$2 + mntopts="${MNTOPTS}" + + mntent="${HOSTNAME}:/${volname} ${mntpt} glusterfs ${mntopts} 0 0" + exists=`grep "${mntpt}" /etc/fstab` + if [ "$exists" == "" ] + then + echo "${mntent}" >> /etc/fstab + fi +} + +parse_args "$@" +if [ "$META" = "$VOL" ] +then + mkdir -p $CTDB_MNT + sleep 5 + mount -t glusterfs -o${MNTOPTS} ${HOSTNAME}:/$VOL "$CTDB_MNT" && \ + add_fstab_entry $VOL $CTDB_MNT + chkconfig ctdb on +fi diff --git a/extras/hook-scripts/start/post/S30samba-start.sh b/extras/hook-scripts/start/post/S30samba-start.sh new file mode 100755 index 00000000000..cac0cbf1464 --- /dev/null +++ b/extras/hook-scripts/start/post/S30samba-start.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +#Need to be copied to hooks/<HOOKS_VER>/start/post + +#TODO: All gluster and samba paths are assumed for fedora like systems. +#Some efforts are required to make it work on other distros. + +#The preferred way of creating a smb share of a gluster volume has changed. +#The old method was to create a fuse mount of the volume and share the mount +#point through samba. +# +#New method eliminates the requirement of fuse mount and changes in fstab. +#glusterfs_vfs plugin for samba makes call to libgfapi to access the volume. +# +#This hook script automagically creates shares for volume on every volume start +#event by adding the entries in smb.conf file and sending SIGHUP to samba. +# +#In smb.conf: +#glusterfs vfs plugin has to be specified as required vfs object. +#Path value is relative to the root of gluster volume;"/" signifies complete +#volume. + +PROGNAME="Ssamba-start" +OPTSPEC="volname:,gd-workdir:,version:,volume-op:,first:" +VOL= +CONFIGFILE= +LOGFILEBASE= +PIDDIR= +GLUSTERD_WORKDIR= +VERSION= +VOLUME_OP= +FIRST= + +function parse_args () { + ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") + eval set -- "$ARGS" + + while true; do + case $1 in + --volname) + shift + VOL=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --version) + shift + VERSION=$1 + ;; + --volume-op) + shift + VOLUME_OP=$1 + ;; + --first) + shift + FIRST=$1 + ;; + *) + shift + break + ;; + esac + + shift + done +} + +function find_config_info () { + cmdout=$(smbd -b 2> /dev/null) + CONFIGFILE=$(echo "$cmdout" | grep CONFIGFILE | awk '{print $2}') + if [ -z "$CONFIGFILE" ]; then + echo "Samba is not installed" + exit 1 + fi + PIDDIR=$(echo "$cmdout" | grep PIDDIR | awk '{print $2}') + LOGFILEBASE=$(echo "$cmdout" | grep 'LOGFILEBASE' | awk '{print $2}') +} + +function add_samba_share () { + volname=$1 + STRING="\n[gluster-$volname]\n" + STRING+="comment = For samba share of volume $volname\n" + STRING+="vfs objects = glusterfs\n" + STRING+="glusterfs:volume = $volname\n" + STRING+="glusterfs:logfile = $LOGFILEBASE/glusterfs-$volname.%%M.log\n" + STRING+="glusterfs:loglevel = 7\n" + STRING+="path = /\n" + STRING+="read only = no\n" + STRING+="kernel share modes = no\n" + printf "$STRING" >> "${CONFIGFILE}" +} + +function sighup_samba () { + pid=$(cat "${PIDDIR}/smbd.pid" 2> /dev/null) + if [ "x$pid" != "x" ] + then + kill -HUP "$pid"; + else + service smb condrestart + fi +} + +function get_smb () { + volname=$1 + uservalue= + + usercifsvalue=$(grep user.cifs "$GLUSTERD_WORKDIR"/vols/"$volname"/info |\ + cut -d"=" -f2) + usersmbvalue=$(grep user.smb "$GLUSTERD_WORKDIR"/vols/"$volname"/info |\ + cut -d"=" -f2) + + if [ -n "$usercifsvalue" ]; then + if [ "$usercifsvalue" = "enable" ] || [ "$usercifsvalue" = "on" ]; then + uservalue="enable" + fi + fi + + if [ -n "$usersmbvalue" ]; then + if [ "$usersmbvalue" = "enable" ] || [ "$usersmbvalue" = "on" ]; then + uservalue="enable" + fi + fi + + echo "$uservalue" +} + +parse_args "$@" + +value=$(get_smb "$VOL") + +if [ -z "$value" ] || [ "$value" != "enable" ]; then + exit 0 +fi + +#Find smb.conf, smbd pid directory and smbd logfile path +find_config_info + +if ! grep --quiet "\[gluster-$VOL\]" "${CONFIGFILE}" ; then + add_samba_share "$VOL" +else + sed -i '/\[gluster-'"$VOL"'\]/,/^$/!b;/available = no/d' "${CONFIGFILE}" +fi +sighup_samba diff --git a/extras/hook-scripts/start/post/S31ganesha-start.sh b/extras/hook-scripts/start/post/S31ganesha-start.sh new file mode 100755 index 00000000000..7ad6f23ad06 --- /dev/null +++ b/extras/hook-scripts/start/post/S31ganesha-start.sh @@ -0,0 +1,122 @@ +#!/bin/bash +PROGNAME="Sganesha-start" +OPTSPEC="volname:,gd-workdir:" +VOL= +declare -i EXPORT_ID +ganesha_key="ganesha.enable" +GANESHA_DIR="/run/gluster/shared_storage/nfs-ganesha" +CONF1="$GANESHA_DIR/ganesha.conf" +GLUSTERD_WORKDIR= + +function parse_args () +{ + ARGS=$(getopt -l $OPTSPEC -o "o" -name $PROGNAME $@) + eval set -- "$ARGS" + + while true; do + case $1 in + --volname) + shift + VOL=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + *) + shift + break + ;; + esac + shift + done +} + + + +#This function generates a new export entry as export.volume_name.conf +function write_conf() +{ +echo -e "# WARNING : Using Gluster CLI will overwrite manual +# changes made to this file. To avoid it, edit the +# file, copy it over to all the NFS-Ganesha nodes +# and run ganesha-ha.sh --refresh-config." + +echo "EXPORT{" +echo " Export_Id = 2;" +echo " Path = \"/$VOL\";" +echo " FSAL {" +echo " name = \"GLUSTER\";" +echo " hostname=\"localhost\";" +echo " volume=\"$VOL\";" +echo " }" +echo " Access_type = RW;" +echo " Disable_ACL = true;" +echo " Squash=\"No_root_squash\";" +echo " Pseudo=\"/$VOL\";" +echo " Protocols = \"3\", \"4\" ;" +echo " Transports = \"UDP\",\"TCP\";" +echo " SecType = \"sys\";" +echo "}" +} + +#It adds the export dynamically by sending dbus signals +function export_add() +{ + dbus-send --print-reply --system --dest=org.ganesha.nfsd \ +/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.AddExport \ +string:$GANESHA_DIR/exports/export.$VOL.conf string:"EXPORT(Export_Id=$EXPORT_ID)" + +} + +# based on src/scripts/ganeshactl/Ganesha/export_mgr.py +function is_exported() +{ + local volume="${1}" + + dbus-send --type=method_call --print-reply --system \ + --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ + org.ganesha.nfsd.exportmgr.ShowExports \ + | grep -w -q "/${volume}" + + return $? +} + +# Check the info file (contains the volume options) to see if Ganesha is +# enabled for this volume. +function ganesha_enabled() +{ + local volume="${1}" + local info_file="${GLUSTERD_WORKDIR}/vols/${VOL}/info" + local enabled="off" + + enabled=$(grep -w ${ganesha_key} ${info_file} | cut -d"=" -f2) + + [ "${enabled}" == "on" ] + + return $? +} + +parse_args $@ + +if ganesha_enabled ${VOL} && ! is_exported ${VOL} +then + if [ ! -e ${GANESHA_DIR}/exports/export.${VOL}.conf ] + then + #Remove export entry from nfs-ganesha.conf + sed -i /$VOL.conf/d $CONF1 + write_conf ${VOL} > ${GANESHA_DIR}/exports/export.${VOL}.conf + EXPORT_ID=`cat $GANESHA_DIR/.export_added` + EXPORT_ID=EXPORT_ID+1 + echo $EXPORT_ID > $GANESHA_DIR/.export_added + sed -i s/Export_Id.*/"Export_Id=$EXPORT_ID;"/ \ + $GANESHA_DIR/exports/export.$VOL.conf + echo "%include \"$GANESHA_DIR/exports/export.$VOL.conf\"" >> $CONF1 + else + EXPORT_ID=$(grep ^[[:space:]]*Export_Id $GANESHA_DIR/exports/export.$VOL.conf |\ + awk -F"[=,;]" '{print $2}' | tr -d '[[:space:]]') + fi + export_add $VOL +fi + +exit 0 diff --git a/extras/hook-scripts/stop/Makefile.am b/extras/hook-scripts/stop/Makefile.am new file mode 100644 index 00000000000..e2ac8e2740b --- /dev/null +++ b/extras/hook-scripts/stop/Makefile.am @@ -0,0 +1,2 @@ +SUBDIRS = pre +CLEANFILES = diff --git a/extras/hook-scripts/stop/pre/Makefile.am b/extras/hook-scripts/stop/pre/Makefile.am new file mode 100644 index 00000000000..9e8d1565e93 --- /dev/null +++ b/extras/hook-scripts/stop/pre/Makefile.am @@ -0,0 +1,6 @@ +EXTRA_DIST = S29CTDB-teardown.sh S30samba-stop.sh + +hookdir = $(GLUSTERD_WORKDIR)/hooks/1/stop/pre/ +if WITH_SERVER +hook_SCRIPTS = $(EXTRA_DIST) +endif diff --git a/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh b/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh new file mode 100755 index 00000000000..0975a00f18d --- /dev/null +++ b/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh @@ -0,0 +1,62 @@ +#! /bin/bash + +CTDB_MNT=/gluster/lock +PROGNAME="ctdb" +OPTSPEC="volname:,last:" +VOL= +LAST= +# $META is the volume that will be used by CTDB as a shared filesystem. +# It is not desirable to use this volume for storing 'data' as well. +# META is set to 'all' (viz. a keyword and hence not a legal volume name) +# to prevent the script from running for volumes it was not intended. +# User needs to set META to the volume that serves CTDB lockfile. +META="all" + +function parse_args () { + ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") + eval set -- "$ARGS" + + while true; do + case $1 in + --volname) + shift + VOL=$1 + ;; + --last) + shift + LAST=$1 + ;; + *) + shift + break + ;; + esac + shift + done +} + + +function remove_fstab_entry () { + mntpt=$1 + fstab="/etc/fstab" + exists=`grep "$mntpt" ${fstab}` + esc_mntpt=$(echo -e $mntpt | sed 's/\//\\\//g') + if [ "$exists" != " " ] + then + sed -i /"$esc_mntpt"/d $fstab + exists=`grep "$mntpt" ${fstab}` + if [ "$exists" != " " ] + then + echo "fstab entry cannot be removed for unknown reason" + exit 1 + fi + fi +} + +parse_args "$@" +if [ "$META" = "$VOL" ] +then + umount "$CTDB_MNT" + chkconfig ctdb off + remove_fstab_entry $CTDB_MNT +fi diff --git a/extras/hook-scripts/stop/pre/S30samba-stop.sh b/extras/hook-scripts/stop/pre/S30samba-stop.sh new file mode 100755 index 00000000000..ea799381d62 --- /dev/null +++ b/extras/hook-scripts/stop/pre/S30samba-stop.sh @@ -0,0 +1,77 @@ +#! /bin/bash + +#Need to be copied to hooks/<HOOKS_VER>/stop/pre + +#TODO: All gluster and samba paths are assumed for fedora like systems. +#Some efforts are required to make it work on other distros. + +#The preferred way of creating a smb share of a gluster volume has changed. +#The old method was to create a fuse mount of the volume and share the mount +#point through samba. +# +#New method eliminates the requirement of fuse mount and changes in fstab. +#glusterfs_vfs plugin for samba makes call to libgfapi to access the volume. +# +#This hook script automagically removes shares for volume on every volume stop +#event by removing the volume related entries(if any) in smb.conf file. + +PROGNAME="Ssamba-stop" +OPTSPEC="volname:,last:" +VOL= +CONFIGFILE= +PIDDIR= +LAST= + +function parse_args () { + ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") + eval set -- "$ARGS" + + while true; do + case $1 in + --volname) + shift + VOL=$1 + ;; + --last) + shift + LAST=$1 + ;; + *) + shift + break + ;; + esac + + shift + done +} + +function find_config_info () { + cmdout=`smbd -b | grep smb.conf` + if [ $? -ne 0 ];then + echo "Samba is not installed" + exit 1 + fi + CONFIGFILE=`echo $cmdout | awk '{print $2}'` + PIDDIR=`smbd -b | grep PIDDIR | awk '{print $2}'` +} + +function deactivate_samba_share () { + volname=$1 + sed -i -e '/^\[gluster-'"$volname"'\]/{ :a' -e 'n; /available = no/H; /^$/!{$!ba;}; x; /./!{ s/^/available = no/; $!{G;x}; $H; }; s/.*//; x; };' ${CONFIGFILE} +} + +function sighup_samba () { + pid=`cat ${PIDDIR}/smbd.pid` + if [ "x$pid" != "x" ] + then + kill -HUP $pid; + else + service smb condrestart + fi +} + +parse_args "$@" +find_config_info +deactivate_samba_share $VOL +sighup_samba diff --git a/extras/identify-hangs.sh b/extras/identify-hangs.sh new file mode 100755 index 00000000000..ebc6bf144aa --- /dev/null +++ b/extras/identify-hangs.sh @@ -0,0 +1,53 @@ +#!/bin/bash +function get_statedump_fnames_without_timestamps +{ + ls | grep -E "[.]dump[.][0-9][0-9]*" | cut -f1-3 -d'.' | sort -u +} + +function get_non_uniq_fields +{ + local statedump_fname_prefix=$1 + print_stack_lkowner_unique_in_one_line "$statedump_fname_prefix" | sort | uniq -c | grep -vE "^\s*1 " | awk '{$1="repeats="$1; print $0}' +} + +function print_stack_lkowner_unique_in_one_line +{ + local statedump_fname_prefix=$1 + sed -e '/./{H;$!d;}' -e 'x;/unique=/!d;/stack=/!d;/lk-owner=/!d;/pid=/!d;' "${statedump_fname_prefix}"* | grep -E "(stack|lk-owner|unique|pid)=" | paste -d " " - - - - +} + +function get_stacks_that_appear_in_multiple_statedumps +{ + #If a stack with same 'unique/lk-owner/stack' appears in multiple statedumps + #print the stack + local statedump_fname_prefix=$1 + while read -r non_uniq_stack; + do + if [ -z "$printed" ]; + then + printed="1" + fi + echo "$statedump_fname_prefix" "$non_uniq_stack" + done < <(get_non_uniq_fields "$statedump_fname_prefix") +} + +statedumpdir=${1} +if [ -z "$statedumpdir" ]; +then + echo "Usage: $0 <statedump-dir>" + exit 1 +fi + +if [ ! -d "$statedumpdir" ]; +then + echo "$statedumpdir: Is not a directory" + echo "Usage: $0 <statedump-dir>" + exit 1 +fi + +cd "$statedumpdir" || exit 1 +for statedump_fname_prefix in $(get_statedump_fnames_without_timestamps); +do + get_stacks_that_appear_in_multiple_statedumps "$statedump_fname_prefix" +done | column -t +echo "NOTE: stacks with lk-owner=\"\"/lk-owner=0000000000000000/unique=0 may not be hung frames and need further inspection" >&2 diff --git a/extras/init.d/Makefile.am b/extras/init.d/Makefile.am index 6f479e7ca5b..8d8cc69571a 100644 --- a/extras/init.d/Makefile.am +++ b/extras/init.d/Makefile.am @@ -1,17 +1,32 @@ -EXTRA_DIST = glusterfsd-Debian glusterfsd-Redhat glusterfsd-SuSE glusterfs-server.plist +EXTRA_DIST = glusterd-Debian glusterd-FreeBSD glusterd-Redhat \ + glusterd-SuSE glusterd.plist glustereventsd-FreeBSD \ + glustereventsd-Redhat glustereventsd-Debian -CLEANFILES = +CLEANFILES = -initdir = @initdir@ +INIT_DIR = @initdir@ +SYSTEMD_DIR = @systemddir@ +LAUNCHD_DIR = @launchddir@ -$(GF_DISTRIBUTION): - $(mkdir_p) $(DESTDIR)$(initdir) - $(INSTALL_PROGRAM) glusterfsd-$(GF_DISTRIBUTION) $(DESTDIR)$(initdir)/glusterfsd +$(GF_DISTRIBUTION): +if WITH_SERVER + @if [ ! -d $(SYSTEMD_DIR) ]; then \ + $(mkdir_p) $(DESTDIR)$(INIT_DIR); \ + $(INSTALL_PROGRAM) glusterd-$(GF_DISTRIBUTION) $(DESTDIR)$(INIT_DIR)/glusterd; \ + fi +endif +if BUILD_EVENTS + @if [ ! -d $(SYSTEMD_DIR) ]; then \ + $(mkdir_p) $(DESTDIR)$(INIT_DIR); \ + $(INSTALL_PROGRAM) glustereventsd-$(GF_DISTRIBUTION) $(DESTDIR)$(INIT_DIR)/glustereventsd; \ + fi +endif install-exec-local: $(GF_DISTRIBUTION) -install-data-local: +install-data-local: if GF_DARWIN_HOST_OS - cp glusterfs-server.plist /Library/LaunchDaemons/com.gluster.glusterfs.plist + $(mkdir_p) $(DESTDIR)$(LAUNCHD_DIR) + $(INSTALL_PROGRAM) glusterd.plist $(DESTDIR)$(LAUNCHD_DIR)/org.gluster.glusterd.plist endif diff --git a/extras/init.d/glusterfsd-Debian.in b/extras/init.d/glusterd-Debian.in index a0c83d53569..918f8592c6e 100755 --- a/extras/init.d/glusterfsd-Debian.in +++ b/extras/init.d/glusterd-Debian.in @@ -1,24 +1,23 @@ #!/bin/sh ### BEGIN INIT INFO -# Provides: glusterfsd +# Provides: glusterd # Required-Start: $local_fs $network # Required-Stop: $local_fs $network # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 -# Short-Description: gluster server -# Description: This file starts / stops the gluster server +# Short-Description: Gluster File System service for volume management +# Description: Gluster File System service for volume management ### END INIT INFO # Author: Chris AtLee <chris@atlee.ca> # Patched by: Matthias Albert < matthias@linux4experts.de> PATH=/sbin:/usr/sbin:/bin:/usr/bin -NAME=glusterfsd +NAME=glusterd SCRIPTNAME=/etc/init.d/$NAME DAEMON=@prefix@/sbin/$NAME PIDFILE=/var/run/$NAME.pid -CONFIGFILE=/etc/glusterfs/glusterfsd.vol -GLUSTERFS_OPTS="-f $CONFIGFILE" +GLUSTERD_OPTS="" PID=`test -f $PIDFILE && cat $PIDFILE` @@ -31,24 +30,16 @@ test -x $DAEMON || exit 0 # Define LSB log_* functions. . /lib/lsb/init-functions -check_config() -{ - if [ ! -f "$CONFIGFILE" ]; then - echo "Config file $CONFIGFILE is missing...exiting!" - exit 0 - fi -} do_start() { - check_config; pidofproc -p $PIDFILE $DAEMON >/dev/null status=$? if [ $status -eq 0 ]; then - log_success_msg "glusterfs server is already running with pid $PID" + log_success_msg "glusterd service is already running with pid $PID" else - log_daemon_msg "Starting glusterfs server" "glusterfsd" - start-stop-daemon --start --quiet --oknodo --pidfile $PIDFILE --startas $DAEMON -- -p $PIDFILE $GLUSTERFS_OPTS + log_daemon_msg "Starting glusterd service" "glusterd" + start-stop-daemon --start --quiet --oknodo --pidfile $PIDFILE --startas $DAEMON -- -p $PIDFILE $GLUSTERD_OPTS log_end_msg $? start_daemon -p $PIDFILE $DAEMON -f $CONFIGFILE return $? @@ -57,7 +48,7 @@ do_start() do_stop() { - log_daemon_msg "Stopping glusterfs server" "glusterfsd" + log_daemon_msg "Stopping glusterd service" "glusterd" start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE log_end_msg $? rm -f $PIDFILE @@ -70,9 +61,9 @@ do_status() pidofproc -p $PIDFILE $DAEMON >/dev/null status=$? if [ $status -eq 0 ]; then - log_success_msg "glusterfs server is running with pid $PID" + log_success_msg "glusterd service is running with pid $PID" else - log_failure_msg "glusterfs server is not running." + log_failure_msg "glusterd service is not running." fi exit $status } diff --git a/extras/init.d/glusterd-FreeBSD.in b/extras/init.d/glusterd-FreeBSD.in new file mode 100644 index 00000000000..21c3da72624 --- /dev/null +++ b/extras/init.d/glusterd-FreeBSD.in @@ -0,0 +1,24 @@ +#!/bin/sh +# +# $FreeBSD$ +# + +# PROVIDE: glusterd + +. /etc/rc.subr + +name="glusterd" +rcvar=`set_rcvar` +command=@prefix@/sbin/${name} +pidfile="/var/run/${name}.pid" +glusterd_flags="-p /var/run/${name}.pid" +start_precmd="glusterd_prestart" + +glusterd_prestart() +{ + mkdir -p @GLUSTERD_WORKDIR@ /var/log/glusterfs + return 0 +} + +load_rc_config $name +run_rc_command "$1" diff --git a/extras/init.d/glusterd-Redhat.in b/extras/init.d/glusterd-Redhat.in new file mode 100755 index 00000000000..94801fe31a5 --- /dev/null +++ b/extras/init.d/glusterd-Redhat.in @@ -0,0 +1,143 @@ +#!/bin/bash +# +# glusterd Startup script for the glusterfs server +# +# chkconfig: - 20 80 +# description: Clustered file-system server + +### BEGIN INIT INFO +# Provides: glusterd +# Required-Start: $local_fs $network +# Required-Stop: $local_fs $network +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: glusterfs server +# Description: Clustered file-system server +### END INIT INFO +# + +# Source function library. +. /etc/rc.d/init.d/functions + +BASE=glusterd + +# Fedora File System Layout dictates /run +[ -e /run ] && RUNDIR="/run" +PIDFILE="${RUNDIR:-/var/run}/${BASE}.pid" + +PID=`test -f $PIDFILE && cat $PIDFILE` + +# Overwriteable from sysconfig +LOG_LEVEL='' +LOG_FILE='' +GLUSTERD_OPTIONS='' +GLUSTERD_NOFILE='65536' + +[ -f /etc/sysconfig/${BASE} ] && . /etc/sysconfig/${BASE} + +[ ! -z $LOG_LEVEL ] && GLUSTERD_OPTIONS="${GLUSTERD_OPTIONS} --log-level ${LOG_LEVEL}" +[ ! -z $LOG_FILE ] && GLUSTERD_OPTIONS="${GLUSTERD_OPTIONS} --log-file ${LOG_FILE}" + +GLUSTERFSD=glusterfsd +GLUSTERFS=glusterfs +GLUSTERD_BIN=@prefix@/sbin/$BASE +GLUSTERD_OPTS="--pid-file=$PIDFILE ${GLUSTERD_OPTIONS}" +GLUSTERD="$GLUSTERD_BIN $GLUSTERD_OPTS" +RETVAL=0 + +LOCKFILE=/var/lock/subsys/${BASE} + +# Start the service $BASE +start() +{ + if pidofproc -p $PIDFILE $GLUSTERD_BIN &> /dev/null; then + echo "glusterd service is already running with pid $PID" + return 0 + else + ulimit -n $GLUSTERD_NOFILE + echo -n $"Starting $BASE:" + daemon $GLUSTERD + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch $LOCKFILE + return $RETVAL + fi +} + +# Stop the service $BASE +stop() +{ + echo -n $"Stopping $BASE:" + if pidofproc -p $PIDFILE $GLUSTERD_BIN &> /dev/null; then + killproc -p $PIDFILE $BASE + else + killproc $BASE + fi + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && rm -f $LOCKFILE + return $RETVAL +} + +restart() +{ + stop + start +} + +reload() +{ + restart +} + +force_reload() +{ + restart +} + +rh_status() +{ + status $BASE +} + +rh_status_q() +{ + rh_status &>/dev/null +} + + +### service arguments ### +case $1 in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + reload) + rh_status_q || exit 7 + $1 + ;; + force-reload) + force_reload + ;; + status) + rh_status + ;; + condrestart|try-restart) + rh_status_q || exit 0 + restart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" + exit 1 +esac + +exit $? diff --git a/extras/init.d/glusterfsd-SuSE.in b/extras/init.d/glusterd-SuSE.in index 43552bb499c..6259bab00b6 100755 --- a/extras/init.d/glusterfsd-SuSE.in +++ b/extras/init.d/glusterd-SuSE.in @@ -1,31 +1,30 @@ #!/bin/bash # ### BEGIN INIT INFO -# Provides: glusterfsd -# Required-Start: $local_fs $network -# Required-Stop: +# Provides: glusterd +# Required-Start: $remote_fs $network +# Required-Stop: $remote_fs $network # Default-Start: 3 5 # Default-Stop: -# Short-Description: GlusterFS server daemon -# Description: All necessary services for GlusterFS clients +# Short-Description: Gluster File System service for volume management +# Description: Gluster File System service for volume management ### END INIT INFO # Get function from functions library . /etc/rc.status -BASE=glusterfsd -GLUSTERFSD_BIN=@prefix@/sbin/$BASE -CONFIGFILE=/etc/glusterfs/glusterfsd.vol -GLUSTERFSD_OPTS="-f $CONFIGFILE" -GSERVER="$GLUSTERFSD_BIN $GLUSTERFSD_OPTS" +BASE=glusterd +GLUSTERD_BIN=@prefix@/sbin/$BASE +GLUSTERD_OPTS="" +GLUSTERD="$GLUSTERD_BIN $GLUSTERD_OPTS" RETVAL=0 # Start the service $BASE start() { echo -n $"Starting $BASE:" - startproc $GSERVER + startproc $GLUSTERD return $? } @@ -33,7 +32,7 @@ start() stop() { echo $"Stopping $BASE:" - killproc $BASE + killproc $BASE return $? } @@ -55,20 +54,24 @@ case $1 in rc_status -v ;; status) - echo -n " glusterfsd" + echo -n " glusterd" if ! checkproc $BASE ;then echo " not running" rc_failed 3 fi rc_status -v ;; + reload) + rc_failed 3 + rc_status -v + ;; restart) $0 stop $0 start rc_status ;; *) - echo $"Usage: $0 {start|stop|status|restart}." + echo $"Usage: $0 {start|stop|status|reload|restart}." exit 1 esac diff --git a/extras/init.d/glusterfs-server.plist.in b/extras/init.d/glusterd.plist.in index 624dfe22d80..3ee9f60c5c5 100644 --- a/extras/init.d/glusterfs-server.plist.in +++ b/extras/init.d/glusterd.plist.in @@ -3,13 +3,11 @@ <plist version="1.0"> <dict> <key>Label</key> - <string>com.gluster.glusterfs</string> + <string>org.gluster.glusterd</string> <key>ProgramArguments</key> <array> - <string>@prefix@/sbin/glusterfsd</string> + <string>@prefix@/sbin/glusterd</string> <string>-N</string> - <string>-f</string> - <string>@prefix@/etc/glusterfs/server.vol</string> </array> </dict> </plist> diff --git a/extras/init.d/glustereventsd-Debian.in b/extras/init.d/glustereventsd-Debian.in new file mode 100644 index 00000000000..6eebdb2b8d8 --- /dev/null +++ b/extras/init.d/glustereventsd-Debian.in @@ -0,0 +1,91 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: glustereventsd +# Required-Start: $local_fs $network +# Required-Stop: $local_fs $network +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Gluster Events Server +# Description: Gluster Events Server +### END INIT INFO + +# Author: Chris AtLee <chris@atlee.ca> +# Patched by: Matthias Albert < matthias@linux4experts.de> + +PATH=/sbin:/usr/sbin:/bin:/usr/bin +NAME=glustereventsd +SCRIPTNAME=/etc/init.d/$NAME +DAEMON=@prefix@/sbin/$NAME +PIDFILE=/var/run/$NAME.pid +GLUSTEREVENTSD_OPTS="" +PID=`test -f $PIDFILE && cat $PIDFILE` + + +# Gracefully exit if the package has been removed. +test -x $DAEMON || exit 0 + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +. /lib/lsb/init-functions + + +do_start() +{ + pidofproc -p $PIDFILE $DAEMON >/dev/null + status=$? + if [ $status -eq 0 ]; then + log_success_msg "glustereventsd service is already running with pid $PID" + else + log_daemon_msg "Starting glustereventsd service" "glustereventsd" + start-stop-daemon --start --quiet --oknodo --pidfile $PIDFILE --startas $DAEMON -- -p $PIDFILE $GLUSTEREVENTSD_OPTS + log_end_msg $? + start_daemon -p $PIDFILE $DAEMON -f $CONFIGFILE + return $? + fi +} + +do_stop() +{ + log_daemon_msg "Stopping glustereventsd service" "glustereventsd" + start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE + log_end_msg $? + rm -f $PIDFILE + killproc -p $PIDFILE $DAEMON + return $? +} + +do_status() +{ + pidofproc -p $PIDFILE $DAEMON >/dev/null + status=$? + if [ $status -eq 0 ]; then + log_success_msg "glustereventsd service is running with pid $PID" + else + log_failure_msg "glustereventsd service is not running." + fi + exit $status +} + +case "$1" in + start) + do_start + ;; + stop) + do_stop + ;; + status) + do_status; + ;; + restart|force-reload) + do_stop + sleep 2 + do_start + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + diff --git a/extras/init.d/glustereventsd-FreeBSD.in b/extras/init.d/glustereventsd-FreeBSD.in new file mode 100644 index 00000000000..2e8303ec6c6 --- /dev/null +++ b/extras/init.d/glustereventsd-FreeBSD.in @@ -0,0 +1,19 @@ +#!/bin/sh +# +# $FreeBSD$ +# + +# PROVIDE: glustereventsd + +. /etc/rc.subr + +name="glustereventsd" +rcvar=`set_rcvar` +command=@prefix@/sbin/${name} +command_interpreter=/usr/local/bin/python +pidfile="/var/run/${name}.pid" +glustereventsd_flags="-p /var/run/${name}.pid" +start_cmd="/usr/sbin/daemon $command ${glustereventsd_flags}" + +load_rc_config $name +run_rc_command "$1" diff --git a/extras/init.d/glustereventsd-Redhat.in b/extras/init.d/glustereventsd-Redhat.in new file mode 100644 index 00000000000..d23ce4c244f --- /dev/null +++ b/extras/init.d/glustereventsd-Redhat.in @@ -0,0 +1,129 @@ +#!/bin/bash +# +# glustereventsd Startup script for the glusterfs Events server +# +# chkconfig: - 20 80 +# description: Gluster Events Server + +### BEGIN INIT INFO +# Provides: glustereventsd +# Required-Start: $local_fs $network +# Required-Stop: $local_fs $network +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: glusterfs Events server +# Description: GlusterFS Events Server +### END INIT INFO +# + +# Source function library. +. /etc/rc.d/init.d/functions + +BASE=glustereventsd + +# Fedora File System Layout dictates /run +[ -e /run ] && RUNDIR="/run" +PIDFILE="${RUNDIR:-/var/run}/${BASE}.pid" + +PID=`test -f $PIDFILE && cat $PIDFILE` + +GLUSTEREVENTSD_BIN=@prefix@/sbin/$BASE +GLUSTEREVENTSD_OPTS="--pid-file=$PIDFILE" +GLUSTEREVENTSD="$GLUSTEREVENTSD_BIN $GLUSTEREVENTSD_OPTS" +RETVAL=0 + +LOCKFILE=/var/lock/subsys/${BASE} + +# Start the service $BASE +start() +{ + if pidofproc -p $PIDFILE $GLUSTEREVENTSD_BIN &> /dev/null; then + echo "glustereventsd service is already running with pid $PID" + return 0 + else + echo -n $"Starting $BASE:" + daemon $GLUSTEREVENTSD & + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch $LOCKFILE + return $RETVAL + fi +} + +# Stop the service $BASE +stop() +{ + echo -n $"Stopping $BASE:" + if pidofproc -p $PIDFILE $GLUSTEREVENTSD_BIN &> /dev/null; then + killproc -p $PIDFILE $BASE + else + killproc $BASE + fi + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && rm -f $LOCKFILE + return $RETVAL +} + +restart() +{ + stop + start +} + +reload() +{ + restart +} + +force_reload() +{ + restart +} + +rh_status() +{ + status $BASE +} + +rh_status_q() +{ + rh_status &>/dev/null +} + + +### service arguments ### +case $1 in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + reload) + rh_status_q || exit 7 + $1 + ;; + force-reload) + force_reload + ;; + status) + rh_status + ;; + condrestart|try-restart) + rh_status_q || exit 0 + restart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" + exit 1 +esac + +exit $? diff --git a/extras/init.d/glusterfsd-Redhat.in b/extras/init.d/glusterfsd-Redhat.in deleted file mode 100755 index 2f5009ef78f..00000000000 --- a/extras/init.d/glusterfsd-Redhat.in +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# -# chkconfig: 35 90 12 -# description: Glusterfsd server -# - -# Get function from functions library -. /etc/rc.d/init.d/functions - -BASE=glusterfsd -GLUSTERFSD_BIN=@exec_prefix@/sbin/$BASE -CONFIGFILE=/etc/glusterfs/glusterfsd.vol -GLUSTERFSD_OPTS="-f $CONFIGFILE" -GSERVER="$GLUSTERFSD_BIN $GLUSTERFSD_OPTS" -RETVAL=0 - -# Start the service $BASE -start() -{ - echo $"Starting $BASE:" - daemon $GSERVER - RETVAL=$? - [ $RETVAL -ne 0 ] && exit $RETVAL -} - -# Stop the service $BASE -stop() -{ - echo $"Stopping $BASE:" - killproc $BASE -} - - -### service arguments ### -case $1 in - start) - start - ;; - stop) - stop - ;; - status) - status $BASE - ;; - restart) - $0 stop - $0 start - ;; - *) - echo $"Usage: $0 {start|stop|status|restart}." - exit 1 -esac - -exit 0 diff --git a/extras/logger.conf.example b/extras/logger.conf.example new file mode 100644 index 00000000000..248be5bda69 --- /dev/null +++ b/extras/logger.conf.example @@ -0,0 +1,13 @@ +# +# Sample logger.conf file to configure enhanced Logging in GlusterFS +# +# To enable enhanced logging capabilities, +# +# 1. rename this file to /etc/glusterfs/logger.conf +# +# 2. rename /etc/rsyslog.d/gluster.conf.example to +# /etc/rsyslog.d/gluster.conf +# +# This change requires restart of all gluster services/volumes and +# rsyslog. +# diff --git a/extras/mount-shared-storage.sh b/extras/mount-shared-storage.sh new file mode 100755 index 00000000000..cc40e13c3e3 --- /dev/null +++ b/extras/mount-shared-storage.sh @@ -0,0 +1,39 @@ +#!/bin/bash +#Post reboot there is a chance in which mounting of shared storage will fail +#This will impact starting of features like NFS-Ganesha. So this script will +#try to mount the shared storage if it fails + +exitStatus=0 + +while IFS= read -r glm +do + IFS=$' \t' read -r -a arr <<< "$glm" + + #Validate storage type is glusterfs + if [ "${arr[2]}" == "glusterfs" ] + then + + #check whether shared storage is mounted + #if it is mounted then mountpoint -q will return a 0 success code + if mountpoint -q "${arr[1]}" + then + echo "${arr[1]} is already mounted" + continue + fi + + mount -t glusterfs -o "${arr[3]}" "${arr[0]}" "${arr[1]}" + #wait for few seconds + sleep 10 + + #recheck mount got succeed + if mountpoint -q "${arr[1]}" + then + echo "${arr[1]} has been mounted" + continue + else + echo "${arr[1]} failed to mount" + exitStatus=1 + fi + fi +done <<< "$(sed '/^#/ d' </etc/fstab | grep 'glusterfs')" +exit $exitStatus diff --git a/extras/ocf/Makefile.am b/extras/ocf/Makefile.am new file mode 100644 index 00000000000..c49a835fbca --- /dev/null +++ b/extras/ocf/Makefile.am @@ -0,0 +1,11 @@ +EXTRA_DIST = glusterd.in volume.in + +# The root of the OCF resource agent hierarchy +# Per the OCF standard, it's always "lib", +# not "lib64" (even on 64-bit platforms). +ocfdir = $(prefix)/lib/ocf + +# The ceph provider directory +radir = $(ocfdir)/resource.d/$(PACKAGE_NAME) + +ra_SCRIPTS = glusterd volume diff --git a/extras/ocf/glusterd.in b/extras/ocf/glusterd.in new file mode 100755 index 00000000000..c119a285d32 --- /dev/null +++ b/extras/ocf/glusterd.in @@ -0,0 +1,212 @@ +#!/bin/sh +# +# glusterd +# +# Description: Manages a glusterd server as a (typically cloned) +# HA resource +# +# Authors: Florian Haas (hastexo Professional Services GmbH) +# +# License: GNU General Public License (GPL) + +####################################################################### +# Initialization: + +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +# Convenience variables +# When sysconfdir and localstatedir aren't passed in as +# configure flags, they're defined in terms of prefix +prefix=@prefix@ +####################################################################### + + +OCF_RESKEY_binary_default="glusterd" +OCF_RESKEY_pid_default="@localstatedir@/run/glusterd.pid" +OCF_RESKEY_socket_default="" +OCF_RESKEY_additional_parameters_default="" + +: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}} +: ${OCF_RESKEY_pid=${OCF_RESKEY_pid_default}} + +glusterd_meta_data() { + cat <<EOF +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="glusterd" version="0.1"> + <version>0.1</version> + <longdesc lang="en"> + </longdesc> + <shortdesc lang="en">Manages a Gluster server</shortdesc> + <parameters> + <parameter name="binary"> + <longdesc lang="en"> + Name of the glusterd executable. Specify a full absolute + path if the binary is not in your \$PATH. + </longdesc> + <shortdesc lang="en">glusterd executable</shortdesc> + <content type="string" default="$OCF_RESKEY_binary_default"/> + </parameter> + <parameter name="pid"> + <longdesc lang="en"> + Path to the glusterd PID file. + </longdesc> + <shortdesc lang="en">PID file</shortdesc> + <content type="string" default="$OCF_RESKEY_pid_default"/> + </parameter> + <parameter name="socket"> + <longdesc lang="en"> + Path to the glusterd UNIX socket file. If unspecified, + glusterd will not listen on any socket. + </longdesc> + <shortdesc lang="en">Socket file</shortdesc> + <content type="string"/> + </parameter> + </parameters> + <actions> + <action name="start" timeout="20" /> + <action name="stop" timeout="20" /> + <action name="monitor" timeout="20" interval="10" /> + <action name="reload" timeout="20" /> + <action name="meta-data" timeout="5" /> + <action name="validate-all" timeout="20" /> + </actions> +</resource-agent> +EOF + +} + +glusterd_start() { + local glusterd_options + # exit immediately if configuration is not valid + glusterd_validate_all || exit $? + + # if resource is already running, bail out early + if glusterd_monitor; then + ocf_log info "Resource is already running" + return $OCF_SUCCESS + fi + + # actually start up the resource here (make sure to immediately + # exit with an $OCF_ERR_ error code if anything goes seriously + # wrong) + glusterd_options="-p $OCF_RESKEY_pid" + if [ -n "$OCF_RESKEY_socket" ]; then + glusterd_options="$glusterd_options -S $OCF_RESKEY_socket" + fi + if [ -n "$OCF_RESKEY_additional_parameters" ]; then + glusterd_options="$glusterd_options $OCF_RESKEY_additional_parameters" + fi + + ocf_run $OCF_RESKEY_binary $glusterd_options || exit $OCF_ERR_GENERIC + + # After the resource has been started, check whether it started up + # correctly. If the resource starts asynchronously, the agent may + # spin on the monitor function here -- if the resource does not + # start up within the defined timeout, the cluster manager will + # consider the start action failed + while ! glusterd_monitor; do + ocf_log debug "Resource has not started yet, waiting" + sleep 1 + done + + # only return $OCF_SUCCESS if _everything_ succeeded as expected + return $OCF_SUCCESS +} + +glusterd_stop() { + local rc + local pid + + # exit immediately if configuration is not valid + glusterd_validate_all || exit $? + + glusterd_monitor + rc=$? + case "$rc" in + "$OCF_SUCCESS") + # Currently running. Normal, expected behavior. + ocf_log debug "Resource is currently running" + ;; + "$OCF_NOT_RUNNING") + # Currently not running. Nothing to do. + ocf_log info "Resource is already stopped" + return $OCF_SUCCESS + ;; + esac + + # actually shut down the resource here (make sure to immediately + # exit with an $OCF_ERR_ error code if anything goes seriously + # wrong) + pid=`cat $OCF_RESKEY_pid` + ocf_run kill -s TERM $pid || exit OCF_ERR_GENERIC + + # After the resource has been stopped, check whether it shut down + # correctly. If the resource stops asynchronously, the agent may + # spin on the monitor function here -- if the resource does not + # shut down within the defined timeout, the cluster manager will + # consider the stop action failed + while glusterd_monitor; do + ocf_log debug "Resource has not stopped yet, waiting" + sleep 1 + done + + # only return $OCF_SUCCESS if _everything_ succeeded as expected + return $OCF_SUCCESS + +} + +glusterd_monitor() { + local pid + + [ -e $OCF_RESKEY_pid ] || return $OCF_NOT_RUNNING + + pid=`cat $OCF_RESKEY_pid` + ocf_run kill -s 0 $pid || return $OCF_NOT_RUNNING + + ocf_log debug "$OCF_RESKEY_binary running with PID $pid" + return $OCF_SUCCESS +} + +glusterd_validate_all() { + # Test for required binaries + check_binary $OCF_RESKEY_binary + + return $OCF_SUCCESS +} + + + +# Make sure meta-data and usage always succeed +case $__OCF_ACTION in +meta-data) glusterd_meta_data + exit $OCF_SUCCESS + ;; +usage|help) glusterd_usage + exit $OCF_SUCCESS + ;; +esac + +# Anything other than meta-data and usage must pass validation +glusterd_validate_all || exit $? + +# Translate each action into the appropriate function call +case $__OCF_ACTION in +start) glusterd_start;; +stop) glusterd_stop;; +status|monitor) glusterd_monitor;; +reload) ocf_log info "Reloading..." + glusterd_start + ;; +validate-all) ;; +notify) exit $OCF_SUCCESS;; +*) glusterd_usage + exit $OCF_ERR_UNIMPLEMENTED + ;; +esac +rc=$? + +# The resource agent may optionally log a debug message +ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION returned $rc" +exit $rc diff --git a/extras/ocf/volume.in b/extras/ocf/volume.in new file mode 100755 index 00000000000..76cc649e55f --- /dev/null +++ b/extras/ocf/volume.in @@ -0,0 +1,276 @@ +#!/bin/sh +# +# glusterd +# +# Description: Manages a glusterd server as a (typically cloned) +# HA resource +# +# Authors: Florian Haas (hastexo Professional Services GmbH) +# Jiri Lunacek (Hosting90 Systems s.r.o.) +# +# License: GNU General Public License (GPL) + +####################################################################### +# Initialization: + +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +# Convenience variables +# When sysconfdir and localstatedir aren't passed in as +# configure flags, they're defined in terms of prefix +prefix=@prefix@ +SHORTHOSTNAME=`hostname -s` +####################################################################### + +OCF_RESKEY_binary_default="gluster" + +: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}} + +volume_meta_data() { + cat <<EOF +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="volume" version="0.1"> + <version>0.1</version> + <longdesc lang="en"> +Manages a GlusterFS volume and monitors its bricks. When a resource of +this type is configured as a clone (as is commonly the case), then it +must have clone ordering enabled. + </longdesc> + <shortdesc lang="en">Manages a GlusterFS volume</shortdesc> + <parameters> + <parameter name="volname" required="1"> + <longdesc lang="en"> + The name of the volume to manage. + </longdesc> + <shortdesc lang="en">volume name</shortdesc> + <content type="string"/> + </parameter> + <parameter name="binary"> + <longdesc lang="en"> + Name of the gluster executable. Specify a full absolute + path if the binary is not in your \$PATH. + </longdesc> + <shortdesc lang="en">gluster executable</shortdesc> + <content type="string" default="$OCF_RESKEY_binary_default"/> + </parameter> + <parameter name="peer_map"> + <longdesc lang="en"> + Mapping of hostname - peer name in the gluster cluster + in format hostname1:peername1,hostname2:peername2,... + </longdesc> + <shortdesc lang="en">gluster peer map</shortdesc> + <content type="string" default=""/> + </parameter> + </parameters> + <actions> + <action name="start" timeout="20" /> + <action name="stop" timeout="20" /> + <action name="monitor" timeout="20" interval="10" /> + <action name="reload" timeout="20" /> + <action name="meta-data" timeout="5" /> + <action name="validate-all" timeout="20" /> + </actions> +</resource-agent> +EOF + +} + +if [ -n "${OCF_RESKEY_peer_map}" ]; then + SHORTHOSTNAME=`echo "${OCF_RESKEY_peer_map}" | egrep -o "$SHORTHOSTNAME\:[^,]+" | awk -F: '{print $2}'` +fi + +volume_getdir() { + local voldir + voldir="@GLUSTERD_WORKDIR@/vols/${OCF_RESKEY_volname}" + + [ -d ${voldir} ] || return 1 + + echo "${voldir}" + return 0 +} + +volume_getpid_dir() { + local volpid_dir + volpid_dir="/var/run/gluster/vols/${OCF_RESKEY_volname}" + + [ -d ${volpid_dir} ] || return 1 + + echo "${volpid_dir}" + return 0 +} + +volume_getbricks() { + local infofile + local voldir + voldir=`volume_getdir` + infofile="${voldir}/info" + + [ -e ${infofile} ] || return 1 + + echo "`sed -n -e "s/^brick-.\+=${SHORTHOSTNAME}://p" < ${infofile}`" + return 0 +} + +volume_getpids() { + local bricks + local pidfile + local infofile + local volpid_dir + + volpid_dir=`volume_getpid_dir` + bricks=`volume_getbricks` + + if [ -z "$bricks" ]; then + return 1 + fi + + for brick in ${bricks}; do + pidfile="${volpid_dir}/${SHORTHOSTNAME}${brick}.pid" + [ -e $pidfile ] || return 1 + cat $pidfile + done + + return 0 +} + +volume_start() { + local volume_options + + # exit immediately if configuration is not valid + volume_validate_all || exit $? + + # if resource is already running, bail out early + if volume_monitor; then + ocf_log info "Resource is already running" + return $OCF_SUCCESS + fi + + # actually start up the resource here + ocf_run "$OCF_RESKEY_binary" \ + volume start "$OCF_RESKEY_volname" force || exit $OCF_ERR_GENERIC + + # After the resource has been started, check whether it started up + # correctly. If the resource starts asynchronously, the agent may + # spin on the monitor function here -- if the resource does not + # start up within the defined timeout, the cluster manager will + # consider the start action failed + while ! volume_monitor; do + ocf_log debug "Resource has not started yet, waiting" + sleep 1 + done + + # only return $OCF_SUCCESS if _everything_ succeeded as expected + return $OCF_SUCCESS +} + +volume_stop() { + local rc + local pid + + # exit immediately if configuration is not valid + volume_validate_all || exit $? + + volume_monitor + rc=$? + case "$rc" in + "$OCF_SUCCESS") + # Currently running. Normal, expected behavior. + ocf_log debug "Resource is currently running" + ;; + "$OCF_NOT_RUNNING") + # Currently not running. Nothing to do. + ocf_log info "Resource is already stopped" + return $OCF_SUCCESS + ;; + esac + + # actually shut down the resource here (make sure to immediately + # exit with an $OCF_ERR_ error code if anything goes seriously + # wrong) + pids=`volume_getpids` + for pid in $pids; do + ocf_run kill -s TERM $pid + done + + # After the resource has been stopped, check whether it shut down + # correctly. If the resource stops asynchronously, the agent may + # spin on the monitor function here -- if the resource does not + # shut down within the defined timeout, the cluster manager will + # consider the stop action failed + while volume_monitor; do + ocf_log debug "Resource has not stopped yet, waiting" + sleep 1 + done + + # only return $OCF_SUCCESS if _everything_ succeeded as expected + return $OCF_SUCCESS + +} + +volume_monitor() { + local pid + + pids=`volume_getpids` || return $OCF_NOT_RUNNING + + for pid in $pids; do + ocf_run kill -s 0 $pid || return $OCF_NOT_RUNNING + done + + ocf_log debug "Local bricks for volume ${OCF_RESKEY_volname} running with PIDs $pids" + return $OCF_SUCCESS +} + +volume_validate_all() { + # Test for configuration errors first + if [ -z "${OCF_RESKEY_volname}" ]; then + ocf_log err 'Missing required parameter "volname"' + return $OCF_ERR_CONFIGURED + fi + + # Test for required binaries + check_binary $OCF_RESKEY_binary + + if [ -z "$SHORTHOSTNAME" ]; then + ocf_log err 'Unable to get host in node map' + return $OCF_ERR_CONFIGURED + fi + + return $OCF_SUCCESS +} + + + +# Make sure meta-data and usage always succeed +case $__OCF_ACTION in +meta-data) volume_meta_data + exit $OCF_SUCCESS + ;; +usage|help) volume_usage + exit $OCF_SUCCESS + ;; +esac + +# Anything other than meta-data and usage must pass validation +volume_validate_all || exit $? + +# Translate each action into the appropriate function call +case $__OCF_ACTION in +start) volume_start;; +stop) volume_stop;; +status|monitor) volume_monitor;; +reload) ocf_log info "Reloading..." + volume_start + ;; +validate-all) ;; +notify) exit $OCF_SUCCESS;; +*) volume_usage + exit $OCF_ERR_UNIMPLEMENTED + ;; +esac +rc=$? + +# The resource agent may optionally log a debug message +ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION returned $rc" +exit $rc diff --git a/extras/peer_add_secret_pub.in b/extras/peer_add_secret_pub.in new file mode 100644 index 00000000000..c9674af353d --- /dev/null +++ b/extras/peer_add_secret_pub.in @@ -0,0 +1,70 @@ +#!/bin/bash + +user=$1 +pub_file=$2 + +if [ "$user" == "" ]; then + echo "Invalid User"; + exit 1; +fi + +if [ "$pub_file" == "" ]; then + echo "Invalid pub file"; + exit 1; +fi + +home_dir=`getent passwd $user | cut -d ':' -f 6`; + +if [ "$home_dir" == "" ]; then + echo "Invalid home dir"; + exit 1; +fi + +authorized_keys_file=$(cat /etc/ssh/sshd_config | \ + grep -e "^AuthorizedKeysFile" | \ + awk '{print $2}' | tail -1); + +# If not set, use default location +if [ "x$authorized_keys_file" == "x" ]; then + authorized_keys_file="%h/.ssh/authorized_keys" +fi + +# If default location +if [ "$authorized_keys_file" == ".ssh/authorized_keys" ]; then + authorized_keys_file="%h/$authorized_keys_file" +fi + +# Replace %u with user name (ex: /etc/ssh/keys/%u/authorized_keys) +authorized_keys_file="${authorized_keys_file//%u/$user}"; + +# Replace %h with home dir (ex: %h/.ssh/authorized_keys) +authorized_keys_file="${authorized_keys_file//%h/$home_dir}"; +ssh_dir=$(dirname $authorized_keys_file); + +if [ ! -d $ssh_dir ]; then + mkdir $ssh_dir; + chmod 700 $ssh_dir; + chown $user: $ssh_dir; +fi + +if [ ! -d $authorized_keys_file ]; then + touch $authorized_keys_file; + chmod 600 $authorized_keys_file; + chown $user: $authorized_keys_file; +fi + +# Restore SELinux security contexts. This is required +# for passwdless SSH to work. + +if type restorecon >/dev/null 2>&1; then + restorecon -F $ssh_dir $authorized_keys_file; +fi + +# Add to authorized_keys file only if not exists already +while read line +do + grep -Fxq "$line" $authorized_keys_file; + [ $? -ne 0 ] && echo "$line" >> $authorized_keys_file; +done < "$GLUSTERD_WORKDIR"/$pub_file; + +exit 0; diff --git a/extras/post-upgrade-script-for-quota.sh b/extras/post-upgrade-script-for-quota.sh new file mode 100755 index 00000000000..15652429056 --- /dev/null +++ b/extras/post-upgrade-script-for-quota.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +#The post-upgrade script must be executed after all the nodes in the cluster +#have upgraded. +#Also, all the clients accessing the given volume must also be upgraded +#before the script is run. +#Make sure glusterd and the brick processes are running on all nodes in the +#cluster post upgrade. +#Execute this script on the node where the pre-upgrade script for quota was run. + +VOL=$1; + +BACKUP_DIR=/var/tmp/glusterfs/quota-config-backup + +function set_limits { + local var=$(gluster volume info $1 | grep 'features.quota'| cut -d" " -f2); + + if [ -z "$var" ] || [ "$var" = "off" ]; then + if [ $2 -eq '0' ]; then + echo "Volume $1 does not have quota enabled. " \ + "Exiting ..." + exit 1 + fi + else + gluster volume set $1 default-soft-limit 80% + if [ $? -ne '0' ]; then + echo "Post-upgrade process failed." \ + "Please address the error and run " \ + "post-upgrade-script.sh on volume $VOL again." + exit 1 + fi + + gluster volume start $1 force + sleep 3; + + local path_array=( $(cat $BACKUP_DIR/vol_$1 | tail -n +3 | awk '{print $1}') ) + local limit_array=( $(cat $BACKUP_DIR/vol_$1 | tail -n +3 | awk '{print $2}') ) + local len=${#path_array[@]} + + for ((j=0; j<$len; j++)) + do + gluster volume quota $1 limit-usage ${path_array[$j]} ${limit_array[j]}; + if [ $? -eq 0 ]; then + echo "Setting limit (${limit_array[j]}) on " \ + "path ${path_array[$j]} has been " \ + "successful" + fi + done + fi; +} + +if [ -z $1 ]; then + echo "Please provide volume name or 'all'"; + exit 1; +fi + +if [ "$1" = "all" ]; then + for VOL in `gluster volume list`; + do + set_limits $VOL '1'; + done + +else + set_limits $1 '0'; +fi diff --git a/extras/pre-upgrade-script-for-quota.sh b/extras/pre-upgrade-script-for-quota.sh new file mode 100755 index 00000000000..9ff15f3b307 --- /dev/null +++ b/extras/pre-upgrade-script-for-quota.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +#Make sure glusterd and the brick processes are running on all nodes in the +#cluster. +#This script must be run prior to upgradation, that too on +#only one of the nodes in the cluster. + +BACKUP_DIR=/var/tmp/glusterfs/quota-config-backup + +mkdir -p $BACKUP_DIR +for i in `gluster volume list`; do + var=$(gluster volume info $i | grep 'features.quota'| cut -d" " -f2); + if [ -z "$var" ] || [ "$var" = "off" ]; then + continue + else + gluster volume quota $i list > $BACKUP_DIR/vol_$i; + fi; +done diff --git a/extras/profiler/glusterfs-profiler b/extras/profiler/glusterfs-profiler new file mode 100755 index 00000000000..aaafd088648 --- /dev/null +++ b/extras/profiler/glusterfs-profiler @@ -0,0 +1,817 @@ +#!/usr/bin/python3 + +# Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + + +# texttable - module for creating simple ASCII tables +# Incorporated from texttable.py downloaded from +# http://jefke.free.fr/stuff/python/texttable/texttable-0.7.0.tar.gz + +import sys +import string + +try: + if sys.version >= '2.3': + import textwrap + elif sys.version >= '2.2': + from optparse import textwrap + else: + from optik import textwrap +except ImportError: + sys.stderr.write("Can't import textwrap module!\n") + raise + +try: + True, False +except NameError: + (True, False) = (1, 0) + +def len(iterable): + """Redefining len here so it will be able to work with non-ASCII characters + """ + if not isinstance(iterable, str): + return iterable.__len__() + + try: + return len(unicode(iterable, 'utf')) + except: + return iterable.__len__() + +class ArraySizeError(Exception): + """Exception raised when specified rows don't fit the required size + """ + + def __init__(self, msg): + self.msg = msg + Exception.__init__(self, msg, '') + + def __str__(self): + return self.msg + +class Texttable: + + BORDER = 1 + HEADER = 1 << 1 + HLINES = 1 << 2 + VLINES = 1 << 3 + + def __init__(self, max_width=80): + """Constructor + + - max_width is an integer, specifying the maximum width of the table + - if set to 0, size is unlimited, therefore cells won't be wrapped + """ + + if max_width <= 0: + max_width = False + self._max_width = max_width + self._deco = Texttable.VLINES | Texttable.HLINES | Texttable.BORDER | \ + Texttable.HEADER + self.set_chars(['-', '|', '+', '=']) + self.reset() + + def reset(self): + """Reset the instance + + - reset rows and header + """ + + self._hline_string = None + self._row_size = None + self._header = [] + self._rows = [] + + def header(self, array): + """Specify the header of the table + """ + + self._check_row_size(array) + self._header = map(str, array) + + def add_row(self, array): + """Add a row in the rows stack + + - cells can contain newlines and tabs + """ + + self._check_row_size(array) + self._rows.append(map(str, array)) + + def add_rows(self, rows, header=True): + """Add several rows in the rows stack + + - The 'rows' argument can be either an iterator returning arrays, + or a by-dimensional array + - 'header' specifies if the first row should be used as the header + of the table + """ + + # nb: don't use 'iter' on by-dimensional arrays, to get a + # usable code for python 2.1 + if header: + if hasattr(rows, '__iter__') and hasattr(rows, 'next'): + self.header(rows.next()) + else: + self.header(rows[0]) + rows = rows[1:] + for row in rows: + self.add_row(row) + + def set_chars(self, array): + """Set the characters used to draw lines between rows and columns + + - the array should contain 4 fields: + + [horizontal, vertical, corner, header] + + - default is set to: + + ['-', '|', '+', '='] + """ + + if len(array) != 4: + raise ArraySizeError, "array should contain 4 characters" + array = [ x[:1] for x in [ str(s) for s in array ] ] + (self._char_horiz, self._char_vert, + self._char_corner, self._char_header) = array + + def set_deco(self, deco): + """Set the table decoration + + - 'deco' can be a combinaison of: + + Texttable.BORDER: Border around the table + Texttable.HEADER: Horizontal line below the header + Texttable.HLINES: Horizontal lines between rows + Texttable.VLINES: Vertical lines between columns + + All of them are enabled by default + + - example: + + Texttable.BORDER | Texttable.HEADER + """ + + self._deco = deco + + def set_cols_align(self, array): + """Set the desired columns alignment + + - the elements of the array should be either "l", "c" or "r": + + * "l": column flushed left + * "c": column centered + * "r": column flushed right + """ + + self._check_row_size(array) + self._align = array + + def set_cols_valign(self, array): + """Set the desired columns vertical alignment + + - the elements of the array should be either "t", "m" or "b": + + * "t": column aligned on the top of the cell + * "m": column aligned on the middle of the cell + * "b": column aligned on the bottom of the cell + """ + + self._check_row_size(array) + self._valign = array + + def set_cols_width(self, array): + """Set the desired columns width + + - the elements of the array should be integers, specifying the + width of each column. For example: + + [10, 20, 5] + """ + + self._check_row_size(array) + try: + array = map(int, array) + if reduce(min, array) <= 0: + raise ValueError + except ValueError: + sys.stderr.write("Wrong argument in column width specification\n") + raise + self._width = array + + def draw(self): + """Draw the table + + - the table is returned as a whole string + """ + + if not self._header and not self._rows: + return + self._compute_cols_width() + self._check_align() + out = "" + if self._has_border(): + out += self._hline() + if self._header: + out += self._draw_line(self._header, isheader=True) + if self._has_header(): + out += self._hline_header() + length = 0 + for row in self._rows: + length += 1 + out += self._draw_line(row) + if self._has_hlines() and length < len(self._rows): + out += self._hline() + if self._has_border(): + out += self._hline() + return out[:-1] + + def _check_row_size(self, array): + """Check that the specified array fits the previous rows size + """ + + if not self._row_size: + self._row_size = len(array) + elif self._row_size != len(array): + raise ArraySizeError, "array should contain %d elements" \ + % self._row_size + + def _has_vlines(self): + """Return a boolean, if vlines are required or not + """ + + return self._deco & Texttable.VLINES > 0 + + def _has_hlines(self): + """Return a boolean, if hlines are required or not + """ + + return self._deco & Texttable.HLINES > 0 + + def _has_border(self): + """Return a boolean, if border is required or not + """ + + return self._deco & Texttable.BORDER > 0 + + def _has_header(self): + """Return a boolean, if header line is required or not + """ + + return self._deco & Texttable.HEADER > 0 + + def _hline_header(self): + """Print header's horizontal line + """ + + return self._build_hline(True) + + def _hline(self): + """Print an horizontal line + """ + + if not self._hline_string: + self._hline_string = self._build_hline() + return self._hline_string + + def _build_hline(self, is_header=False): + """Return a string used to separated rows or separate header from + rows + """ + horiz = self._char_horiz + if (is_header): + horiz = self._char_header + # compute cell separator + s = "%s%s%s" % (horiz, [horiz, self._char_corner][self._has_vlines()], + horiz) + # build the line + l = s.join([horiz*n for n in self._width]) + # add border if needed + if self._has_border(): + l = "%s%s%s%s%s\n" % (self._char_corner, horiz, l, horiz, + self._char_corner) + else: + l += "\n" + return l + + def _len_cell(self, cell): + """Return the width of the cell + + Special characters are taken into account to return the width of the + cell, such like newlines and tabs + """ + + cell_lines = cell.split('\n') + maxi = 0 + for line in cell_lines: + length = 0 + parts = line.split('\t') + for part, i in zip(parts, range(1, len(parts) + 1)): + length = length + len(part) + if i < len(parts): + length = (length/8 + 1)*8 + maxi = max(maxi, length) + return maxi + + def _compute_cols_width(self): + """Return an array with the width of each column + + If a specific width has been specified, exit. If the total of the + columns width exceed the table desired width, another width will be + computed to fit, and cells will be wrapped. + """ + + if hasattr(self, "_width"): + return + maxi = [] + if self._header: + maxi = [ self._len_cell(x) for x in self._header ] + for row in self._rows: + for cell,i in zip(row, range(len(row))): + try: + maxi[i] = max(maxi[i], self._len_cell(cell)) + except (TypeError, IndexError): + maxi.append(self._len_cell(cell)) + items = len(maxi) + length = reduce(lambda x,y: x+y, maxi) + if self._max_width and length + items*3 + 1 > self._max_width: + maxi = [(self._max_width - items*3 -1) / items \ + for n in range(items)] + self._width = maxi + + def _check_align(self): + """Check if alignment has been specified, set default one if not + """ + + if not hasattr(self, "_align"): + self._align = ["l"]*self._row_size + if not hasattr(self, "_valign"): + self._valign = ["t"]*self._row_size + + def _draw_line(self, line, isheader=False): + """Draw a line + + Loop over a single cell length, over all the cells + """ + + line = self._splitit(line, isheader) + space = " " + out = "" + for i in range(len(line[0])): + if self._has_border(): + out += "%s " % self._char_vert + length = 0 + for cell, width, align in zip(line, self._width, self._align): + length += 1 + cell_line = cell[i] + fill = width - len(cell_line) + if isheader: + align = "c" + if align == "r": + out += "%s " % (fill * space + cell_line) + elif align == "c": + out += "%s " % (fill/2 * space + cell_line \ + + (fill/2 + fill%2) * space) + else: + out += "%s " % (cell_line + fill * space) + if length < len(line): + out += "%s " % [space, self._char_vert][self._has_vlines()] + out += "%s\n" % ['', self._char_vert][self._has_border()] + return out + + def _splitit(self, line, isheader): + """Split each element of line to fit the column width + + Each element is turned into a list, result of the wrapping of the + string to the desired width + """ + + line_wrapped = [] + for cell, width in zip(line, self._width): + array = [] + for c in cell.split('\n'): + array.extend(textwrap.wrap(unicode(c, 'utf'), width)) + line_wrapped.append(array) + max_cell_lines = reduce(max, map(len, line_wrapped)) + for cell, valign in zip(line_wrapped, self._valign): + if isheader: + valign = "t" + if valign == "m": + missing = max_cell_lines - len(cell) + cell[:0] = [""] * (missing / 2) + cell.extend([""] * (missing / 2 + missing % 2)) + elif valign == "b": + cell[:0] = [""] * (max_cell_lines - len(cell)) + else: + cell.extend([""] * (max_cell_lines - len(cell))) + return line_wrapped + + +# Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com> +# This file is part of GlusterFS. + +# GlusterFS is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3 of the License, +# or (at your option) any later version. + +# GlusterFS is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# <http://www.gnu.org/licenses/>. + +graph_available = True + +try: + import numpy as np + import matplotlib.pyplot as plt +except ImportError: + graph_available = False + +import re +import sys + +from optparse import OptionParser + +# Global dict-of-dict holding the latency data +# latency[xlator-name][op-name] + +latencies = {} +counts = {} +totals = {} + +def collect_data (f): + """Collect latency data from the file object f and store it in + the global variable @latencies""" + + # example dump file line: + # fuse.latency.TRUNCATE=3147.000,4 + + for line in f: + m = re.search ("(\w+)\.\w+.(\w+)=(\w+\.\w+),(\w+),(\w+.\w+)", line) + if m and float(m.group(3)) != 0: + xlator = m.group(1) + op = m.group(2) + time = m.group(3) + count = m.group(4) + total = m.group(5) + + if not xlator in latencies.keys(): + latencies[xlator] = dict() + + if not xlator in counts.keys(): + counts[xlator] = dict() + + if not xlator in totals.keys(): + totals[xlator] = dict() + + latencies[xlator][op] = time + counts[xlator][op] = count + totals[xlator][op] = total + + +def calc_latency_heights (xlator_order): + heights = map (lambda x: [], xlator_order) + + N = len (xlator_order) + for i in range (N): + xl = xlator_order[i] + + k = latencies[xl].keys() + k.sort() + + if i == len (xlator_order) - 1: + # bottom-most xlator + heights[i] = [float (latencies[xl][key]) for key in k] + + else: + next_xl = xlator_order[i+1] + this_xl_time = [latencies[xl][key] for key in k] + next_xl_time = [latencies[next_xl][key] for key in k] + + heights[i] = map (lambda x, y: float (x) - float (y), + this_xl_time, next_xl_time) + return heights + +# have sufficient number of colors +colors = ["violet", "blue", "green", "yellow", "orange", "red"] + +def latency_profile (title, xlator_order, mode): + heights = calc_latency_heights (xlator_order) + + N = len (latencies[xlator_order[0]].keys()) + Nxl = len (xlator_order) + ind = np.arange (N) + width = 0.35 + + pieces = map (lambda x: [], xlator_order) + bottoms = map (lambda x: [], xlator_order) + + bottoms[Nxl-1] = map (lambda x: 0, latencies[xlator_order[0]].keys()) + + k = latencies[xlator_order[0]].keys() + k.sort() + + for i in range (Nxl-1): + xl = xlator_order[i+1] + bottoms[i] = [float(latencies[xl][key]) for key in k] + + if mode == 'text': + print "\n%sLatency profile for %s\n" % (' '*20, title) + print "Average latency (microseconds):\n" + + table = Texttable() + + table.set_cols_align(["l", "r"] + ["r"] * len(xlator_order)) + rows = [] + + header = ['OP', 'OP Average (us)'] + xlator_order + rows = [] + + for op in k: + sum = reduce (lambda x, y: x + y, [heights[xlator_order.index(xl)][k.index(op)] for xl in xlator_order], + 0) + + row = [op] + row += ["%5.2f" % sum] + + for xl in xlator_order: + op_index = k.index(op) + row += ["%5.2f" % (heights[xlator_order.index(xl)][op_index])] + + rows.append(row) + + def row_sort(r1, r2): + v1 = float(r1[1]) + v2 = float(r2[1]) + + if v1 < v2: + return -1 + elif v1 == v2: + return 0 + else: + return 1 + + rows.sort(row_sort, reverse=True) + table.add_rows([header] + rows) + print table.draw() + + elif mode == 'graph': + for i in range(Nxl): + pieces[i] = plt.bar (ind, heights[i], width, color=colors[i], + bottom=bottoms[i]) + + plt.ylabel ("Average Latency (microseconds)") + plt.title ("Latency Profile for '%s'" % title) + k = latencies[xlator_order[0]].keys() + k.sort () + plt.xticks (ind+width/2., k) + + m = round (max(map (float, latencies[xlator_order[0]].values())), -2) + plt.yticks (np.arange(0, m + m*0.1, m/10)) + plt.legend (map (lambda p: p[0], pieces), xlator_order) + + plt.show () + else: + print "Unknown mode specified!" + sys.exit(1) + + +def fop_distribution (title, xlator_order, mode): + plt.ylabel ("Percentage of calls") + plt.title ("FOP distribution for '%s'" % title) + k = counts[xlator_order[0]].keys() + k.sort () + + N = len (latencies[xlator_order[0]].keys()) + ind = np.arange(N) + width = 0.35 + + total = 0 + top_xl = xlator_order[0] + for op in k: + total += int(counts[top_xl][op]) + + heights = [] + + for op in k: + heights.append (float(counts[top_xl][op])/total * 100) + + if mode == 'text': + print "\n%sFOP distribution for %s\n" % (' '*20, title) + print "Total number of calls: %d\n" % total + + table = Texttable() + + table.set_cols_align(["l", "r", "r"]) + + rows = [] + header = ["OP", "% of Calls", "Count"] + + for op in k: + row = [op, "%5.2f" % (float(counts[top_xl][op])/total * 100), counts[top_xl][op]] + rows.append(row) + + def row_sort(r1, r2): + v1 = float(r1[1]) + v2 = float(r2[1]) + + if v1 < v2: + return -1 + elif v1 == v2: + return 0 + else: + return 1 + + rows.sort(row_sort, reverse=True) + table.add_rows([header] + rows) + print table.draw() + + elif mode == 'graph': + bars = plt.bar (ind, heights, width, color="red") + + for bar in bars: + height = bar.get_height() + plt.text (bar.get_x()+bar.get_width()/2., 1.05*height, + "%d%%" % int(height)) + + plt.xticks(ind+width/2., k) + plt.yticks(np.arange (0, 110, 10)) + + plt.show() + else: + print "mode not specified!" + sys.exit(1) + + +def calc_workload_heights (xlator_order, scaling): + workload_heights = map (lambda x: [], xlator_order) + + top_xl = xlator_order[0] + + N = len (xlator_order) + for i in range (N): + xl = xlator_order[i] + + k = totals[xl].keys() + k.sort() + + if i == len (xlator_order) - 1: + # bottom-most xlator + workload_heights[i] = [float (totals[xl][key]) / float(totals[top_xl][key]) * scaling[k.index(key)] for key in k] + + else: + next_xl = xlator_order[i+1] + this_xl_time = [float(totals[xl][key]) / float(totals[top_xl][key]) * scaling[k.index(key)] for key in k] + next_xl_time = [float(totals[next_xl][key]) / float(totals[top_xl][key]) * scaling[k.index(key)] for key in k] + + workload_heights[i] = map (lambda x, y: (float (x) - float (y)), + this_xl_time, next_xl_time) + + return workload_heights + +def workload_profile(title, xlator_order, mode): + plt.ylabel ("Percentage of Total Time") + plt.title ("Workload Profile for '%s'" % title) + k = totals[xlator_order[0]].keys() + k.sort () + + N = len(totals[xlator_order[0]].keys()) + Nxl = len(xlator_order) + ind = np.arange(N) + width = 0.35 + + total = 0 + top_xl = xlator_order[0] + for op in k: + total += float(totals[top_xl][op]) + + p_heights = [] + + for op in k: + p_heights.append (float(totals[top_xl][op])/total * 100) + + heights = calc_workload_heights (xlator_order, p_heights) + + pieces = map (lambda x: [], xlator_order) + bottoms = map (lambda x: [], xlator_order) + + bottoms[Nxl-1] = map (lambda x: 0, totals[xlator_order[0]].keys()) + + for i in range (Nxl-1): + xl = xlator_order[i+1] + k = totals[xl].keys() + k.sort() + + bottoms[i] = [float(totals[xl][key]) / float(totals[top_xl][key]) * p_heights[k.index(key)] for key in k] + + if mode == 'text': + print "\n%sWorkload profile for %s\n" % (' '*20, title) + print "Total Time: %d microseconds = %.1f seconds = %.1f minutes\n" % (total, total / 1000000.0, total / 6000000.0) + + table = Texttable() + table.set_cols_align(["l", "r"] + ["r"] * len(xlator_order)) + rows = [] + + header = ['OP', 'OP Total (%)'] + xlator_order + rows = [] + + for op in k: + sum = reduce (lambda x, y: x + y, [heights[xlator_order.index(xl)][k.index(op)] for xl in xlator_order], + 0) + row = [op] + row += ["%5.2f" % sum] + + for xl in xlator_order: + op_index = k.index(op) + row += ["%5.2f" % heights[xlator_order.index(xl)][op_index]] + + rows.append(row) + + def row_sort(r1, r2): + v1 = float(r1[1]) + v2 = float(r2[1]) + + if v1 < v2: + return -1 + elif v1 == v2: + return 0 + else: + return 1 + + rows.sort(row_sort, reverse=True) + table.add_rows([header] + rows) + print table.draw() + + elif mode == 'graph': + for i in range(Nxl): + pieces[i] = plt.bar (ind, heights[i], width, color=colors[i], + bottom=bottoms[i]) + + for key in k: + bar = pieces[Nxl-1][k.index(key)] + plt.text (bar.get_x() + bar.get_width()/2., 1.05*p_heights[k.index(key)], + "%d%%" % int(p_heights[k.index(key)])) + + plt.xticks(ind+width/2., k) + plt.yticks(np.arange (0, 110, 10)) + plt.legend (map (lambda p: p[0], pieces), xlator_order) + + plt.show() + else: + print "Unknown mode specified!" + sys.exit(1) + + +def main (): + parser = OptionParser(usage="usage: %prog [-l | -d | -w] -x <xlator order> <state dump file>") + parser.add_option("-l", "--latency", dest="latency", action="store_true", + help="Produce latency profile") + parser.add_option("-d", "--distribution", dest="distribution", action="store_true", + help="Produce distribution of FOPs") + parser.add_option("-w", "--workload", dest="workload", action="store_true", + help="Produce workload profile") + parser.add_option("-t", "--title", dest="title", help="Set the title of the graph") + parser.add_option("-x", "--xlator-order", dest="xlator_order", help="Specify the order of xlators") + parser.add_option("-m", "--mode", dest="mode", help="Output format, can be text[default] or graph") + + (options, args) = parser.parse_args() + + if len(args) != 1: + parser.error("Incorrect number of arguments") + + if (options.xlator_order): + xlator_order = options.xlator_order.split() + else: + print "xlator order must be specified" + sys.exit(1) + + collect_data(file (args[0], 'r')) + + mode = 'text' + if (options.mode): + mode = options.mode + if options.mode == 'graph' and graph_available == False: + print "matplotlib not available, falling back to text mode" + mode = 'text' + + if (options.latency): + latency_profile (options.title, xlator_order, mode) + + if (options.distribution): + fop_distribution(options.title, xlator_order, mode) + + if (options.workload): + workload_profile(options.title, xlator_order, mode) + +main () diff --git a/extras/python/Makefile.am b/extras/python/Makefile.am new file mode 100644 index 00000000000..7d81fa0319b --- /dev/null +++ b/extras/python/Makefile.am @@ -0,0 +1,7 @@ +if HAVE_PYTHON +# Install __init__.py into the Python site-packages area +pypkgdir = @BUILD_PYTHON_SITE_PACKAGES@/gluster +pypkg_PYTHON = __init__.py +endif + +EXTRA_DIST = __init__.py diff --git a/extras/python/__init__.py b/extras/python/__init__.py new file mode 100644 index 00000000000..3ad9513f40e --- /dev/null +++ b/extras/python/__init__.py @@ -0,0 +1,2 @@ +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/extras/quota/contri-add.sh b/extras/quota/contri-add.sh new file mode 100755 index 00000000000..7db5edd5d20 --- /dev/null +++ b/extras/quota/contri-add.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# This script adds contributions of files/directories in backend to volume +# size. +# It can also be used to debug by passing dir as first argument, in which case +# it will just add contributions from immediate children of a directory and +# displays only if added contributions from immediate children is different +# from size stored in directory. +# For Eg., find <backend-directory> -type d -exec ./contri-add.sh dir \{} \; +# will list all the directories which have descrepancies in their +# size/contributions. + +usage () +{ + echo >&2 "usage: $0 <file|dir> <list-of-backend-directories>" +} + +add_contributions () +{ + local var=0 + local count=0 + + SIZE=`getfattr -h -e hex -n trusted.glusterfs.quota.size $2 2>&1 | sed -e '/^#/d' | sed -e '/^getfattr/d' | sed -e '/^$/d' | cut -d'=' -f 2` + CONTRI=`getfattr -h -e hex -d -m trusted.glusterfs.quota.*.contri $2 2>&1 | sed -e '/^#/d' | sed -e '/^getfattr/d' | sed -e '/^$/d' | cut -d'=' -f 2` + + if [ $1 == "file" ]; then + PATHS=`find $2 ! -type d | sed -e "\|^$2$|d" | sed -e '/^[ \t]*$/d'` + else + PATHS=`find $2 -maxdepth 1 | sed -e "\|^$2$|d" | sed -e '/^[ \t]*$/d'` + fi + + if [ -z "$PATHS" ]; then + return 0 + fi + + CONTRIBUTIONS=`echo $PATHS | xargs getfattr -h -e hex -d -m trusted.glusterfs.quota.*.contri 2>&1 | sed -e '/^#/d' | sed -e '/^getfattr/d' | sed -e '/^$/d' | cut -d'=' -f 2 | sed -e 's/^[ \t]*\([^ \t]*\)/\1/g'` + + if [ -n "$CONTRIBUTIONS" ]; then + for i in $CONTRIBUTIONS; do + count=$(($count + 1)) + var=$(($var + $i)) + done + fi + + if [ $1 == "file" ] || [ $var -ne $(($SIZE)) ] || [ $(($SIZE)) -ne $(($CONTRI)) ]; then + if [ $1 == "dir" ]; then + TMP_PATH=`echo $2 | sed -e "s/\/home\/export\/[0-9]*/\/mnt\/raghu/g"` + stat $TMP_PATH > /dev/null + fi + + echo "file count $count" + echo "added contribution of $2=$var" + echo "size stored in xattrs on $2=$(($SIZE))" + echo "contribution of $2 to its parent directory=$(($CONTRI))" + echo "==============================================================" + fi +} + + +main () +{ + [ $# -lt 1 ] && usage + + TYPE=$1 + + shift 1 + + for i in $@; do + add_contributions $TYPE $i + done +} + +main $@
\ No newline at end of file diff --git a/extras/quota/log_accounting.sh b/extras/quota/log_accounting.sh new file mode 100755 index 00000000000..e2dd87b84d7 --- /dev/null +++ b/extras/quota/log_accounting.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# The script does an accounting of all directories using command 'du' and +# using gluster. We can then compare the two to identify accounting mismatch +# THere can be minor mismatch because gluster only accounts for the size of +# files. Direcotries can take up upto 4kB space on FS per directory. THis +# size is accounted by du and not by gluster. However the difference would +# not be significant. + +mountpoint=$1 +volname=$2 + +usage () +{ + echo >&2 "usage: $0 <mountpoint> <volume name>" + exit +} + +[ $# -lt 2 ] && usage + +cd $mountpoint +du -h | head -n -1 | tr -d '.' |awk '{ for (i = 2; i <= NF; i++) { printf("%s ", $i);} print "" }' > /tmp/gluster_quota_1 +cat /tmp/gluster_quota_1 | sed 's/ $//' | sed 's/ /\\ /g' | sed 's/(/\\(/g' | sed 's/)/\\)/g' |xargs gluster v quota $volname list > /tmp/gluster_quota_2 +du -h | head -n -1 |awk '{ for (i = 2; i <= NF; i++) { printf("%s %s", $i, $1);} print "" }' | tr -d '.' > /tmp/gluster_quota_3 +cat /tmp/gluster_quota_2 /tmp/gluster_quota_3 | sort > /tmp/gluster_quota_4 +find . -type d > /tmp/gluster_quota_5 +tar -cvf /tmp/gluster_quota_files.tar /tmp/gluster_quota_* diff --git a/extras/quota/quota_fsck.py b/extras/quota/quota_fsck.py new file mode 100755 index 00000000000..e62f7fc52a3 --- /dev/null +++ b/extras/quota/quota_fsck.py @@ -0,0 +1,377 @@ +#!/usr/bin/python3 +# The following script enables, Detecting, Reporting and Fixing +# anomalies in quota accounting. Run this script with -h option +# for further details. + +''' + Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +''' +from __future__ import print_function +import os, sys, re +from stat import * +import subprocess +import argparse +import xattr + +aggr_size = {} +verbose_mode = False +mnt_path = None +brick_path = None +obj_fix_count = 0 +file_count = 0 +dir_count = 0 + +#CONSTANTS +KB = 1024 +MB = 1048576 +GB = 1048576 * 1024 +TB = 1048576 * 1048576 + +QUOTA_VERBOSE = 0 +QUOTA_META_ABSENT = 1 +QUOTA_SIZE_MISMATCH = 2 + +IS_DIRTY ='0x3100' +IS_CLEAN ='0x3000' + + +epilog_msg=''' + The script attempts to find any gluster accounting issues in the + filesystem at the given subtree. The script crawls the given + subdirectory tree doing a stat for all files and compares the + size reported by gluster quota with the size reported by stat + calls. Any mismatch is reported. In addition integrity of marker + xattrs are verified. + ''' + +def print_msg(log_type, path, xattr_dict = {}, stbuf = "", dir_size = None): + if log_type == QUOTA_VERBOSE: + print('%-24s %-60s\nxattr_values: %s\n%s\n' % ("Verbose", path, xattr_dict, stbuf)) + elif log_type == QUOTA_META_ABSENT: + print('%-24s %-60s\n%s\n' % ("Quota-Meta Absent", path, xattr_dict)) + elif log_type == QUOTA_SIZE_MISMATCH: + print("mismatch") + if dir_size is not None: + print('%24s %60s %12s %12s' % ("Size Mismatch", path, + xattr_dict, dir_size)) + else: + print('%-24s %-60s %-12s %-12s' % ("Size Mismatch", path, xattr_dict, + stbuf.st_size)) + +def size_differs_lot(s1, s2): + ''' + There could be minor accounting differences between the stat based + accounting and gluster accounting. To avoid these from throwing lot + of false positives in our logs. using a threshold of 1M for now. + TODO: For a deeply nested directory, at higher levels in hierarchy + differences may not be significant, hence this check needs to be improved. + ''' + if abs(s1-s2) > 0: + return True + else: + return False + +def fix_hardlink_accounting(curr_dict, accounted_dict, curr_size): + ''' + Hard links are messy.. we have to account them for their parent + directory. But, stop accounting at the most common ancestor. + Eg: + say we have 3 hardlinks : /d1/d2/h1, /d1/d3/h2 and /d1/h3 + + suppose we encounter the hard links h1 first , then h2 and then h3. + while accounting for h1, we account the size until root(d2->d1->/) + while accounting for h2, we need to account only till d3. (as d1 + and / are accounted for this inode). + while accounting for h3 we should not account at all.. as all + its ancestors are already accounted for same inode. + + curr_dict : dict of hardlinks that were seen and + accounted by the current iteration. + accounted_dict : dict of hardlinks that has already been + accounted for. + + size : size of the object as accounted by the + curr_iteration. + + Return vale: + curr_size : size reduced by hardlink sizes for those + hardlinks that has already been accounted + in current subtree. + Also delete the duplicate link from curr_dict. + ''' + + dual_accounted_links = set(curr_dict.keys()) & set(accounted_dict.keys()) + for link in dual_accounted_links: + curr_size = curr_size - curr_dict[link] + del curr_dict[link] + return curr_size + + +def fix_xattr(file_name, mark_dirty): + global obj_fix_count + global mnt_path + + if mnt_path is None: + return + if mark_dirty: + print("MARKING DIRTY: " + file_name) + out = subprocess.check_output (["/usr/bin/setfattr", "-n", + "trusted.glusterfs.quota.dirty", + "-v", IS_DIRTY, file_name]) + rel_path = os.path.relpath(file_name, brick_path) + print("stat on " + mnt_path + "/" + rel_path) + stbuf = os.lstat(mnt_path + "/" + rel_path) + + obj_fix_count += 1 + +def get_quota_xattr_brick(dpath): + out = subprocess.check_output (["/usr/bin/getfattr", "--no-dereference", + "-d", "-m.", "-e", "hex", dpath]) + pairs = out.splitlines() + + ''' + Sample output to be parsed: + [root@dhcp35-100 mnt]# getfattr -d -m. -e hex /export/b1/B0/d14/d13/ + # file: export/b1/B0/d14/d13/ + security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a7573725f743a733000 + trusted.gfid=0xbae5e0d2d05043de9fd851d91ecf63e8 + trusted.glusterfs.dht=0x000000010000000000000000ffffffff + trusted.glusterfs.dht.mds=0x00000000 + trusted.glusterfs.quota.6a7675a3-b85a-40c5-830b-de9229d702ce.contri.39=0x00000000000000000000000000000000000000000000000e + trusted.glusterfs.quota.dirty=0x3000 + trusted.glusterfs.quota.size.39=0x00000000000000000000000000000000000000000000000e + ''' + + ''' + xattr_dict dictionary holds quota related xattrs + eg: + ''' + + xattr_dict = {} + xattr_dict['parents'] = {} + + for xattr in pairs[1:]: + xattr = xattr.decode("utf-8") + xattr_key = xattr.split("=")[0] + if xattr_key == "": + # skip any empty lines + continue + elif not re.search("quota", xattr_key): + # skip all non quota xattr. + continue + + xattr_value = xattr.split("=")[1] + if re.search("contri", xattr_key): + + xattr_version = xattr_key.split(".")[5] + if 'version' not in xattr_dict: + xattr_dict['version'] = xattr_version + else: + if xattr_version != xattr_dict['version']: + print("Multiple xattr version found") + + + cur_parent = xattr_key.split(".")[3] + if cur_parent not in xattr_dict['parents']: + xattr_dict['parents'][cur_parent] = {} + + contri_dict = xattr_dict['parents'][cur_parent] + if len(xattr_value) == 34: + # 34 bytes implies file contri xattr + # contri format =0x< 16bytes file size><16bytes file count> + # size is obtained in iatt, file count = 1, dir count=0 + contri_dict['contri_size'] = int(xattr_value[2:18], 16) + contri_dict['contri_file_count'] = int(xattr_value[18:34], 16) + contri_dict['contri_dir_count'] = 0 + else: + # This is a directory contri. + contri_dict['contri_size'] = int(xattr_value[2:18], 16) + contri_dict['contri_file_count'] = int(xattr_value[18:34], 16) + contri_dict['contri_dir_count'] = int(xattr_value[34:], 16) + + elif re.search("size", xattr_key): + xattr_dict['size'] = int(xattr_value[2:18], 16) + xattr_dict['file_count'] = int(xattr_value[18:34], 16) + xattr_dict['dir_count'] = int(xattr_value[34:], 16) + elif re.search("dirty", xattr_key): + if xattr_value == IS_CLEAN: + xattr_dict['dirty'] = False + elif xattr_value == IS_DIRTY: + xattr_dict['dirty'] = True + elif re.search("limit_objects", xattr_key): + xattr_dict['limit_objects'] = int(xattr_value[2:18], 16) + elif re.search("limit_set", xattr_key): + xattr_dict['limit_set'] = int(xattr_value[2:18], 16) + + return xattr_dict + +def verify_file_xattr(path, stbuf = None): + + global file_count + file_count += 1 + + if stbuf is None: + stbuf = os.lstat(path) + + xattr_dict = get_quota_xattr_brick(path) + + for parent in xattr_dict['parents']: + contri_dict = xattr_dict['parents'][parent] + + if 'contri_size' not in contri_dict or \ + 'contri_file_count' not in contri_dict or \ + 'contri_dir_count' not in contri_dict: + print_msg(QUOTA_META_ABSENT, path, xattr_dict, stbuf) + fix_xattr(path, False) + return + elif size_differs_lot(contri_dict['contri_size'], stbuf.st_size): + print_msg(QUOTA_SIZE_MISMATCH, path, xattr_dict, stbuf) + fix_xattr(path, False) + return + + if verbose_mode is True: + print_msg(QUOTA_VERBOSE, path, xattr_dict, stbuf) + + +def verify_dir_xattr(path, dir_size): + + global dir_count + dir_count += 1 + xattr_dict = get_quota_xattr_brick(path) + + stbuf = os.lstat(path) + + for parent in xattr_dict['parents']: + contri_dict = xattr_dict['parents'][parent] + + if 'size' not in xattr_dict or 'contri_size' not in contri_dict: + print_msg(QUOTA_META_ABSENT, path) + fix_xattr(path, True) + return + elif size_differs_lot(dir_size, xattr_dict['size']) or \ + size_differs_lot(contri_dict['contri_size'], xattr_dict['size']): + print_msg(QUOTA_SIZE_MISMATCH, path, xattr_dict, stbuf, dir_size) + fix_xattr(path, True) + return + + if verbose_mode is True: + print_msg("VERBOSE", path, xattr_dict, stbuf, dir_size) + + +def walktree(t_dir, hard_link_dict): + '''recursively descend the directory tree rooted at dir, + aggregating the size + t_dir : directory to walk over. + hard_link_dict : dict of inodes with multiple hard_links under t_dir + ''' + global aggr_size + aggr_size[t_dir] = 0 + + for entry in os.listdir(t_dir): + pathname = os.path.join(t_dir, entry) + stbuf = os.lstat(pathname) + if S_ISDIR(stbuf.st_mode): + # It's a directory, recurse into it + if entry == '.glusterfs': + print("skipping " + pathname) + continue + descendent_hardlinks = {} + subtree_size = walktree(pathname, descendent_hardlinks) + + subtree_size = fix_hardlink_accounting(descendent_hardlinks, + hard_link_dict, + subtree_size) + + aggr_size[t_dir] = aggr_size[t_dir] + subtree_size + + elif S_ISREG(stbuf.st_mode) or S_ISLNK(stbuf.st_mode): + # Even a symbolic link file may have multiple hardlinks. + + file_size = stbuf.st_size + if stbuf.st_nlink > 2: + # send a single element dict to check if file is accounted. + file_size = fix_hardlink_accounting({stbuf.st_ino:stbuf.st_size}, + hard_link_dict, + stbuf.st_size) + + if file_size == 0: + print_msg("HARD_LINK (skipped)", pathname, "", + stbuf) + else: + print_msg("HARD_LINK (accounted)", pathname, "", + stbuf) + hard_link_dict[stbuf.st_ino] = stbuf.st_size + + if t_dir in aggr_size: + aggr_size[t_dir] = aggr_size[t_dir] + file_size + else: + aggr_size[t_dir] = file_size + verify_file_xattr(pathname, stbuf) + + else: + # Unknown file type, print a message + print('Skipping %s, due to file mode' % (pathname)) + + if t_dir not in aggr_size: + aggr_size[t_dir] = 0 + + verify_dir_xattr(t_dir, aggr_size[t_dir]) + # du also accounts for t_directory sizes + # aggr_size[t_dir] += 4096 + + #cleanup + ret = aggr_size[t_dir] + del aggr_size[t_dir] + return ret + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Diagnose quota accounting issues.', epilog=epilog_msg) + parser.add_argument('brick_path', nargs=1, + help='The brick path (or any descendent sub-directory of brick path)', + ) + parser.add_argument('--full-logs', dest='verbose', action='store_true', + help=''' + log all the xattr values and stat values reported + for analysis. [CAUTION: This can give lot of output + depending on FS depth. So one has to make sure enough + disk space exists if redirecting to file] + ''' + ) + parser.add_argument('--fix-issues', metavar='mount_path', dest='mnt', action='store', + help=''' + fix accounting issues where the xattr values disagree + with stat sizes reported by gluster. A mount is also + required for this option to be used. + [CAUTION: This will directly modify backend xattr] + ''' + ) + parser.add_argument('--sub-dir', metavar='sub_dir', dest='sub_dir', action='store', + help=''' + limit the crawling and accounting verification/correction + to a specific subdirectory. + ''' + ) + + args = parser.parse_args() + verbose_mode = args.verbose + brick_path = args.brick_path[0] + sub_dir = args.sub_dir + mnt_path = args.mnt + hard_link_dict = {} + if sub_dir is not None: + walktree(os.path.join(brick_path, sub_dir), hard_link_dict) + else: + walktree(brick_path, hard_link_dict) + + print("Files verified : " + str(file_count)) + print("Directories verified : " + str(dir_count)) + if mnt_path is not None: + print("Objects Fixed : " + str(obj_fix_count)) diff --git a/extras/quota/xattr_analysis.py b/extras/quota/xattr_analysis.py new file mode 100755 index 00000000000..7bd7d96374c --- /dev/null +++ b/extras/quota/xattr_analysis.py @@ -0,0 +1,73 @@ +#!/usr/bin/python3 +# Below script has two purposes +# 1. Display xattr of entire FS tree in a human readable form +# 2. Display all the directory where contri and size mismatch. +# (If there are any directory with contri and size mismatch that are not dirty +# then that highlights a propagation issue) +# The script takes only one input LOG _FILE generated from the command, +# find <brick_path> | xargs getfattr -d -m. -e hex > log_gluster_xattr + +from __future__ import print_function +import re +import subprocess +import sys +from hurry.filesize import size + +if len(sys.argv) < 2: + sys.exit('Usage: %s log_gluster_xattr \n' + 'to generate log_gluster_xattr use: \n' + 'find <brick_path> | xargs getfattr -d -m. -e hex > log_gluster_xattr' + % sys.argv[0]) +LOG_FILE=sys.argv[1] + +def get_quota_xattr_brick(): + out = subprocess.check_output (["/usr/bin/cat", LOG_FILE]) + pairs = out.splitlines() + + xdict = {} + mismatch_size = [('====contri_size===', '====size====')] + for xattr in pairs: + k = xattr.split("=")[0] + if re.search("# file:", k): + print(xdict) + filename=k + print("=====" + filename + "=======") + xdict = {} + elif k is "": + pass + else: + print(xattr) + v = xattr.split("=")[1] + if re.search("contri", k): + if len(v) == 34: + # for files size is obtained in iatt, file count should be 1, dir count=0 + xdict['contri_file_count'] = int(v[18:34], 16) + xdict['contri_dir_count'] = 0 + else: + xdict['contri_size'] = size(int(v[2:18], 16)) + xdict['contri_file_count'] = int(v[18:34], 16) + xdict['contri_dir_count'] = int(v[34:], 16) + elif re.search("size", k): + xdict['size'] = size(int(v[2:18], 16)) + xdict['file_count'] = int(v[18:34], 16) + xdict['dir_count'] = int(v[34:], 16) + elif re.search("dirty", k): + if v == '0x3000': + xdict['dirty'] = False + elif v == '0x3100': + xdict['dirty'] = True + elif re.search("limit_objects", k): + xdict['limit_objects'] = int(v[2:18], 16) + elif re.search("limit_set", k): + xdict['limit_set'] = size(int(v[2:18], 16)) + + if 'size' in xdict and 'contri_size' in xdict and xdict['size'] != xdict['contri_size']: + mismatch_size.append((xdict['contri_size'], xdict['size'], filename)) + + for values in mismatch_size: + print(values) + + +if __name__ == '__main__': + get_quota_xattr_brick() + diff --git a/extras/rebalance.py b/extras/rebalance.py new file mode 100755 index 00000000000..37c68ebbb42 --- /dev/null +++ b/extras/rebalance.py @@ -0,0 +1,309 @@ +#!/usr/bin/python3 + +from __future__ import print_function + +import atexit +import copy +import optparse +import os +import pipes +import shutil +import string +import subprocess +import sys +import tempfile +import volfilter +import platform + +# It's just more convenient to have named fields. +class Brick: + def __init__ (self, path, name): + self.path = path + self.sv_name = name + self.size = 0 + self.curr_size = 0 + self.good_size = 0 + def set_size (self, size): + self.size = size + def set_range (self, rs, re): + self.r_start = rs + self.r_end = re + self.curr_size = self.r_end - self.r_start + 1 + def __repr__ (self): + value = self.path[:] + value += "(%d," % self.size + if self.curr_size: + value += "0x%x,0x%x)" % (self.r_start, self.r_end) + else: + value += "-)" + return value + +def get_bricks (host, vol): + t = pipes.Template() + t.prepend("gluster --remote-host=%s system getspec %s"%(host, vol), ".-") + return t.open(None, "r") + +def generate_stanza (vf, all_xlators, cur_subvol): + sv_list = [] + for sv in cur_subvol.subvols: + generate_stanza(vf, all_xlators, sv) + sv_list.append(sv.name) + vf.write("volume %s\n" % cur_subvol.name) + vf.write(" type %s\n" % cur_subvol.type) + for kvpair in cur_subvol.opts.items(): + vf.write(" option %s %s\n" % kvpair) + if sv_list: + vf.write(" subvolumes %s\n" % ''.join(sv_list)) + vf.write("end-volume\n\n") + + +def mount_brick (localpath, all_xlators, dht_subvol): + + # Generate a volfile. + vf_name = localpath + ".vol" + vf = open(vf_name, "w") + generate_stanza(vf, all_xlators, dht_subvol) + vf.flush() + vf.close() + + # Create a brick directory and mount the brick there. + os.mkdir(localpath) + subprocess.call(["glusterfs", "-f", vf_name, localpath]) + +# We use the command-line tools because there's no getxattr support in the +# Python standard library (which is ridiculous IMO). Adding the xattr package +# from PyPI would create a new and difficult dependency because the bits to +# satisfy it don't seem to exist in Fedora. We already expect the command-line +# tools to be there, so it's safer just to rely on them. +# +# We might have to revisit this if we get as far as actually issuing millions +# of setxattr requests. Even then, it might be better to do that part with a C +# program which has only a build-time dependency. +def get_range (brick): + t = pipes.Template() + cmd = "getfattr -e hex -n trusted.glusterfs.dht %s 2> /dev/null" + t.prepend(cmd%brick, ".-") + t.append("grep ^trusted.glusterfs.dht=", "--") + f = t.open(None, "r") + try: + value = f.readline().rstrip().split('=')[1][2:] + except: + print("could not get layout for %s (might be OK)" % brick) + return None + v_start = int("0x"+value[16:24], 16) + v_end = int("0x"+value[24:32], 16) + return (v_start, v_end) + +def calc_sizes (bricks, total): + leftover = 1 << 32 + for b in bricks: + if b.size: + b.good_size = (b.size << 32) / total + leftover -= b.good_size + else: + b.good_size = 0 + if leftover: + # Add the leftover to an old brick if we can. + for b in bricks: + if b.good_size: + b.good_size += leftover + break + else: + # Fine, just add it wherever. + bricks[0].good_size += leftover + +# Normalization means sorting the bricks by r_start and (b) ensuring that there +# are no gaps. +def normalize (in_bricks): + out_bricks = [] + curr_hash = 0 + used = 0 + while curr_hash < (1<<32): + curr_best = None + for b in in_bricks: + if b.r_start == curr_hash: + used += 1 + out_bricks.append(b) + in_bricks.remove(b) + curr_hash = b.r_end + 1 + break + else: + print("gap found at 0x%08x" % curr_hash) + sys.exit(1) + return out_bricks + in_bricks, used + +def get_score (bricks): + score = 0 + curr_hash = 0 + for b in bricks: + if not b.curr_size: + curr_hash += b.good_size + continue + new_start = curr_hash + curr_hash += b.good_size + new_end = curr_hash - 1 + if new_start > b.r_start: + max_start = new_start + else: + max_start = b.r_start + if new_end < b.r_end: + min_end = new_end + else: + min_end = b.r_end + if max_start <= min_end: + score += (min_end - max_start + 1) + return score + +if __name__ == "__main__": + + my_usage = "%prog [options] server volume [directory]" + parser = optparse.OptionParser(usage=my_usage) + parser.add_option("-f", "--free-space", dest="free_space", + default=False, action="store_true", + help="use free space instead of total space") + parser.add_option("-l", "--leave-mounted", dest="leave_mounted", + default=False, action="store_true", + help="leave subvolumes mounted") + parser.add_option("-v", "--verbose", dest="verbose", + default=False, action="store_true", + help="verbose output") + options, args = parser.parse_args() + + if len(args) == 3: + fix_dir = args[2] + else: + if len(args) != 2: + parser.print_help() + sys.exit(1) + fix_dir = None + hostname, volname = args[:2] + + # Make sure stuff gets cleaned up, even if there are exceptions. + orig_dir = os.getcwd() + work_dir = tempfile.mkdtemp() + bricks = [] + def cleanup_workdir (): + os.chdir(orig_dir) + if options.verbose: + print("Cleaning up %s" % work_dir) + for b in bricks: + subprocess.call(["umount", b.path]) + shutil.rmtree(work_dir) + if not options.leave_mounted: + atexit.register(cleanup_workdir) + os.chdir(work_dir) + + # Mount each brick individually, so we can issue brick-specific calls. + if options.verbose: + print("Mounting subvolumes...") + index = 0 + volfile_pipe = get_bricks(hostname, volname) + all_xlators, last_xlator = volfilter.load(volfile_pipe) + for dht_vol in all_xlators.itervalues(): + if dht_vol.type == "cluster/distribute": + break + else: + print("no DHT volume found") + sys.exit(1) + for sv in dht_vol.subvols: + #print "found subvol %s" % sv.name + lpath = "%s/brick%s" % (work_dir, index) + index += 1 + mount_brick(lpath, all_xlators, sv) + bricks.append(Brick(lpath, sv.name)) + if index == 0: + print("no bricks") + sys.exit(1) + + # Collect all of the sizes. + if options.verbose: + print("Collecting information...") + total = 0 + for b in bricks: + info = os.statvfs(b.path) + # On FreeBSD f_bsize (info[0]) contains the optimal I/O size, + # not the block size as it's found on Linux. In this case we + # use f_frsize (info[1]). + if platform.system() == 'FreeBSD': + bsize = info[1] + else: + bsize = info[0] + # We want a standard unit even if different bricks use + # different block sizes. The size is chosen to avoid overflows + # for very large bricks with very small block sizes, but also + # accommodate filesystems which use very large block sizes to + # cheat on benchmarks. + blocksper100mb = 104857600 / bsize + if options.free_space: + size = info[3] / blocksper100mb + else: + size = info[2] / blocksper100mb + if size <= 0: + print("brick %s has invalid size %d" % (b.path, size)) + sys.exit(1) + b.set_size(size) + total += size + + # Collect all of the layout information. + for b in bricks: + hash_range = get_range(b.path) + if hash_range is not None: + rs, re = hash_range + if rs > re: + print("%s has backwards hash range" % b.path) + sys.exit(1) + b.set_range(hash_range[0], hash_range[1]) + + if options.verbose: + print("Calculating new layouts...") + calc_sizes(bricks, total) + bricks, used = normalize(bricks) + + # We can't afford O(n!) here, but O(n^2) should be OK and the result + # should be almost as good. + while used < len(bricks): + best_place = used + best_score = get_score(bricks) + for i in range(used): + new_bricks = bricks[:] + del new_bricks[used] + new_bricks.insert(i, bricks[used]) + new_score = get_score(new_bricks) + if new_score > best_score: + best_place = i + best_score = new_score + if best_place != used: + nb = bricks[used] + del bricks[used] + bricks.insert(best_place, nb) + used += 1 + + # Finalize whatever we decided on. + curr_hash = 0 + for b in bricks: + b.r_start = curr_hash + curr_hash += b.good_size + b.r_end = curr_hash - 1 + + print("Here are the xattr values for your size-weighted layout:") + for b in bricks: + print(" %s: 0x0000000200000000%08x%08x" % ( + b.sv_name, b.r_start, b.r_end)) + + if fix_dir: + if options.verbose: + print("Fixing layout for %s" % fix_dir) + for b in bricks: + value = "0x0000000200000000%08x%08x" % ( + b.r_start, b.r_end) + path = "%s/%s" % (b.path, fix_dir) + cmd = "setfattr -n trusted.glusterfs.dht -v %s %s" % ( + value, path) + print(cmd) + + if options.leave_mounted: + print("The following subvolumes are still mounted:") + for b in bricks: + print("%s on %s" % (b.sv_name, b.path)) + print("Don't forget to clean up when you're done.") + diff --git a/extras/run-gluster.tmpfiles.in b/extras/run-gluster.tmpfiles.in new file mode 100644 index 00000000000..329f2dde6db --- /dev/null +++ b/extras/run-gluster.tmpfiles.in @@ -0,0 +1,2 @@ +# hardcoding /run for now, should be detected while building from source? +d /run/gluster 0775 gluster gluster - diff --git a/extras/scale-n-defrag.sh b/extras/scale-n-defrag.sh deleted file mode 100644 index 1031b3931a8..00000000000 --- a/extras/scale-n-defrag.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -# This script runs over the GlusterFS mountpoint (from just one client) -# to handle the distribution of 'data', after the distribute translator's -# subvolumes count changes. -# -# (c) 2009 Gluster Inc, <http://www.gluster.com/> -# -# -# Make sure the following variables are properly initialized - -MOUNTPOINT=/tmp/testdir -directory_to_be_scaled="${MOUNTPOINT}/" - -logdir=$(dirname $0) -cd $logdir -LOGDIR=$(pwd) -cd - - -# The below command is enough to make sure the new layout will be scaled across new -# nodes. -find ${directory_to_be_scaled} -type d -exec setfattr -x "trusted.glusterfs.dht" {} \; - -# Now do a lookup on files so the scaling/re-hashing is done -find ${directory_to_be_scaled} > /dev/null - - -# copy the defrag (to copy data across for new nodes (for linkfiles)) -# - - -cd ${directory_to_be_scaled}; -for dir in *; do - echo "Defragmenting directory ${directory_to_be_scaled}/$dir ($LOGDIR/defrag-store-$dir.log)" - $LOGDIR/defrag.sh $dir >> $LOGDIR/defrag-store-$dir.log 2>&1 - echo Completed directory ${directory_to_be_scaled}/$dir -done diff --git a/extras/snap_scheduler/Makefile.am b/extras/snap_scheduler/Makefile.am new file mode 100644 index 00000000000..782f139016f --- /dev/null +++ b/extras/snap_scheduler/Makefile.am @@ -0,0 +1,9 @@ +snap_schedulerdir = $(sbindir)/ + +if WITH_SERVER +snap_scheduler_SCRIPTS = gcron.py snap_scheduler.py conf.py +endif + +EXTRA_DIST = gcron.py snap_scheduler.py conf.py + +CLEANFILES = diff --git a/extras/snap_scheduler/README.md b/extras/snap_scheduler/README.md new file mode 100644 index 00000000000..1316bb76469 --- /dev/null +++ b/extras/snap_scheduler/README.md @@ -0,0 +1,125 @@ +Snapshot Scheduler +============================== + +SUMMARY +------- + +GlusterFS volume snapshot provides point-in-time copy of a GlusterFS volume. Currently, GlusterFS volume snapshots can be easily scheduled by setting up cron jobs on one of the nodes in the GlusterFS trusted storage pool. This has a single point failure (SPOF), as scheduled jobs can be missed if the node running the cron jobs dies. + +We can avoid the SPOF by distributing the cron jobs to all nodes of the trusted storage pool. + +DETAILED DESCRIPTION +-------------------- + +The solution to the above problems involves the usage of: + +* A shared storage - This can be any shared storage (another gluster volume, a NFS mount, etc.) that will be used to share the schedule configuration and will help in the coordination of the jobs. +* An agent - This agent will perform the actual snapshot commands, instead of cron. It will contain the logic to perform coordinated snapshots. +* A helper script - This script will allow the user to initialise the scheduler on the local node, enable/disable scheduling, add/edit/list/delete snapshot schedules. +* cronie - It is the default cron daemon shipped with RHEL. It invokes the agent at the appropriate intervals as mentioned by the user to perform the snapshot operation on the volume as mentioned by the user in the schedule. + +INITIAL SETUP +------------- + +The administrator needs to create a shared storage that can be available to nodes across the cluster. A GlusterFS volume can also be used for the same. It is preferable that the *shared volume* be a replicate volume to avoid SPOF. + +Once the shared storage is created, it should be mounted on all nodes in the trusted storage pool which will be participating in the scheduling. The location where the shared_storage should be mounted (/var/run/gluster/snaps/shared_storage) in these nodes is fixed and is not configurable. Each node participating in the scheduling then needs to perform an initialisation of the snapshot scheduler by invoking the following: + +snap_scheduler.py init + +NOTE: This command needs to be run on all the nodes participating in the scheduling + +HELPER SCRIPT +------------- + +The helper script(snap_scheduler.py) will initialise the scheduler on the local node, enable/disable scheduling, add/edit/list/delete snapshot schedules. + +a) snap_scheduler.py init + +This command initialises the snap_scheduler and interfaces it with the crond running on the local node. This is the first step, before executing any scheduling related commands from a node. + +NOTE: The helper script needs to be run with this option on all the nodes participating in the scheduling. Other options of the helper script can be run independently from any node, where initialisation has been successfully completed. + +b) snap_scheduler.py enable + +The snap scheduler is disabled by default after initialisation. This command enables the snap scheduler. + +c) snap_scheduler.py disable + +This command disables the snap scheduler. + +d) snap_scheduler.py status + +This command displays the current status(Enabled/Disabled) of the snap scheduler. + +e) snap_scheduler.py add "Job Name" "Schedule" "Volume Name" + +This command adds a new snapshot schedule. All the arguments must be provided within double-quotes(""). It takes three arguments: + +-> Job Name: This name uniquely identifies this particular schedule, and can be used to reference this schedule for future events like edit/delete. If a schedule already exists for the specified Job Name, the add command will fail. + +-> Schedule: The schedules are accepted in the format crond understands:- + +Example of job definition: +.---------------- minute (0 - 59) +| .------------- hour (0 - 23) +| | .---------- day of month (1 - 31) +| | | .------- month (1 - 12) OR jan,feb,mar,apr ... +| | | | .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat +| | | | | +* * * * * user-name command to be executed +Although we accept all valid cron schedules, currently we support granularity of snapshot schedules to a maximum of half-hourly snapshots. + +-> Volume Name: The name of the volume on which the scheduled snapshot operation will be performed. + +f) snap_scheduler.py edit "Job Name" "Schedule" "Volume Name" + +This command edits an existing snapshot schedule. It takes the same three arguments that the add option takes. All the arguments must be provided within double-quotes(""). If a schedule does not exists for the specified Job Name, the edit command will fail. + +g) snap_scheduler.py delete "Job Name" + +This command deletes an existing snapshot schedule. It takes the job name of the schedule as argument. The argument must be provided within double-quotes(""). If a schedule does not exists for the specified Job Name, the delete command will fail. + +h) snap_scheduler.py list + +This command lists the existing snapshot schedules in the following manner: Pseudocode: + +# snap_scheduler.py list +JOB_NAME SCHEDULE OPERATION VOLUME NAME +-------------------------------------------------------------------- +Job0 * * * * * Snapshot Create test_vol + +THE AGENT +--------- + +The snapshots scheduled with the help of the helper script, are read by crond which then invokes the agent(gcron.py) at the scheduled intervals to perform the snapshot operations on the specified volumes. It then performs the scheduled snapshots using the following algorithm to coordinate. + +start_time = get current time +lock_file = job_name passed as an argument +vol_name = volume name psased as an argument +try POSIX locking the $lock_file + if lock is obtained, then + mod_time = Get modification time of $entry + if $mod_time < $start_time, then + Take snapshot of $entry.name (Volume name) + if snapshot failed, then + log the failure + Update modification time of $entry to current time + unlock the $entry + +The coordination with other scripts running on other nodes, is handled by the use of POSIX locks. All the instances of the script will attempt to lock the lock_file which is essentialy an empty file with the job name, and one which gets the lock will take the snapshot. + +To prevent redoing a done task, the script will make use of the mtime attribute of the entry. At the beginning execution, the script would have saved its start time. Once the script obtains the lock on the lock_file, before taking the snapshot, it compares the mtime of the entry with the start time. The snapshot will only be taken if the mtime is smaller than start time. Once the snapshot command completes, the script will update the mtime of the lock_file to the current time before unlocking. + +If a snapshot command fails, the script will log the failure (in syslog) and continue with its operation. It will not attempt to retry the failed snapshot in the current schedule, but will attempt it again in the next schedules. It is left to the administrator to monitor the logs and decide what to do after a failure. + +ASSUMPTIONS AND LIMITATIONS +--------------------------- + +It is assumed that all nodes in the have their times synced using NTP or any other mechanism. This is a hard requirement for this feature to work. + +The administrator needs to have python2.7 or higher installed, as well as the argparse module installed, to be able to use the helper script(snap_scheduler.py). + +There is a latency of one minute, between providing a command by the helper script and that command taking effect. Hence, currently we do not support snapshot schedules with per minute granularity. + +The administrator can however leverage the scheduler to schedule snapshots with granularity of half-hourly/hourly/daily/weekly/monthly/yearly periodic intervals. They can also schedule snapshots, which are customised mentioning which minute of the hour, which day of the week, which week of the month, and which month of the year, they want to schedule the snapshot operation. diff --git a/extras/snap_scheduler/conf.py.in b/extras/snap_scheduler/conf.py.in new file mode 100644 index 00000000000..6dcca0534a7 --- /dev/null +++ b/extras/snap_scheduler/conf.py.in @@ -0,0 +1,11 @@ +# +# Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. + +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. +# + +GLUSTERFS_LIBEXECDIR = '@GLUSTERFS_LIBEXECDIR@' diff --git a/extras/snap_scheduler/gcron.py b/extras/snap_scheduler/gcron.py new file mode 100755 index 00000000000..0e4df77d481 --- /dev/null +++ b/extras/snap_scheduler/gcron.py @@ -0,0 +1,190 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +from __future__ import print_function +import subprocess +import os +import os.path +import sys +import time +import logging +import logging.handlers +import fcntl + + +GCRON_TASKS = "/run/gluster/shared_storage/snaps/glusterfs_snap_cron_tasks" +GCRON_CROND_TASK = "/etc/cron.d/glusterfs_snap_cron_tasks" +GCRON_RELOAD_FLAG = "/var/run/gluster/crond_task_reload_flag" +LOCK_FILE_DIR = "/run/gluster/shared_storage/snaps/lock_files/" +log = logging.getLogger("gcron-logger") +start_time = 0.0 + + +def initLogger(script_name): + log.setLevel(logging.DEBUG) + logFormat = "[%(asctime)s %(filename)s:%(lineno)s %(funcName)s] "\ + "%(levelname)s %(message)s" + formatter = logging.Formatter(logFormat) + + sh = logging.handlers.SysLogHandler() + sh.setLevel(logging.ERROR) + sh.setFormatter(formatter) + + process = subprocess.Popen(["gluster", "--print-logdir"], + stdout=subprocess.PIPE, + universal_newlines=True) + out, err = process.communicate() + if process.returncode == 0: + logfile = os.path.join(out.strip(), script_name[:-3]+".log") + + fh = logging.FileHandler(logfile) + fh.setLevel(logging.DEBUG) + fh.setFormatter(formatter) + + log.addHandler(sh) + log.addHandler(fh) + + +def takeSnap(volname="", snapname=""): + success = True + if volname == "": + log.debug("No volname given") + return False + if snapname == "": + log.debug("No snapname given") + return False + + cli = ["gluster", + "snapshot", + "create", + snapname, + volname] + log.debug("Running command '%s'", " ".join(cli)) + + p = subprocess.Popen(cli, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate() + rv = p.returncode + + log.debug("Command '%s' returned '%d'", " ".join(cli), rv) + + if rv: + log.error("Snapshot of %s failed", volname) + log.error("Command output:") + log.error(err) + success = False + else: + log.info("Snapshot of %s successful", volname) + + return success + + +def doJob(name, lockFile, jobFunc, volname): + success = True + try: + f = os.open(lockFile, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK) + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + mtime = os.path.getmtime(lockFile) + global start_time + log.debug("%s last modified at %s", lockFile, time.ctime(mtime)) + if mtime < start_time: + log.debug("Processing job %s", name) + if jobFunc(volname, name): + log.info("Job %s succeeded", name) + else: + log.error("Job %s failed", name) + success = False + os.utime(lockFile, None) + else: + log.info("Job %s has been processed already", name) + fcntl.flock(f, fcntl.LOCK_UN) + except (OSError, IOError): + log.info("Job %s is being processed by another agent", name) + os.close(f) + except (OSError, IOError) as e: + log.debug("Failed to open lock file %s : %s", lockFile, e) + log.error("Failed to process job %s", name) + success = False + + return success + + +def main(): + script_name = os.path.basename(__file__) + initLogger(script_name) + global start_time + if sys.argv[1] == "--update": + if not os.path.exists(GCRON_TASKS): + # Create a flag in /var/run/gluster which indicates that this + # node doesn't have access to GCRON_TASKS right now, so that + # when the mount is available and GCRON_TASKS is available + # the flag will tell this routine to reload GCRON_CROND_TASK + try: + f = os.open(GCRON_RELOAD_FLAG, + os.O_CREAT | os.O_NONBLOCK, 0o644) + os.close(f) + except OSError as e: + if errno != EEXIST: + log.error("Failed to create %s : %s", + GCRON_RELOAD_FLAG, e) + output("Failed to create %s. Error: %s" + % (GCRON_RELOAD_FLAG, e)) + return + + if not os.path.exists(GCRON_CROND_TASK): + return + + # As GCRON_TASKS exists now, we should check if GCRON_RELOAD_FLAG + # also exists. If so we should touch GCRON_CROND_TASK and remove + # the GCRON_RELOAD_FLAG + if os.path.exists(GCRON_RELOAD_FLAG): + try: + os.remove(GCRON_RELOAD_FLAG); + process = subprocess.Popen(["touch", "-h", GCRON_CROND_TASK], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = process.communicate() + if process.returncode != 0: + log.error("Failed to touch %s. Error: %s.", + GCRON_CROND_TASK, err) + except (IOError, OSError) as e: + log.error("Failed to touch %s. Error: %s.", + GCRON_CROND_TASK, e) + return + if os.lstat(GCRON_TASKS).st_mtime > \ + os.lstat(GCRON_CROND_TASK).st_mtime: + try: + process = subprocess.Popen(["touch", "-h", GCRON_CROND_TASK], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = process.communicate() + if process.returncode != 0: + log.error("Failed to touch %s. Error: %s.", + GCRON_CROND_TASK, err) + except IOError as e: + log.error("Failed to touch %s. Error: %s.", + GCRON_CROND_TASK, e) + return + + volname = sys.argv[1] + jobname = sys.argv[2] + locking_file = os.path.join(LOCK_FILE_DIR, jobname) + log.debug("locking_file = %s", locking_file) + log.debug("volname = %s", volname) + log.debug("jobname = %s", jobname) + + start_time = int(time.time()) + + doJob("Scheduled-" + jobname + "-" + volname, locking_file, takeSnap, volname) + + +if __name__ == "__main__": + main() diff --git a/extras/snap_scheduler/snap_scheduler.py b/extras/snap_scheduler/snap_scheduler.py new file mode 100755 index 00000000000..e8fcc449a9b --- /dev/null +++ b/extras/snap_scheduler/snap_scheduler.py @@ -0,0 +1,941 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +from __future__ import print_function +import subprocess +import os +import os.path +import logging +import argparse +import fcntl +import logging.handlers +import sys +import shutil +from errno import EEXIST +from conf import GLUSTERFS_LIBEXECDIR +sys.path.insert(1, GLUSTERFS_LIBEXECDIR) + +EVENTS_ENABLED = True +try: + from events.eventtypes import SNAPSHOT_SCHEDULER_INITIALISED \ + as EVENT_SNAPSHOT_SCHEDULER_INITIALISED + from events.eventtypes import SNAPSHOT_SCHEDULER_INIT_FAILED \ + as EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED + from events.eventtypes import SNAPSHOT_SCHEDULER_DISABLED \ + as EVENT_SNAPSHOT_SCHEDULER_DISABLED + from events.eventtypes import SNAPSHOT_SCHEDULER_DISABLE_FAILED \ + as EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED + from events.eventtypes import SNAPSHOT_SCHEDULER_ENABLED \ + as EVENT_SNAPSHOT_SCHEDULER_ENABLED + from events.eventtypes import SNAPSHOT_SCHEDULER_ENABLE_FAILED \ + as EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED + from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_ADDED \ + as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED + from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED \ + as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED + from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_DELETED \ + as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED + from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED \ + as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED + from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_EDITED \ + as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED + from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED \ + as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED +except ImportError: + # Events APIs not installed, dummy eventtypes with None + EVENTS_ENABLED = False + EVENT_SNAPSHOT_SCHEDULER_INITIALISED = None + EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED = None + EVENT_SNAPSHOT_SCHEDULER_DISABLED = None + EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED = None + EVENT_SNAPSHOT_SCHEDULER_ENABLED = None + EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED = None + EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED = None + EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED = None + EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED = None + EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED = None + EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED = None + EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED = None + +SCRIPT_NAME = "snap_scheduler" +scheduler_enabled = False +log = logging.getLogger(SCRIPT_NAME) +SHARED_STORAGE_DIR="/run/gluster/shared_storage" +GCRON_DISABLED = SHARED_STORAGE_DIR+"/snaps/gcron_disabled" +GCRON_ENABLED = SHARED_STORAGE_DIR+"/snaps/gcron_enabled" +GCRON_TASKS = SHARED_STORAGE_DIR+"/snaps/glusterfs_snap_cron_tasks" +GCRON_CROND_TASK = "/etc/cron.d/glusterfs_snap_cron_tasks" +LOCK_FILE_DIR = SHARED_STORAGE_DIR+"/snaps/lock_files/" +LOCK_FILE = LOCK_FILE_DIR+"lock_file" +TMP_FILE = SHARED_STORAGE_DIR+"/snaps/tmp_file" +GCRON_UPDATE_TASK = "/etc/cron.d/gcron_update_task" +CURRENT_SCHEDULER = SHARED_STORAGE_DIR+"/snaps/current_scheduler" +tasks = {} +longest_field = 12 +current_scheduler = "" + +INTERNAL_ERROR = 2 +SHARED_STORAGE_DIR_DOESNT_EXIST = 3 +SHARED_STORAGE_NOT_MOUNTED = 4 +ANOTHER_TRANSACTION_IN_PROGRESS = 5 +INIT_FAILED = 6 +SCHEDULING_ALREADY_DISABLED = 7 +SCHEDULING_ALREADY_ENABLED = 8 +NODE_NOT_INITIALISED = 9 +ANOTHER_SCHEDULER_ACTIVE = 10 +JOB_ALREADY_EXISTS = 11 +JOB_NOT_FOUND = 12 +INVALID_JOBNAME = 13 +INVALID_VOLNAME = 14 +INVALID_SCHEDULE = 15 +INVALID_ARG = 16 +VOLUME_DOES_NOT_EXIST = 17 + +def print_error (error_num): + if error_num == INTERNAL_ERROR: + return "Internal Error" + elif error_num == SHARED_STORAGE_DIR_DOESNT_EXIST: + return "The shared storage directory ("+SHARED_STORAGE_DIR+")" \ + " does not exist." + elif error_num == SHARED_STORAGE_NOT_MOUNTED: + return "The shared storage directory ("+SHARED_STORAGE_DIR+")" \ + " is not mounted." + elif error_num == ANOTHER_TRANSACTION_IN_PROGRESS: + return "Another transaction is in progress." + elif error_num == INIT_FAILED: + return "Initialisation failed." + elif error_num == SCHEDULING_ALREADY_DISABLED: + return "Snapshot scheduler is already disabled." + elif error_num == SCHEDULING_ALREADY_ENABLED: + return "Snapshot scheduler is already enabled." + elif error_num == NODE_NOT_INITIALISED: + return "The node is not initialised." + elif error_num == ANOTHER_SCHEDULER_ACTIVE: + return "Another scheduler is active." + elif error_num == JOB_ALREADY_EXISTS: + return "The job already exists." + elif error_num == JOB_NOT_FOUND: + return "The job cannot be found." + elif error_num == INVALID_JOBNAME: + return "The job name is invalid." + elif error_num == INVALID_VOLNAME: + return "The volume name is invalid." + elif error_num == INVALID_SCHEDULE: + return "The schedule is invalid." + elif error_num == INVALID_ARG: + return "The argument is invalid." + elif error_num == VOLUME_DOES_NOT_EXIST: + return "The volume does not exist." + +def output(msg): + print("%s: %s" % (SCRIPT_NAME, msg)) + + +def initLogger(): + log.setLevel(logging.DEBUG) + logFormat = "[%(asctime)s %(filename)s:%(lineno)s %(funcName)s] "\ + "%(levelname)s %(message)s" + formatter = logging.Formatter(logFormat) + + sh = logging.handlers.SysLogHandler() + sh.setLevel(logging.ERROR) + sh.setFormatter(formatter) + + process = subprocess.Popen(["gluster", "--print-logdir"], + stdout=subprocess.PIPE, universal_newlines=True) + logfile = os.path.join(process.stdout.read()[:-1], SCRIPT_NAME + ".log") + + fh = logging.FileHandler(logfile) + fh.setLevel(logging.DEBUG) + fh.setFormatter(formatter) + + log.addHandler(sh) + log.addHandler(fh) + + +def scheduler_status(): + ret = INTERNAL_ERROR + global scheduler_enabled + try: + f = os.path.realpath(GCRON_TASKS) + if f != os.path.realpath(GCRON_ENABLED) or not os.path.exists(GCRON_ENABLED): + log.info("Snapshot scheduler is currently disabled.") + scheduler_enabled = False + else: + log.info("Snapshot scheduler is currently enabled.") + scheduler_enabled = True + ret = 0 + except: + log.error("Failed to enable snapshot scheduling. Error: " + "Failed to check the status of %s.", GCRON_DISABLED) + + return ret + +def enable_scheduler(): + ret = scheduler_status() + if ret == 0: + if not scheduler_enabled: + + # Check if another scheduler is active. + ret = get_current_scheduler() + if ret == 0: + if (current_scheduler != "none"): + print_str = "Failed to enable snapshot scheduling. " \ + "Error: Another scheduler is active." + log.error(print_str) + output(print_str) + ret = ANOTHER_SCHEDULER_ACTIVE + return ret + else: + print_str = "Failed to get current scheduler info." + log.error(print_str) + output(print_str) + return ret + + log.info("Enabling snapshot scheduler.") + try: + if os.path.exists(GCRON_DISABLED): + os.remove(GCRON_DISABLED) + if os.path.lexists(GCRON_TASKS): + os.remove(GCRON_TASKS) + try: + f = os.open(GCRON_ENABLED, os.O_CREAT | os.O_NONBLOCK, + 0o644) + os.close(f) + except OSError as e: + log.error("Failed to open %s. Error: %s.", + GCRON_ENABLED, e) + ret = INTERNAL_ERROR + return ret + os.symlink(GCRON_ENABLED, GCRON_TASKS) + update_current_scheduler("cli") + log.info("Snapshot scheduling is enabled") + output("Snapshot scheduling is enabled") + ret = 0 + except OSError as e: + print_str = ("Failed to enable snapshot scheduling." + "Error: {{}}" + e) + log.error(print_str) + output(print_str) + ret = INTERNAL_ERROR + else: + print_str = "Failed to enable snapshot scheduling. " \ + "Error: Snapshot scheduling is already enabled." + log.error(print_str) + output(print_str) + ret = SCHEDULING_ALREADY_ENABLED + else: + print_str = "Failed to enable snapshot scheduling. " \ + "Error: Failed to check scheduler status." + log.error(print_str) + output(print_str) + + return ret + + +def disable_scheduler(): + ret = scheduler_status() + if ret == 0: + if scheduler_enabled: + log.info("Disabling snapshot scheduler.") + try: + # Check if another scheduler is active. If not, then + # update current scheduler to "none". Else do nothing. + ret = get_current_scheduler() + if ret == 0: + if (current_scheduler == "cli"): + update_current_scheduler("none") + else: + print_str = "Failed to disable snapshot scheduling. " \ + "Error: Failed to get current scheduler info." + log.error(print_str) + output(print_str) + return ret + + if os.path.exists(GCRON_DISABLED): + os.remove(GCRON_DISABLED) + if os.path.lexists(GCRON_TASKS): + os.remove(GCRON_TASKS) + f = os.open(GCRON_DISABLED, os.O_CREAT, 0o644) + os.close(f) + os.symlink(GCRON_DISABLED, GCRON_TASKS) + log.info("Snapshot scheduling is disabled") + output("Snapshot scheduling is disabled") + ret = 0 + except OSError as e: + print_str = ("Failed to disable snapshot scheduling. Error: " + + e) + log.error(print_str) + output(print_str) + ret = INTERNAL_ERROR + else: + print_str = "Failed to disable scheduling. " \ + "Error: Snapshot scheduling is already disabled." + log.error(print_str) + output(print_str) + ret = SCHEDULING_ALREADY_DISABLED + else: + print_str = "Failed to disable snapshot scheduling. " \ + "Error: Failed to check scheduler status." + log.error(print_str) + output(print_str) + ret = INTERNAL_ERROR + + return ret + + +def load_tasks_from_file(): + global tasks + global longest_field + try: + with open(GCRON_ENABLED, 'r') as f: + for line in f: + line = line.rstrip('\n') + if not line: + break + line = line.split("gcron.py") + schedule = line[0].split("root")[0].rstrip(' ') + line = line[1].split(" ") + volname = line[1] + jobname = line[2] + longest_field = max(longest_field, len(jobname), len(volname), + len(schedule)) + tasks[jobname] = schedule+":"+volname + f.close() + ret = 0 + except IOError as e: + log.error("Failed to open %s. Error: %s.", GCRON_ENABLED, e) + ret = INTERNAL_ERROR + + return ret + + +def get_current_scheduler(): + global current_scheduler + try: + with open(CURRENT_SCHEDULER, 'r') as f: + current_scheduler = f.readline().rstrip('\n') + f.close() + ret = 0 + except IOError as e: + log.error("Failed to open %s. Error: %s.", CURRENT_SCHEDULER, e) + ret = INTERNAL_ERROR + + return ret + + +def list_schedules(): + log.info("Listing snapshot schedules.") + ret = load_tasks_from_file() + if ret == 0: + if len(tasks) == 0: + output("No snapshots scheduled") + else: + jobname = "JOB_NAME".ljust(longest_field+5) + schedule = "SCHEDULE".ljust(longest_field+5) + operation = "OPERATION".ljust(longest_field+5) + volname = "VOLUME NAME".ljust(longest_field+5) + hyphens = "".ljust((longest_field+5) * 4, '-') + print(jobname+schedule+operation+volname) + print(hyphens) + for key in sorted(tasks): + jobname = key.ljust(longest_field+5) + schedule = tasks[key].split(":")[0].ljust( + longest_field + 5) + volname = tasks[key].split(":")[1].ljust( + longest_field + 5) + operation = "Snapshot Create".ljust(longest_field+5) + print(jobname+schedule+operation+volname) + ret = 0 + else: + print_str = "Failed to list snapshot schedules. " \ + "Error: Failed to load tasks from "+GCRON_ENABLED + log.error(print_str) + output(print_str) + + return ret + + +def write_tasks_to_file(): + try: + with open(TMP_FILE, "w", 0o644) as f: + # If tasks is empty, just create an empty tmp file + if len(tasks) != 0: + for key in sorted(tasks): + jobname = key + schedule = tasks[key].split(":")[0] + volname = tasks[key].split(":")[1] + f.write("%s root PATH=$PATH:/usr/local/sbin:/usr/sbin " + "gcron.py %s %s\n" % (schedule, volname, jobname)) + f.write("\n") + f.flush() + os.fsync(f.fileno()) + f.close() + except IOError as e: + log.error("Failed to open %s. Error: %s.", TMP_FILE, e) + ret = INTERNAL_ERROR + return ret + + shutil.move(TMP_FILE, GCRON_ENABLED) + ret = 0 + + return ret + +def update_current_scheduler(data): + try: + with open(TMP_FILE, "w", 0o644) as f: + f.write("%s" % data) + f.flush() + os.fsync(f.fileno()) + f.close() + except IOError as e: + log.error("Failed to open %s. Error: %s.", TMP_FILE, e) + ret = INTERNAL_ERROR + return ret + + shutil.move(TMP_FILE, CURRENT_SCHEDULER) + ret = 0 + + return ret + + +def isVolumePresent(volname): + success = False + if volname == "": + log.debug("No volname given") + return success + + cli = ["gluster", + "volume", + "info", + volname] + log.debug("Running command '%s'", " ".join(cli)) + + p = subprocess.Popen(cli, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate() + rv = p.returncode + + log.debug("Command '%s' returned '%d'", " ".join(cli), rv) + + if rv: + log.error("Command output:") + log.error(err) + else: + success = True; + + return success + + +def add_schedules(jobname, schedule, volname): + log.info("Adding snapshot schedules.") + ret = load_tasks_from_file() + if ret == 0: + if jobname in tasks: + print_str = ("%s already exists in schedule. Use " + "'edit' to modify %s" % (jobname, jobname)) + log.error(print_str) + output(print_str) + ret = JOB_ALREADY_EXISTS + else: + if not isVolumePresent(volname): + print_str = ("Volume %s does not exist. Create %s and retry." % + (volname, volname)) + log.error(print_str) + output(print_str) + ret = VOLUME_DOES_NOT_EXIST + else: + tasks[jobname] = schedule + ":" + volname + ret = write_tasks_to_file() + if ret == 0: + # Create a LOCK_FILE for the job + job_lockfile = LOCK_FILE_DIR + jobname + try: + f = os.open(job_lockfile, os.O_CREAT | os.O_NONBLOCK, + 0o644) + os.close(f) + except OSError as e: + log.error("Failed to open %s. Error: %s.", + job_lockfile, e) + ret = INTERNAL_ERROR + return ret + log.info("Successfully added snapshot schedule %s" % + jobname) + output("Successfully added snapshot schedule") + ret = 0 + else: + print_str = "Failed to add snapshot schedule. " \ + "Error: Failed to load tasks from "+GCRON_ENABLED + log.error(print_str) + output(print_str) + + return ret + + +def delete_schedules(jobname): + log.info("Delete snapshot schedules.") + ret = load_tasks_from_file() + if ret == 0: + if jobname in tasks: + del tasks[jobname] + ret = write_tasks_to_file() + if ret == 0: + # Delete the LOCK_FILE for the job + job_lockfile = LOCK_FILE_DIR+jobname + try: + os.remove(job_lockfile) + except OSError as e: + log.error("Failed to open %s. Error: %s.", + job_lockfile, e) + ret = INTERNAL_ERROR + return ret + log.info("Successfully deleted snapshot schedule %s" + % jobname) + output("Successfully deleted snapshot schedule") + ret = 0 + else: + print_str = ("Failed to delete %s. Error: No such " + "job scheduled" % jobname) + log.error(print_str) + output(print_str) + ret = JOB_NOT_FOUND + else: + print_str = "Failed to delete snapshot schedule. " \ + "Error: Failed to load tasks from "+GCRON_ENABLED + log.error(print_str) + output(print_str) + + return ret + + +def edit_schedules(jobname, schedule, volname): + log.info("Editing snapshot schedules.") + ret = load_tasks_from_file() + if ret == 0: + if jobname in tasks: + if not isVolumePresent(volname): + print_str = ("Volume %s does not exist. Create %s and retry." % + (volname, volname)) + log.error(print_str) + output(print_str) + ret = VOLUME_DOES_NOT_EXIST + else: + tasks[jobname] = schedule+":"+volname + ret = write_tasks_to_file() + if ret == 0: + log.info("Successfully edited snapshot schedule %s" % + jobname) + output("Successfully edited snapshot schedule") + else: + print_str = ("Failed to edit %s. Error: No such " + "job scheduled" % jobname) + log.error(print_str) + output(print_str) + ret = JOB_NOT_FOUND + else: + print_str = "Failed to edit snapshot schedule. " \ + "Error: Failed to load tasks from "+GCRON_ENABLED + log.error(print_str) + output(print_str) + + return ret + +def get_bool_val(): + getsebool_cli = ["getsebool", + "-a"] + p1 = subprocess.Popen(getsebool_cli, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + grep_cmd = ["grep", + "cron_system_cronjob_use_shares"] + p2 = subprocess.Popen(grep_cmd, stdin=p1.stdout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + p1.stdout.close() + output, err = p2.communicate() + rv = p2.returncode + + if rv: + log.error("Command output:") + log.error(err) + return -1 + + bool_val = output.split()[2] + log.debug("Bool value = '%s'", bool_val) + + return bool_val + +def get_selinux_status(): + getenforce_cli = ["getenforce"] + log.debug("Running command '%s'", " ".join(getenforce_cli)) + + try: + p1 = subprocess.Popen(getenforce_cli, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError as oserr: + log.error("Failed to run the command \"getenforce\". Error: %s" %\ + oserr) + return -1 + + output, err = p1.communicate() + rv = p1.returncode + + if rv: + log.error("Command output:") + log.error(err) + return -1 + else: + selinux_status=output.rstrip() + log.debug("selinux status: %s", selinux_status) + + return selinux_status + +def set_cronjob_user_share(): + selinux_status = get_selinux_status() + if (selinux_status == -1): + log.error("Failed to get selinux status") + return -1 + elif (selinux_status == "Disabled"): + return 0 + + bool_val = get_bool_val() + # In case of a failure (where the boolean value is not) + # present in the system, we should not proceed further + # We should only proceed when the value is "off" + if (bool_val == -1 or bool_val != "off"): + return 0 + + setsebool_cli = ["setsebool", "-P", + "cron_system_cronjob_use_shares", + "on"] + log.debug("Running command '%s'", " ".join(setsebool_cli)) + + p1 = subprocess.Popen(setsebool_cli, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + output, err = p1.communicate() + rv = p1.returncode + + if rv: + log.error("Command output:") + log.error(err) + return rv + + bool_val = get_bool_val() + if (bool_val == "on"): + return 0 + else: + # In case of an error or if boolean is not on + # we return a failure here + return -1 + +def initialise_scheduler(): + ret = set_cronjob_user_share() + if ret: + log.error("Failed to set selinux boolean " + "cron_system_cronjob_use_shares to 'on'") + return ret + + try: + with open(TMP_FILE, "w+", 0o644) as f: + updater = ("* * * * * root PATH=$PATH:/usr/local/sbin:" + "/usr/sbin gcron.py --update\n") + f.write("%s\n" % updater) + f.flush() + os.fsync(f.fileno()) + f.close() + except IOError as e: + log.error("Failed to open %s. Error: %s.", TMP_FILE, e) + ret = INIT_FAILED + return ret + + shutil.move(TMP_FILE, GCRON_UPDATE_TASK) + + if not os.path.lexists(GCRON_TASKS): + try: + f = open(GCRON_TASKS, "w", 0o644) + f.close() + except IOError as e: + log.error("Failed to open %s. Error: %s.", GCRON_TASKS, e) + ret = INIT_FAILED + return ret + + if os.path.lexists(GCRON_CROND_TASK): + os.remove(GCRON_CROND_TASK) + + os.symlink(GCRON_TASKS, GCRON_CROND_TASK) + + log.info("Successfully initialised snapshot scheduler for this node") + output("Successfully initialised snapshot scheduler for this node") + gf_event (EVENT_SNAPSHOT_SCHEDULER_INITIALISED, status="Success") + + ret = 0 + return ret + + +def syntax_checker(args): + if hasattr(args, 'jobname'): + if (len(args.jobname.split()) != 1): + output("Invalid Jobname. Jobname should not be empty and should not contain \" \" character.") + ret = INVALID_JOBNAME + return ret + args.jobname=args.jobname.strip() + + if hasattr(args, 'volname'): + if (len(args.volname.split()) != 1): + output("Invalid Volname. Volname should not be empty and should not contain \" \" character.") + ret = INVALID_VOLNAME + return ret + args.volname=args.volname.strip() + + if hasattr(args, 'schedule'): + if (len(args.schedule.split()) != 5): + output("Invalid Schedule. Please refer to the following for adding a valid cron schedule") + print ("* * * * *") + print ("| | | | |") + print ("| | | | +---- Day of the Week (range: 1-7, 1 standing for Monday)") + print ("| | | +------ Month of the Year (range: 1-12)") + print ("| | +-------- Day of the Month (range: 1-31)") + print ("| +---------- Hour (range: 0-23)") + print ("+------------ Minute (range: 0-59)") + ret = INVALID_SCHEDULE + return ret + + ret = 0 + return ret + + +def perform_operation(args): + if not os.path.exists(CURRENT_SCHEDULER): + update_current_scheduler("none") + + # Initialise snapshot scheduler on local node + if args.action == "init": + ret = initialise_scheduler() + if ret != 0: + output("Failed to initialise snapshot scheduling") + gf_event (EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED, + error=print_error(ret)) + return ret + + # Disable snapshot scheduler + if args.action == "disable_force": + ret = disable_scheduler() + if ret == 0: + subprocess.Popen(["touch", "-h", GCRON_TASKS]) + gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLED, + status="Successfully Disabled") + else: + gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED, + error=print_error(ret)) + return ret + + # Check if the symlink to GCRON_TASKS is properly set in the shared storage + if (not os.path.lexists(GCRON_UPDATE_TASK) or + not os.path.lexists(GCRON_CROND_TASK) or + os.readlink(GCRON_CROND_TASK) != GCRON_TASKS): + print_str = ("Please run 'snap_scheduler.py' init to initialise " + "the snap scheduler for the local node.") + log.error(print_str) + output(print_str) + ret = NODE_NOT_INITIALISED + return ret + + # Check status of snapshot scheduler. + if args.action == "status": + ret = scheduler_status() + if ret == 0: + if scheduler_enabled: + output("Snapshot scheduling status: Enabled") + else: + output("Snapshot scheduling status: Disabled") + else: + output("Failed to check status of snapshot scheduler") + return ret + + # Enable snapshot scheduler + if args.action == "enable": + ret = enable_scheduler() + if ret == 0: + subprocess.Popen(["touch", "-h", GCRON_TASKS]) + gf_event (EVENT_SNAPSHOT_SCHEDULER_ENABLED, + status="Successfully Enabled") + else: + gf_event (EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED, + error=print_error(ret)) + return ret + + # Disable snapshot scheduler + if args.action == "disable": + ret = disable_scheduler() + if ret == 0: + subprocess.Popen(["touch", "-h", GCRON_TASKS]) + gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLED, + status="Successfully Disabled") + else: + gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED, + error=print_error(ret)) + return ret + + # List snapshot schedules + if args.action == "list": + ret = list_schedules() + return ret + + # Add snapshot schedules + if args.action == "add": + ret = syntax_checker(args) + if ret != 0: + return ret + ret = add_schedules(args.jobname, args.schedule, args.volname) + if ret == 0: + subprocess.Popen(["touch", "-h", GCRON_TASKS]) + gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED, + status="Successfully added job "+args.jobname) + else: + gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED, + status="Failed to add job "+args.jobname, + error=print_error(ret)) + return ret + + # Delete snapshot schedules + if args.action == "delete": + ret = syntax_checker(args) + if ret != 0: + return ret + ret = delete_schedules(args.jobname) + if ret == 0: + subprocess.Popen(["touch", "-h", GCRON_TASKS]) + gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED, + status="Successfully deleted job "+args.jobname) + else: + gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED, + status="Failed to delete job "+args.jobname, + error=print_error(ret)) + return ret + + # Edit snapshot schedules + if args.action == "edit": + ret = syntax_checker(args) + if ret != 0: + return ret + ret = edit_schedules(args.jobname, args.schedule, args.volname) + if ret == 0: + subprocess.Popen(["touch", "-h", GCRON_TASKS]) + gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED, + status="Successfully edited job "+args.jobname) + else: + gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED, + status="Failed to edit job "+args.jobname, + error=print_error(ret)) + return ret + + ret = INVALID_ARG + return ret + +def gf_event(event_type, **kwargs): + if EVENTS_ENABLED: + from events.gf_event import gf_event as gfevent + gfevent(event_type, **kwargs) + + +def main(argv): + initLogger() + ret = -1 + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="action", + metavar=('{init, status, enable,' + ' disable, list, add,' + ' delete, edit}')) + subparsers.add_parser('init', + help="Initialise the node for snapshot scheduling") + + subparsers.add_parser("status", + help="Check if snapshot scheduling is " + "enabled or disabled") + subparsers.add_parser("enable", + help="Enable snapshot scheduling") + subparsers.add_parser("disable", + help="Disable snapshot scheduling") + subparsers.add_parser("disable_force") + subparsers.add_parser("list", + help="List snapshot schedules") + parser_add = subparsers.add_parser("add", + help="Add snapshot schedules") + parser_add.add_argument("jobname", help="Job Name") + parser_add.add_argument("schedule", help="Schedule") + parser_add.add_argument("volname", help="Volume Name") + + parser_delete = subparsers.add_parser("delete", + help="Delete snapshot schedules") + parser_delete.add_argument("jobname", help="Job Name") + parser_edit = subparsers.add_parser("edit", + help="Edit snapshot schedules") + parser_edit.add_argument("jobname", help="Job Name") + parser_edit.add_argument("schedule", help="Schedule") + parser_edit.add_argument("volname", help="Volume Name") + + args = parser.parse_args(argv) + + if not os.path.exists(SHARED_STORAGE_DIR): + output("Failed: "+SHARED_STORAGE_DIR+" does not exist.") + return SHARED_STORAGE_DIR_DOESNT_EXIST + + if not os.path.ismount(SHARED_STORAGE_DIR): + output("Failed: Shared storage is not mounted at "+SHARED_STORAGE_DIR) + return SHARED_STORAGE_NOT_MOUNTED + + if not os.path.exists(SHARED_STORAGE_DIR+"/snaps/"): + try: + os.makedirs(SHARED_STORAGE_DIR+"/snaps/") + except OSError as e: + if errno != EEXIST: + log.error("Failed to create %s : %s", SHARED_STORAGE_DIR+"/snaps/", e) + output("Failed to create %s. Error: %s" + % (SHARED_STORAGE_DIR+"/snaps/", e)) + return INTERNAL_ERROR + + if not os.path.exists(GCRON_ENABLED): + f = os.open(GCRON_ENABLED, os.O_CREAT | os.O_NONBLOCK, 0o644) + os.close(f) + + if not os.path.exists(LOCK_FILE_DIR): + try: + os.makedirs(LOCK_FILE_DIR) + except OSError as e: + if errno != EEXIST: + log.error("Failed to create %s : %s", LOCK_FILE_DIR, e) + output("Failed to create %s. Error: %s" + % (LOCK_FILE_DIR, e)) + return INTERNAL_ERROR + + try: + f = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK, 0o644) + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + ret = perform_operation(args) + fcntl.flock(f, fcntl.LOCK_UN) + except IOError: + log.info("%s is being processed by another agent.", LOCK_FILE) + output("Another snap_scheduler command is running. " + "Please try again after some time.") + return ANOTHER_TRANSACTION_IN_PROGRESS + os.close(f) + except OSError as e: + log.error("Failed to open %s : %s", LOCK_FILE, e) + output("Failed to open %s. Error: %s" % (LOCK_FILE, e)) + return INTERNAL_ERROR + + return ret + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/extras/specgen.scm b/extras/specgen.scm index b188d88a597..0edbb6f6280 100755 --- a/extras/specgen.scm +++ b/extras/specgen.scm @@ -1,7 +1,7 @@ #!/usr/bin/guile -s !# -;;; Copyright (C) 2007-2009 Gluster Inc. <http://www.gluster.com> +;;; Copyright (c) 2007-2011 Gluster Inc. <http://www.gluster.com> ;;; ;;; This program is free software; you can redistribute it and/or modify ;;; it under the terms of the GNU General Public License as published by diff --git a/extras/statedumpparse.rb b/extras/statedumpparse.rb new file mode 100755 index 00000000000..1aff43377db --- /dev/null +++ b/extras/statedumpparse.rb @@ -0,0 +1,208 @@ +#!/usr/bin/env ruby + +require 'time' +require 'optparse' + +unless Array.instance_methods.include? :to_h + class Array + def to_h + h = {} + each { |k,v| h[k]=v } + h + end + end +end + +# statedump.c:gf_proc_dump_mempool_info uses a five-dash record separator, +# client.c:client_fd_lk_ctx_dump uses a six-dash record separator. +ARRSEP = /^(-{5,6}=-{5,6})?$/ +HEAD = /^\[(.*)\]$/ +INPUT_FORMATS = %w[statedump json] + +format = 'json' +input_format = 'statedump' +tz = '+0000' +memstat_select,memstat_reject = //,/\Z./ +OptionParser.new do |op| + op.banner << " [<] <STATEDUMP>" + op.on("-f", "--format=F", "json/yaml/memstat(-[plain|human|json])") { |s| format = s } + op.on("--input-format=F", INPUT_FORMATS.join(?/)) { |s| input_format = s } + op.on("--timezone=T", + "time zone to apply to zoneless timestamps [default UTC]") { |s| tz = s } + op.on("--memstat-select=RX", "memstat: select memory types matching RX") { |s| + memstat_select = Regexp.new s + } + op.on("--memstat-reject=RX", "memstat: reject memory types matching RX") { |s| + memstat_reject = Regexp.new s + } +end.parse! + + +if format =~ /\Amemstat(?:-(.*))?/ + memstat_type = $1 || 'plain' + unless %w[plain human json].include? memstat_type + raise "unknown memstat type #{memstat_type.dump}" + end + format = 'memstat' +end + +repr, logsep = case format +when 'yaml' + require 'yaml' + + [proc { |e| e.to_yaml }, "\n"] +when 'json', 'memstat' + require 'json' + + [proc { |e| e.to_json }, " "] +else + raise "unkonwn format '#{format}'" +end +formatter = proc { |e| puts repr.call(e) } + +INPUT_FORMATS.include? input_format or raise "unkwown input format '#{input_format}'" + +dumpinfo = {} + +# parse a statedump entry +elem_cbk = proc { |s,&cbk| + arraylike = false + s.grep(/\S/).empty? and next + head = nil + while s.last =~ /^\s*$/ + s.pop + end + body = catch { |misc2| + s[0] =~ HEAD ? (head = $1) : (throw misc2) + body = [[]] + s[1..-1].each { |l| + if l =~ ARRSEP + arraylike = true + body << [] + next + end + body.last << l + } + + body.reject(&:empty?).map { |e| + ea = e.map { |l| + k,v = l.split("=",2) + m = /\A(0|-?[1-9]\d*)(\.\d+)?\Z/.match v + [k, m ? (m[2] ? Float(v) : Integer(v)) : v] + } + begin + ea.to_h + rescue + throw misc2 + end + } + } + + if body + cbk.call [head, arraylike ? body : (body.empty? ? {} : body[0])] + else + STDERR.puts ["WARNING: failed to parse record:", repr.call(s)].join(logsep) + end +} + +# aggregator routine +aggr = case format +when 'memstat' + meminfo = {} + # commit memory-related entries to meminfo + proc { |k,r| + case k + when /memusage/ + (meminfo["GF_MALLOC"]||={})[k] ||= r["size"] if k =~ memstat_select and k !~ memstat_reject + when "mempool" + r.each {|e| + kk = "mempool:#{e['pool-name']}" + (meminfo["mempool"]||={})[kk] ||= e["size"] if kk =~ memstat_select and kk !~ memstat_reject + } + end + } +else + # just format data, don't actually aggregate anything + proc { |pair| formatter.call pair } +end + +# processing the data +case input_format +when 'statedump' + acc = [] + $<.each { |l| + l = l.strip + if l =~ /^(DUMP-(?:START|END)-TIME):\s+(.*)/ + dumpinfo["_meta"]||={} + (dumpinfo["_meta"]["date"]||={})[$1] = Time.parse([$2, tz].join " ") + next + end + + if l =~ HEAD + elem_cbk.call(acc, &aggr) + acc = [l] + next + end + + acc << l + } + elem_cbk.call(acc, &aggr) +when 'json' + $<.each { |l| + r = JSON.load l + case r + when Array + aggr[r] + when Hash + dumpinfo.merge! r + end + } +end + +# final actions: output aggregated data +case format +when 'memstat' + ma = meminfo.values.map(&:to_a).inject(:+) + totals = meminfo.map { |coll,h| [coll, h.values.inject(:+)] }.to_h + tt = ma.transpose[1].inject(:+) + + summary_sep,showm = case memstat_type + when 'json' + ["", proc { |k,v| puts({type: k, value: v}.to_json) }] + when 'plain', 'human' + # human-friendly number representation + hr = proc { |n| + qa = %w[B kB MB GB] + q = ((1...qa.size).find {|i| n < (1 << i*10)} || qa.size) - 1 + "%.2f%s" % [n.to_f / (1 << q*10), qa[q]] + } + + templ = "%{val} %{key}" + tft = proc { |t| t } + nft = if memstat_type == 'human' + nw = [ma.transpose[1], totals.values, tt].flatten.map{|n| hr[n].size}.max + proc { |n| + hn = hr[n] + " " * (nw - hn.size) + hn + } + else + nw = tt.to_s.size + proc { |n| "%#{nw}d" % n } + end + ## Alternative template, key first: + # templ = "%{key} %{val}" + # tw = ma.transpose[0].map(&:size).max + # tft = proc { |t| t + " " * [tw - t.size, 0].max } + # nft = (memstat_type == 'human') ? hr : proc { |n| n } + ["\n", proc { |k,v| puts templ % {key: tft[k], val: nft[v]} }] + else + raise 'this should be impossible' + end + + ma.sort_by { |k,v| v }.each(&showm) + print summary_sep + totals.each { |coll,t| showm.call "Total #{coll}", t } + showm.call "TOTAL", tt +else + formatter.call dumpinfo +end diff --git a/extras/stop-all-gluster-processes.sh b/extras/stop-all-gluster-processes.sh new file mode 100755 index 00000000000..710aaf5fd3c --- /dev/null +++ b/extras/stop-all-gluster-processes.sh @@ -0,0 +1,193 @@ +#!/bin/bash +# +# Kill all the processes/services except glusterd +# +# Usage: ./extras/stop-all-gluster-processes.sh [-g] [-h] +# options: +# -g Terminate in graceful mode +# -h Show this message, then exit +# +# eg: +# 1. ./extras/stop-all-gluster-processes.sh +# 2. ./extras/stop-all-gluster-processes.sh -g +# +# By default, this script executes in force mode, i.e. all of brick, gsyncd +# and other glustershd services/processes are killed without checking for +# ongoing tasks such as geo-rep, self-heal, rebalance and etc. which may lead +# to inconsistency after the node is brought back. +# +# On specifying '-g' option this script works in graceful mode, to maintain +# data consistency the script fails with a valid exit code incase if any of +# the gluster processes are busy in doing their jobs. +# +# The author of page [1] proposes user-defined exit codes to the range 64 - 113 +# Find the better explanation behind the choice in the link +# +# The exit code returned by stop-all-gluster-processes.sh: +# 0 No errors/Success +# 64 Rebalance is in progress +# 65 Self-Heal is in progress +# 66 Tier daemon running on this node +# 127 option not found +# +# [1] http://www.tldp.org/LDP/abs/html/exitcodes.html + + +# global +errors=0 + +# find the mounts and return their pids +get_mount_pids() +{ + local opts + local pid + + for opts in $(grep -w fuse.glusterfs /proc/mounts| awk '{print $1":/"$2}'); + do + IFS=' ' read -r -a volinfo <<< $(echo "${opts}" | sed 's/:\// /g') + pid+="$(ps -Ao pid,args | grep -w "volfile-server=${volinfo[0]}" | + grep -w "volfile-id=/${volinfo[1]}" | grep -w "${volinfo[2]}" | + awk '{print $1}') " + done + echo "${pid}" +} + +# handle mount processes i.e. 'glusterfs' +kill_mounts() +{ + local signal=${1} + local pid + + for pid in $(get_mount_pids); + do + echo "sending SIG${signal} to mount process with pid: ${pid}"; + kill -${signal} ${pid}; + done +} + +# handle brick processes and node services +kill_bricks_and_services() +{ + local signal=${1} + local pidfile + local pid + + for pidfile in $(find /var/run/gluster/ -name '*.pid'); + do + local pid=$(cat ${pidfile}); + echo "sending SIG${signal} to pid: ${pid}"; + kill -${signal} ${pid}; + done +} + +# for geo-replication, only 'monitor' has pid file written, other +# processes are not having a pid file, so get it through 'ps' and +# handle these processes +kill_georep_gsync() +{ + local signal=${1} + + # FIXME: add strick/better check + local gsyncpid=$(ps -Ao pid,args | grep gluster | grep gsync | + awk '{print $1}'); + if [ -n "${gsyncpid}" ] + then + echo "sending SIG${signal} to geo-rep gsync process ${gsyncpid}"; + kill -${signal} ${gsyncpid} || errors=$((${errors} + 1)); + fi +} + +# check if all processes are ready to die +check_background_tasks() +{ + volumes=$(gluster vol list) + quit=0 + for volname in ${volumes}; + do + # tiering + if [[ $(gluster volume tier ${volname} status 2> /dev/null | + grep "localhost" | grep -c "in progress") -gt 0 ]] + then + quit=66 + break; + fi + + # rebalance + if [[ $(gluster volume rebalance ${volname} status 2> /dev/null | + grep -c "in progress") -gt 0 ]] + then + quit=64 + break; + fi + + # self heal + if [[ $(gluster volume heal ${volname} info | grep "Number of entries" | + awk '{ sum+=$4} END {print sum}') -gt 0 ]]; + then + quit=65 + break; + fi + + # geo-rep, snapshot and quota doesn't need grace checks, + # as they ensures the consistancy on force kills + done + + echo ${quit} +} + +usage() +{ + cat <<EOM +Usage: $0 [-g] [-h] + options: + -g Terminate in graceful mode + -h Show this message, then exit + +eg: + 1. $0 + 2. $0 -g +EOM +} + +main() +{ + while getopts "gh" opt; do + case $opt in + g) + # graceful mode + quit=$(check_background_tasks) + if [[ ${quit} -ne 0 ]] + then + exit ${quit}; + fi + # else safe to kill + ;; + h) + usage + exit 0; + ;; + *) + usage + exit 127; + ;; + esac + done + # remove all the options that have been parsed by getopts + shift $((OPTIND-1)) + + kill_mounts TERM + kill_georep_gsync TERM + kill_bricks_and_services TERM + + sleep 5; + echo "" + + # still not Terminated? let's pass SIGKILL + kill_mounts KILL + kill_georep_gsync KILL + kill_bricks_and_services KILL + + exit ${errors}; +} + +main "$@" diff --git a/extras/stripe-merge.c b/extras/stripe-merge.c index 3f8e4b1244d..e013a6e6e8a 100644 --- a/extras/stripe-merge.c +++ b/extras/stripe-merge.c @@ -1,48 +1,503 @@ +/* + Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +/* + * stripe-merge.c + * + * This program recovers an original file based on the striped files stored on + * the individual bricks of a striped volume. The file format and stripe + * geometry is validated through the extended attributes stored in the file. + * + * TODO: Support optional xattr recovery (i.e., user xattrs). Perhaps provide a + * command-line flag to toggle this behavior. + */ + #include <stdio.h> -#include <unistd.h> -#include <fcntl.h> #include <sys/types.h> #include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <stdlib.h> +#include <stdint.h> +#include <errno.h> +#include <string.h> +#include <sys/xattr.h> +#include <fnmatch.h> -int -main (int argc, char *argv[]) -{ - int fds[argc-1]; - char buf[argc-1][4096]; - int i; - int max_ret, ret; - - if (argc < 2) { - printf ("Usage: %s file1 file2 ... >file\n", argv[0]); - return 1; - } - - for (i=0; i<argc-1; i++) { - fds[i] = open (argv[i+1], O_RDONLY); - if (fds[i] == -1) { - perror (argv[i+1]); - return 1; - } - } - - max_ret = 0; - do { - char newbuf[4096] = {0, }; - int j; - - max_ret = 0; - for (i=0; i<argc-1; i++) { - memset (buf[i], 0, 4096); - ret = read (fds[i], buf[i], 4096); - if (ret > max_ret) - max_ret = ret; - } - for (i=0; i<max_ret;i++) - for (j=0; j<argc-1; j++) - newbuf[i] |= buf[j][i]; - write (1, newbuf, max_ret); - } while (max_ret); - - return 0; +#define ATTRNAME_STRIPE_INDEX "trusted.*.stripe-index" +#define ATTRNAME_STRIPE_COUNT "trusted.*.stripe-count" +#define ATTRNAME_STRIPE_SIZE "trusted.*.stripe-size" +#define ATTRNAME_STRIPE_COALESCE "trusted.*.stripe-coalesce" + +#define INVALID_FD -1 +#define INVALID_MODE UINT32_MAX + +struct file_stripe_info { + int stripe_count; + int stripe_size; + int coalesce; + mode_t mode; + int fd[0]; +}; + +static int +close_files(struct file_stripe_info *); + +static struct file_stripe_info * +alloc_file_stripe_info(int count) +{ + int i; + struct file_stripe_info *finfo; + + finfo = calloc(1, sizeof(struct file_stripe_info) + (sizeof(int) * count)); + if (!finfo) + return NULL; + + for (i = 0; i < count; i++) + finfo->fd[i] = INVALID_FD; + + finfo->mode = INVALID_MODE; + finfo->coalesce = INVALID_FD; + + return finfo; +} + +/* + * Search for an attribute matching the provided pattern. Return a count for + * the total number of matching entries (including 0). Allocate a buffer for + * the first matching entry found. + */ +static int +get_stripe_attr_name(const char *path, const char *pattern, char **attrname) +{ + char attrbuf[4096]; + char *ptr, *match = NULL; + int len, r, match_count = 0; + + if (!path || !pattern || !attrname) + return -1; + + len = listxattr(path, attrbuf, sizeof(attrbuf)); + if (len < 0) + return len; + + ptr = attrbuf; + while (ptr) { + r = fnmatch(pattern, ptr, 0); + if (!r) { + if (!match) + match = ptr; + match_count++; + } else if (r != FNM_NOMATCH) { + return -1; + } + + len -= strlen(ptr) + 1; + if (len > 0) + ptr += strlen(ptr) + 1; + else + ptr = NULL; + } + + if (match) + *attrname = strdup(match); + + return match_count; } +/* + * Get the integer representation of a named attribute. + */ +static int +get_stripe_attr_val(const char *path, const char *attr, int *val) +{ + char attrbuf[4096]; + int len; + + if (!path || !attr || !val) + return -1; + + len = getxattr(path, attr, attrbuf, sizeof(attrbuf)); + if (len < 0) + return len; + + *val = atoi(attrbuf); + + return 0; +} + +/* + * Get an attribute name/value (assumed to be an integer) pair based on a + * specified search pattern. A buffer is allocated for the exact attr name + * returned. Optionally, skip the pattern search if a buffer is provided + * (which should contain an attribute name). + * + * Returns the attribute count or -1 on error. The value parameter is set only + * when a single attribute is found. + */ +static int +get_attr(const char *path, const char *pattern, char **buf, int *val) +{ + int count = 1; + + if (!buf) + return -1; + + if (!*buf) { + count = get_stripe_attr_name(path, pattern, buf); + if (count > 1) { + /* pattern isn't good enough */ + fprintf(stderr, + "ERROR: duplicate attributes found " + "matching pattern: %s\n", + pattern); + free(*buf); + *buf = NULL; + return count; + } else if (count < 1) { + return count; + } + } + + if (get_stripe_attr_val(path, *buf, val) < 0) + return -1; + + return count; +} + +/* + * validate_and_open_files() + * + * Open the provided source files and validate the extended attributes. Verify + * that the geometric attributes are consistent across all of the files and + * print a warning if any files are missing. We proceed without error in the + * latter case to support partial recovery. + */ +static struct file_stripe_info * +validate_and_open_files(char *paths[], int count) +{ + int i, val, tmp; + struct stat sbuf; + char *stripe_count_attr = NULL; + char *stripe_size_attr = NULL; + char *stripe_index_attr = NULL; + char *stripe_coalesce_attr = NULL; + struct file_stripe_info *finfo = NULL; + + for (i = 0; i < count; i++) { + if (!paths[i]) + goto err; + + /* + * Check the stripe count first so we can allocate the info + * struct with the appropriate number of fds. + */ + if (get_attr(paths[i], ATTRNAME_STRIPE_COUNT, &stripe_count_attr, + &val) != 1) { + fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i], + ATTRNAME_STRIPE_COUNT); + goto err; + } + if (!finfo) { + finfo = alloc_file_stripe_info(val); + if (!finfo) + goto err; + + if (val != count) + fprintf(stderr, + "WARNING: %s: stripe-count " + "(%d) != file count (%d). Result may " + "be incomplete.\n", + paths[i], val, count); + + finfo->stripe_count = val; + } else if (val != finfo->stripe_count) { + fprintf(stderr, + "ERROR %s: invalid stripe count: %d " + "(expected %d)\n", + paths[i], val, finfo->stripe_count); + goto err; + } + + /* + * Get and validate the chunk size. + */ + if (get_attr(paths[i], ATTRNAME_STRIPE_SIZE, &stripe_size_attr, &val) != + 1) { + fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i], + ATTRNAME_STRIPE_SIZE); + goto err; + } + + if (!finfo->stripe_size) { + finfo->stripe_size = val; + } else if (val != finfo->stripe_size) { + fprintf(stderr, + "ERROR: %s: invalid stripe size: %d " + "(expected %d)\n", + paths[i], val, finfo->stripe_size); + goto err; + } + + /* + * stripe-coalesce is a backward compatible attribute. If the + * attribute does not exist, assume a value of zero for the + * traditional stripe format. + */ + tmp = get_attr(paths[i], ATTRNAME_STRIPE_COALESCE, + &stripe_coalesce_attr, &val); + if (!tmp) { + val = 0; + } else if (tmp != 1) { + fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i], + ATTRNAME_STRIPE_COALESCE); + goto err; + } + + if (finfo->coalesce == INVALID_FD) { + finfo->coalesce = val; + } else if (val != finfo->coalesce) { + fprintf(stderr, "ERROR: %s: invalid coalesce flag\n", paths[i]); + goto err; + } + + /* + * Get/validate the stripe index and open the file in the + * appropriate fd slot. + */ + if (get_attr(paths[i], ATTRNAME_STRIPE_INDEX, &stripe_index_attr, + &val) != 1) { + fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i], + ATTRNAME_STRIPE_INDEX); + goto err; + } + if (finfo->fd[val] != INVALID_FD) { + fprintf(stderr, + "ERROR: %s: duplicate stripe index: " + "%d\n", + paths[i], val); + goto err; + } + + finfo->fd[val] = open(paths[i], O_RDONLY); + if (finfo->fd[val] < 0) + goto err; + + /* + * Get the creation mode for the file. + */ + if (fstat(finfo->fd[val], &sbuf) < 0) + goto err; + if (finfo->mode == INVALID_MODE) { + finfo->mode = sbuf.st_mode; + } else if (sbuf.st_mode != finfo->mode) { + fprintf(stderr, "ERROR: %s: invalid mode\n", paths[i]); + goto err; + } + } + + free(stripe_count_attr); + free(stripe_size_attr); + free(stripe_index_attr); + free(stripe_coalesce_attr); + + return finfo; +err: + + free(stripe_count_attr); + free(stripe_size_attr); + free(stripe_index_attr); + free(stripe_coalesce_attr); + + if (finfo) { + close_files(finfo); + free(finfo); + } + + return NULL; +} + +static int +close_files(struct file_stripe_info *finfo) +{ + int i, ret; + + if (!finfo) + return -1; + + for (i = 0; i < finfo->stripe_count; i++) { + if (finfo->fd[i] == INVALID_FD) + continue; + + ret = close(finfo->fd[i]); + if (ret < 0) + return ret; + } + + return ret; +} + +/* + * Generate the original file using files striped in the coalesced format. + * Data in the striped files is stored at a coalesced offset based on the + * stripe number. + * + * Walk through the finfo fds (which are already ordered) and and iteratively + * copy stripe_size bytes from the source files to the target file. If a source + * file is missing, seek past the associated stripe_size bytes in the target + * file. + */ +static int +generate_file_coalesce(int target, struct file_stripe_info *finfo) +{ + char *buf; + int ret = 0; + int r, w, i; + + buf = malloc(finfo->stripe_size); + if (!buf) + return -1; + + i = 0; + while (1) { + if (finfo->fd[i] == INVALID_FD) { + if (lseek(target, finfo->stripe_size, SEEK_CUR) < 0) + break; + + i = (i + 1) % finfo->stripe_count; + continue; + } + + r = read(finfo->fd[i], buf, finfo->stripe_size); + if (r < 0) { + ret = r; + break; + } + if (!r) + break; + + w = write(target, buf, r); + if (w < 0) { + ret = w; + break; + } + + i = (i + 1) % finfo->stripe_count; + } + + free(buf); + return ret; +} + +/* + * Generate the original file using files striped with the traditional stripe + * format. Data in the striped files is stored at the equivalent offset from + * the source file. + */ +static int +generate_file_traditional(int target, struct file_stripe_info *finfo) +{ + int i, j, max_ret, ret; + char buf[finfo->stripe_count][4096]; + + do { + char newbuf[4096] = { + 0, + }; + + max_ret = 0; + for (i = 0; i < finfo->stripe_count; i++) { + memset(buf[i], 0, 4096); + ret = read(finfo->fd[i], buf[i], 4096); + if (ret > max_ret) + max_ret = ret; + } + for (i = 0; i < max_ret; i++) + for (j = 0; j < finfo->stripe_count; j++) + newbuf[i] |= buf[j][i]; + write(target, newbuf, max_ret); + } while (max_ret); + + return 0; +} + +static int +generate_file(int target, struct file_stripe_info *finfo) +{ + if (finfo->coalesce) + return generate_file_coalesce(target, finfo); + + return generate_file_traditional(target, finfo); +} + +static void +usage(char *name) +{ + fprintf(stderr, + "Usage: %s [-o <outputfile>] <inputfile1> " + "<inputfile2> ...\n", + name); +} + +int +main(int argc, char *argv[]) +{ + int file_count, opt; + char *opath = NULL; + int targetfd; + struct file_stripe_info *finfo; + + while ((opt = getopt(argc, argv, "o:")) != -1) { + switch (opt) { + case 'o': + opath = optarg; + break; + default: + usage(argv[0]); + return -1; + } + } + + file_count = argc - optind; + + if (!opath || !file_count) { + usage(argv[0]); + return -1; + } + + finfo = validate_and_open_files(&argv[optind], file_count); + if (!finfo) + goto err; + + targetfd = open(opath, O_RDWR | O_CREAT, finfo->mode); + if (targetfd < 0) + goto err; + + if (generate_file(targetfd, finfo) < 0) + goto err; + + if (fsync(targetfd) < 0) + fprintf(stderr, "ERROR: %s\n", strerror(errno)); + if (close(targetfd) < 0) + fprintf(stderr, "ERROR: %s\n", strerror(errno)); + + close_files(finfo); + free(finfo); + + return 0; + +err: + if (finfo) { + close_files(finfo); + free(finfo); + } + + return -1; +} diff --git a/extras/systemd/Makefile.am b/extras/systemd/Makefile.am new file mode 100644 index 00000000000..61446a9b84a --- /dev/null +++ b/extras/systemd/Makefile.am @@ -0,0 +1,17 @@ +CLEANFILES = glusterd.service glustereventsd.service glusterfssharedstorage.service gluster-ta-volume.service +EXTRA_DIST = glusterd.service.in glustereventsd.service.in glusterfssharedstorage.service.in gluster-ta-volume.service.in + +if USE_SYSTEMD +systemd_DATA = gluster-ta-volume.service +endif + +if WITH_SERVER +if USE_SYSTEMD +# systemddir is already defined through configure.ac +systemd_DATA += glusterd.service glusterfssharedstorage.service + +if BUILD_EVENTS +systemd_DATA += glustereventsd.service +endif +endif +endif diff --git a/extras/systemd/gluster-ta-volume.service.in b/extras/systemd/gluster-ta-volume.service.in new file mode 100644 index 00000000000..2802bca05bf --- /dev/null +++ b/extras/systemd/gluster-ta-volume.service.in @@ -0,0 +1,13 @@ +[Unit] +Description=GlusterFS, Thin-arbiter process to maintain quorum for replica volume +After=network.target + +[Service] +Environment="LOG_LEVEL=WARNING" +ExecStart=@prefix@/sbin/glusterfsd -N --volfile-id ta -f @GLUSTERD_WORKDIR@/thin-arbiter/thin-arbiter.vol --brick-port 24007 --xlator-option ta-server.transport.socket.listen-port=24007 -LWARNING +Restart=always +KillMode=process +SuccessExitStatus=15 + +[Install] +WantedBy=multi-user.target diff --git a/extras/systemd/glusterd.service.in b/extras/systemd/glusterd.service.in new file mode 100644 index 00000000000..abb0d82911f --- /dev/null +++ b/extras/systemd/glusterd.service.in @@ -0,0 +1,26 @@ +[Unit] +Description=GlusterFS, a clustered file-system server +Documentation=man:glusterd(8) +StartLimitBurst=6 +StartLimitIntervalSec=3600 +Requires=@RPCBIND_SERVICE@ +After=network.target @RPCBIND_SERVICE@ +Before=network-online.target + +[Service] +Type=forking +PIDFile=@localstatedir@/run/glusterd.pid +LimitNOFILE=65536 +Environment="LOG_LEVEL=INFO" +EnvironmentFile=-@SYSCONF_DIR@/sysconfig/glusterd +ExecStart=@prefix@/sbin/glusterd -p @localstatedir@/run/glusterd.pid --log-level $LOG_LEVEL $GLUSTERD_OPTIONS +KillMode=process +TimeoutSec=300 +SuccessExitStatus=15 +Restart=on-abnormal +RestartSec=60 +StartLimitBurst=6 +StartLimitInterval=3600 + +[Install] +WantedBy=multi-user.target diff --git a/extras/systemd/glustereventsd.service.in b/extras/systemd/glustereventsd.service.in new file mode 100644 index 00000000000..f80b78199f6 --- /dev/null +++ b/extras/systemd/glustereventsd.service.in @@ -0,0 +1,16 @@ +[Unit] +Description=Gluster Events Notifier +After=network.target +Documentation=man:glustereventsd(8) + + +[Service] +Environment=PYTHONPATH=@BUILD_PYTHON_SITE_PACKAGES_EXPANDED@:$PYTHONPATH +Type=simple +ExecStart=@SBIN_DIR@/glustereventsd --pid-file @localstatedir@/run/glustereventsd.pid +ExecReload=/bin/kill -SIGUSR2 $MAINPID +KillMode=control-group +PIDFile=@localstatedir@/run/glustereventsd.pid + +[Install] +WantedBy=multi-user.target diff --git a/extras/systemd/glusterfssharedstorage.service.in b/extras/systemd/glusterfssharedstorage.service.in new file mode 100644 index 00000000000..723ff49afb7 --- /dev/null +++ b/extras/systemd/glusterfssharedstorage.service.in @@ -0,0 +1,13 @@ +[Unit] +Description=Mount glusterfs sharedstorage +Requires=glusterd.service remote-fs-pre.target local-fs.target + +[Service] +Type=forking +ExecStart=@GLUSTERFS_LIBEXECDIR@/mount-shared-storage.sh +Restart=on-failure +RestartSec=3 +RestartForceExitStatus=1 + +[Install] +WantedBy=multi-user.target diff --git a/extras/test/bug-920583.t b/extras/test/bug-920583.t new file mode 100755 index 00000000000..eedbb800ac0 --- /dev/null +++ b/extras/test/bug-920583.t @@ -0,0 +1,50 @@ +#!/bin/bash + +##Copy this file to tests/bugs before running run.sh (cp extras/test/bug-920583.t tests/bugs/) + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc + +cleanup; +logdir=`gluster --print-logdir` + +## Start and create a volume +TEST glusterd; +TEST pidof glusterd; + +TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8}; + +## Verify volume is is created +EXPECT "$V0" volinfo_field $V0 'Volume Name'; +EXPECT 'Created' volinfo_field $V0 'Status'; + +## Start volume and verify +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; + +function log-file-name() +{ + logfilename=$M0".log" + echo ${logfilename:1} | tr / - +} + +log_file=$logdir"/"`log-file-name` + +lookup_unhashed_count=`grep "adding option 'lookup-unhashed'" $log_file | wc -l` +no_child_down_count=`grep "adding option 'assert-no-child-down'" $log_file | wc -l` +mount -t glusterfs $H0:/$V0 $M0 -o "xlator-option=*dht.assert-no-child-down=yes,xlator-option=*dht.lookup-unhashed=yes" +touch $M0/file1; + +new_lookup_unhashed_count=`grep "adding option 'lookup-unhashed'" $log_file | wc -l` +new_no_child_down_count=`grep "adding option 'assert-no-child-down'" $log_file | wc -l` +EXPECT "1" expr $new_lookup_unhashed_count - $lookup_unhashed_count +EXPECT "1" expr $new_no_child_down_count - $no_child_down_count + +## Finish up +TEST $CLI volume stop $V0; +EXPECT 'Stopped' volinfo_field $V0 'Status'; + +TEST $CLI volume delete $V0; +TEST ! $CLI volume info $V0; + +cleanup; diff --git a/extras/test/gluster_commands.sh b/extras/test/gluster_commands.sh new file mode 100755 index 00000000000..cb2a55fd5b5 --- /dev/null +++ b/extras/test/gluster_commands.sh @@ -0,0 +1,265 @@ +#!/bin/bash + + +# Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + + +# This script tests the basics gluster cli commands. + +echo "Starting glusterd" +glusterd +if [ $? -ne 0 ]; then + echo "Could not start glusterd.Exiting" + exit; +else + echo "glusterd started" +fi + +if [ ! -d "/exports" ]; then + mkdir /exports; + mkdir /exports/exp{1..10}; +else + mkdir /exports/exp{1..10}; +fi + +if [ ! -d "/mnt/client" ]; then + mkdir /mnt/client -p; +fi + + +set -e #exit at the first error that happens + +# create distribute volume and try start, mount, add-brick, replace-brick, remove-brick, stop, unmount, delete + +echo "Creating distribute volume........" +gluster volume create vol `hostname`:/exports/exp1 +gluster volume info + +echo "Starting distribute volume........" +gluster volume start vol +gluster volume info +sleep 1 +mount -t glusterfs `hostname`:vol /mnt/client +sleep 1 +df -h + +echo "adding-brick......." +gluster volume add-brick vol `hostname`:/exports/exp2 +gluster volume info +sleep 1 +umount /mnt/client +mount -t glusterfs `hostname`:vol /mnt/client +df -h +sleep 1 + +echo "replacing brick......" +gluster volume replace-brick vol `hostname`:/exports/exp1 `hostname`:/exports/exp3 start +#sleep for 5 seconds +sleep 5 +gluster volume replace-brick vol `hostname`:/exports/exp1 `hostname`:/exports/exp3 status +gluster volume replace-brick vol `hostname`:/exports/exp1 `hostname`:/exports/exp3 pause +gluster volume replace-brick vol `hostname`:/exports/exp1 `hostname`:/exports/exp3 status +gluster volume replace-brick vol `hostname`:/exports/exp1 `hostname`:/exports/exp3 start +#sleep for 5 seconds +sleep 5 +gluster volume replace-brick vol `hostname`:/exports/exp1 `hostname`:/exports/exp3 status +gluster volume replace-brick vol `hostname`:/exports/exp1 `hostname`:/exports/exp3 commit + + +echo "replcing brick for abort operation" +gluster volume replace-brick vol `hostname`:/exports/exp3 `hostname`:/exports/exp1 start +#sleep for 5 seconds +sleep 5 +gluster volume replace-brick vol `hostname`:/exports/exp3 `hostname`:/exports/exp1 status +gluster volume replace-brick vol `hostname`:/exports/exp3 `hostname`:/exports/exp1 pause +gluster volume replace-brick vol `hostname`:/exports/exp3 `hostname`:/exports/exp1 status +gluster volume replace-brick vol `hostname`:/exports/exp3 `hostname`:/exports/exp1 start +#sleep for 5 seconds +sleep 5 +gluster volume replace-brick vol `hostname`:/exports/exp3 `hostname`:/exports/exp1 status +gluster volume replace-brick vol `hostname`:/exports/exp3 `hostname`:/exports/exp1 abort + + +gluster volume info +sleep 1 +df -h +sleep 1 + +echo "removing brick......." +gluster --mode=script volume remove-brick vol `hostname`:/exports/exp2 +gluster volume info +sleep 1 +df -h +sleep 1 + +echo "stopping distribute volume......" +gluster --mode=script volume stop vol +gluster volume info +sleep 1 +umount /mnt/client +df -h + +echo "deleting distribute volume......" +gluster --mode=script volume delete vol +gluster volume info +sleep 1 + +# create replicate volume and try start, mount, add-brick, replace-brick, remove-brick, stop, unmount, delete +echo "creating replicate volume......" +gluster volume create mirror replica 2 `hostname`:/exports/exp1 `hostname`:/exports/exp2 +gluster volume info +sleep 1 + +echo "starting replicate volume......" +gluster volume start mirror +gluster volume info +sleep 1 +mount -t glusterfs `hostname`:mirror /mnt/client +sleep 1 +df -h +sleep 1 + +echo "adding-brick......." +gluster volume add-brick mirror `hostname`:/exports/exp3 `hostname`:/exports/exp4 +gluster volume info +sleep 1 +df -h +sleep 1 + +echo "replacing-brick....." +gluster volume replace-brick mirror `hostname`:/exports/exp1 `hostname`:/exports/exp5 start +#sleep for 5 seconds +sleep 5 +gluster volume replace-brick mirror `hostname`:/exports/exp1 `hostname`:/exports/exp5 status +gluster volume replace-brick mirror `hostname`:/exports/exp1 `hostname`:/exports/exp5 pause +gluster volume replace-brick mirror `hostname`:/exports/exp1 `hostname`:/exports/exp5 status +gluster volume replace-brick mirror `hostname`:/exports/exp1 `hostname`:/exports/exp5 start +#sleep for 5 seconds +sleep 5 +gluster volume replace-brick mirror `hostname`:/exports/exp1 `hostname`:/exports/exp5 status +gluster volume replace-brick mirror `hostname`:/exports/exp1 `hostname`:/exports/exp5 commit +gluster volume info +sleep 1 +df -h +sleep 1 + +echo "replacing brick for abort operation" +gluster volume replace-brick mirror `hostname`:/exports/exp5 `hostname`:/exports/exp1 start +#sleep for 5 seconds +sleep 5 +gluster volume replace-brick mirror `hostname`:/exports/exp5 `hostname`:/exports/exp1 status +gluster volume replace-brick mirror `hostname`:/exports/exp5 `hostname`:/exports/exp1 pause +gluster volume replace-brick mirror `hostname`:/exports/exp5 `hostname`:/exports/exp1 status +gluster volume replace-brick mirror `hostname`:/exports/exp5 `hostname`:/exports/exp1 start +#sleep for 5 seconds +sleep 5 +gluster volume replace-brick mirror `hostname`:/exports/exp5 `hostname`:/exports/exp1 status +gluster volume replace-brick mirror `hostname`:/exports/exp5 `hostname`:/exports/exp1 abort + +gluster volume info +sleep 1 +df -h +sleep 1 + +echo "removeing-brick....." +gluster --mode=script volume remove-brick mirror `hostname`:/exports/exp3 `hostname`:/exports/exp4 +gluster volume info +sleep 1 +df -h +sleep 1 + +echo "stopping replicate volume....." +gluster --mode=script volume stop mirror +gluster volume info +sleep 1 +umount /mnt/client +df -h + +echo "deleting replicate volume....." +gluster --mode=script volume delete mirror +gluster volume info +sleep 1 + +# create stripe volume and try start, mount, add-brick, replace-brick, remove-brick, stop, unmount, delete + +echo "creating stripe volume....." +gluster volume create str stripe 2 `hostname`:/exports/exp1 `hostname`:/exports/exp2 +gluster volume info +sleep 1 + +echo "starting stripe volume....." +gluster volume start str +gluster volume info +sleep 1 +mount -t glusterfs `hostname`:str /mnt/client +sleep 1 +df -h +sleep 1 + +echo "adding brick...." +gluster volume add-brick str `hostname`:/exports/exp3 `hostname`:/exports/exp4 +gluster volume info +sleep 1 +df -h +sleep 1 + +echo "replacing brick....." +gluster volume replace-brick str `hostname`:/exports/exp1 `hostname`:/exports/exp5 start +#sleep for 5 seconds +sleep 5 +gluster volume replace-brick str `hostname`:/exports/exp1 `hostname`:/exports/exp5 status +gluster volume replace-brick str `hostname`:/exports/exp1 `hostname`:/exports/exp5 pause +gluster volume replace-brick str `hostname`:/exports/exp1 `hostname`:/exports/exp5 status +gluster volume replace-brick str `hostname`:/exports/exp1 `hostname`:/exports/exp5 start +#sleep for 5 seconds +sleep 5 +gluster volume replace-brick str `hostname`:/exports/exp1 `hostname`:/exports/exp5 status +gluster volume replace-brick str `hostname`:/exports/exp1 `hostname`:/exports/exp5 commit + +gluster volume info +sleep 1 +df -h +sleep 1 + +echo "replacing brick for abort operation" +gluster volume replace-brick str `hostname`:/exports/exp5 `hostname`:/exports/exp1 start +#sleep for 5 seconds +sleep 5 +gluster volume replace-brick str `hostname`:/exports/exp5 `hostname`:/exports/exp1 status +gluster volume replace-brick str `hostname`:/exports/exp5 `hostname`:/exports/exp1 pause +gluster volume replace-brick str `hostname`:/exports/exp5 `hostname`:/exports/exp1 status +gluster volume replace-brick str `hostname`:/exports/exp5 `hostname`:/exports/exp1 start +#sleep for 5 seconds +sleep 5 +gluster volume replace-brick str `hostname`:/exports/exp5 `hostname`:/exports/exp1 status +gluster volume replace-brick str `hostname`:/exports/exp5 `hostname`:/exports/exp1 abort + +gluster volume info +sleep 1 +df -h +sleep 1 + +echo "removing-brick....." +gluster --mode=script volume remove-brick str `hostname`:/exports/exp3 `hostname`:/exports/exp4 +gluster volume info +sleep 1 +df -h +sleep 1 + +echo "stopping stripe volume....." +gluster --mode=script volume stop str +gluster volume info +sleep 1 +umount /mnt/client +df -h + +echo "deleting stripe volume....." +gluster --mode=script volume delete str +gluster volume info + diff --git a/extras/test/ld-preload-test/README b/extras/test/ld-preload-test/README index 725b94023c8..df80003844b 100644 --- a/extras/test/ld-preload-test/README +++ b/extras/test/ld-preload-test/README @@ -32,9 +32,9 @@ Instructions $ make (We've tested the tool on Debian and CentOS. If there are build errors or -warnings, please do report them to glusterfs-devel@nongnu.org.) +warnings, please do report them to gluster-devel@gluster.org.) 2. Run the test. $ ./test-preload.sh > preload.log -3. Mail the log to glusterfs-devel@nongnu.org. +3. Mail the log to gluster-devel@gluster.org. diff --git a/extras/test/ld-preload-test/ld-preload-lib.c b/extras/test/ld-preload-test/ld-preload-lib.c index 18ee3b993da..d120c053a69 100644 --- a/extras/test/ld-preload-test/ld-preload-lib.c +++ b/extras/test/ld-preload-test/ld-preload-lib.c @@ -1,23 +1,13 @@ /* - Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com> + Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com> This file is part of GlusterFS. - GlusterFS is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - GlusterFS is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see - <http://www.gnu.org/licenses/>. + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. */ - /* LD PRELOAD'able library * A very simple library that intercepts booster supported system calls * and prints a log message to stdout. @@ -44,594 +34,582 @@ #include <fcntl.h> #include <sys/stat.h> #include <dirent.h> -#include <attr/xattr.h> +#include <sys/xattr.h> #include <sys/sendfile.h> /* Err number that is assigned to errno so that test application can * verify that the function was intercepted correctly. */ -#define PRELOAD_ERRNO_VERF 6449 -#define set_errno() (errno = PRELOAD_ERRNO_VERF) +#define PRELOAD_ERRNO_VERF 6449 +#define set_errno() (errno = PRELOAD_ERRNO_VERF) -inline void -intercept (char *call, int tabs) +void +intercept(char *call, int tabs) { - while (tabs > 0) { - fprintf (stdout, "\t"); - --tabs; - } + while (tabs > 0) { + fprintf(stdout, "\t"); + --tabs; + } - fprintf (stdout, "Intercepted by %s", call); + fprintf(stdout, "Intercepted by %s", call); } int -creat64 (const char *pathname, mode_t mode) +creat64(const char *pathname, mode_t mode) { - intercept ("creat64", 2); - set_errno (); - return -1; + intercept("creat64", 2); + set_errno(); + return -1; } int -creat (const char *pathname, mode_t mode) +creat(const char *pathname, mode_t mode) { - intercept ("creat", 2); - set_errno (); - return -1; + intercept("creat", 2); + set_errno(); + return -1; } - int -close (int fd) +close(int fd) { - intercept ("close", 2); - set_errno (); - return -1; + intercept("close", 2); + set_errno(); + return -1; } int -open64 (const char *pathname, int flags, ...) +open64(const char *pathname, int flags, ...) { - intercept ("open64", 2); - set_errno (); - return -1; + intercept("open64", 2); + set_errno(); + return -1; } - int -open (const char *pathname, int flags, ...) +open(const char *pathname, int flags, ...) { - intercept ("open", 2); - set_errno (); - return -1; + intercept("open", 2); + set_errno(); + return -1; } ssize_t -read (int fd, void *buf, size_t count) +read(int fd, void *buf, size_t count) { - intercept ("read", 2); - set_errno (); - return -1; + intercept("read", 2); + set_errno(); + return -1; } ssize_t -readv (int fd, const struct iovec *vector, int count) +readv(int fd, const struct iovec *vector, int count) { - intercept ("readv", 2); - set_errno (); - return -1; + intercept("readv", 2); + set_errno(); + return -1; } ssize_t -pread (int fd, void *buf, size_t count, unsigned long offset) +pread(int fd, void *buf, size_t count, unsigned long offset) { - intercept ("pread", 2); - set_errno (); - return -1; + intercept("pread", 2); + set_errno(); + return -1; } - ssize_t -pread64 (int fd, void *buf, size_t count, uint64_t offset) +pread64(int fd, void *buf, size_t count, uint64_t offset) { - intercept ("pread64", 2); - set_errno (); - return -1; + intercept("pread64", 2); + set_errno(); + return -1; } ssize_t -write (int fd, const void *buf, size_t count) +write(int fd, const void *buf, size_t count) { - intercept ("write", 2); - set_errno (); - return -1; + intercept("write", 2); + set_errno(); + return -1; } ssize_t -writev (int fd, const struct iovec *vector, int count) +writev(int fd, const struct iovec *vector, int count) { - intercept ("writev", 2); - set_errno (); - return -1; + intercept("writev", 2); + set_errno(); + return -1; } ssize_t -pwrite (int fd, const void *buf, size_t count, unsigned long offset) +pwrite(int fd, const void *buf, size_t count, unsigned long offset) { - intercept ("pwrite", 2); - set_errno (); - return -1; + intercept("pwrite", 2); + set_errno(); + return -1; } ssize_t -pwrite64 (int fd, const void *buf, size_t count, uint64_t offset) +pwrite64(int fd, const void *buf, size_t count, uint64_t offset) { - intercept ("pwrite64", 2); - set_errno (); - return -1; + intercept("pwrite64", 2); + set_errno(); + return -1; } - off_t -lseek (int fildes, unsigned long offset, int whence) +lseek(int fildes, unsigned long offset, int whence) { - intercept ("lseek", 2); - set_errno (); - return -1; + intercept("lseek", 2); + set_errno(); + return -1; } off_t -lseek64 (int fildes, uint64_t offset, int whence) +lseek64(int fildes, uint64_t offset, int whence) { - intercept ("lseek64", 2); - set_errno (); - return -1; + intercept("lseek64", 2); + set_errno(); + return -1; } - int -dup (int fd) +dup(int fd) { - intercept ("dup", 2); - set_errno (); - return -1; + intercept("dup", 2); + set_errno(); + return -1; } int -dup2 (int oldfd, int newfd) +dup2(int oldfd, int newfd) { - intercept ("dup2", 2); - set_errno (); - return -1; + intercept("dup2", 2); + set_errno(); + return -1; } int -mkdir (const char *pathname, mode_t mode) +mkdir(const char *pathname, mode_t mode) { - intercept ("mkdir", 2); - set_errno (); - return -1; + intercept("mkdir", 2); + set_errno(); + return -1; } int -rmdir (const char *pathname) +rmdir(const char *pathname) { - intercept ("rmdir", 2); - set_errno (); - return -1; + intercept("rmdir", 2); + set_errno(); + return -1; } int -chmod (const char *pathname, mode_t mode) +chmod(const char *pathname, mode_t mode) { - intercept ("chmod", 2); - set_errno (); - return -1; + intercept("chmod", 2); + set_errno(); + return -1; } int -chown (const char *pathname, uid_t owner, gid_t group) +chown(const char *pathname, uid_t owner, gid_t group) { - intercept ("chown", 2); - set_errno (); - return -1; + intercept("chown", 2); + set_errno(); + return -1; } int -fchmod (int fd, mode_t mode) +fchmod(int fd, mode_t mode) { - intercept ("fchmod", 2); - set_errno (); - return -1; + intercept("fchmod", 2); + set_errno(); + return -1; } int -fchown (int fd, uid_t uid, gid_t gid) +fchown(int fd, uid_t uid, gid_t gid) { - intercept ("fchown", 2); - set_errno (); - return -1; + intercept("fchown", 2); + set_errno(); + return -1; } -int fsync (int fd) +int +fsync(int fd) { - intercept ("fsync", 2); - set_errno (); - return -1; + intercept("fsync", 2); + set_errno(); + return -1; } - int -ftruncate (int fd, off_t length) +ftruncate(int fd, off_t length) { - intercept ("ftruncate", 1); - set_errno (); - return -1; + intercept("ftruncate", 1); + set_errno(); + return -1; } - int -ftruncate64 (int fd, off_t length) +ftruncate64(int fd, off_t length) { - intercept ("ftruncate64", 1); - set_errno (); - return -1; + intercept("ftruncate64", 1); + set_errno(); + return -1; } int -link (const char *oldpath, const char *newname) +link(const char *oldpath, const char *newname) { - intercept ("link", 2); - set_errno (); - return -1; + intercept("link", 2); + set_errno(); + return -1; } int -rename (const char *oldpath, const char *newpath) +rename(const char *oldpath, const char *newpath) { - intercept ("rename", 2); - set_errno (); - return -1; + intercept("rename", 2); + set_errno(); + return -1; } int -utimes (const char *path, const struct timeval times[2]) +utimes(const char *path, const struct timeval times[2]) { - intercept ("utimes", 2); - set_errno (); - return -1; + intercept("utimes", 2); + set_errno(); + return -1; } int -utime (const char *path, const struct utimbuf *buf) +futimes(int fd, const struct timeval times[2]) { - intercept ("utime", 2); - set_errno (); - return -1; + intercept("futimes", 2); + set_errno(); + return -1; } - int -mknod (const char *path, mode_t mode, dev_t dev) +utime(const char *path, const struct utimbuf *buf) { - intercept ("mknod", 2); - set_errno (); - return -1; + intercept("utime", 2); + set_errno(); + return -1; } int -__xmknod (int ver, const char *path, mode_t mode, dev_t *dev) +mknod(const char *path, mode_t mode, dev_t dev) { - intercept ("__xmknod", 2); - set_errno (); - return -1; + intercept("mknod", 2); + set_errno(); + return -1; } int -mkfifo (const char *path, mode_t mode) +__xmknod(int ver, const char *path, mode_t mode, dev_t *dev) { - intercept ("mkfifo", 2); - set_errno (); - return -1; + intercept("__xmknod", 2); + set_errno(); + return -1; } int -unlink (const char *path) +mkfifo(const char *path, mode_t mode) { - intercept ("unlink", 2); - set_errno (); - return -1; + intercept("mkfifo", 2); + set_errno(); + return -1; } - int -symlink (const char *oldpath, const char *newpath) +unlink(const char *path) { - intercept ("symlink", 2); - set_errno (); - return -1; + intercept("unlink", 2); + set_errno(); + return -1; } int -readlink (const char *path, char *buf, size_t bufsize) +symlink(const char *oldpath, const char *newpath) { - intercept ("readlink", 1); - set_errno (); - return -1; + intercept("symlink", 2); + set_errno(); + return -1; } +int +readlink(const char *path, char *buf, size_t bufsize) +{ + intercept("readlink", 1); + set_errno(); + return -1; +} char * -realpath (const char *path, char *resolved) +realpath(const char *path, char *resolved) { - intercept ("realpath", 1); - set_errno (); - return NULL; + intercept("realpath", 1); + set_errno(); + return NULL; } - DIR * -opendir (const char *path) +opendir(const char *path) { - intercept ("opendir", 2); - set_errno (); - return NULL; + intercept("opendir", 2); + set_errno(); + return NULL; } - struct dirent * -readdir (DIR *dir) +readdir(DIR *dir) { - intercept ("readdir\t", 2); - set_errno (); - return NULL; + intercept("readdir\t", 2); + set_errno(); + return NULL; } struct dirent * -readdir64 (DIR *dir) +readdir64(DIR *dir) { - intercept ("readdir64", 2); - set_errno (); - return NULL; + intercept("readdir64", 2); + set_errno(); + return NULL; } - int -readdir_r (DIR *dir, struct dirent *entry, struct dirent **result) +readdir_r(DIR *dir, struct dirent *entry, struct dirent **result) { - intercept ("readdir_r", 1); - set_errno (); - return -1; + intercept("readdir_r", 1); + set_errno(); + return -1; } int -readdir64_r (DIR *dir, struct dirent *entry, struct dirent **result) +readdir64_r(DIR *dir, struct dirent *entry, struct dirent **result) { - intercept ("readdir64_r", 1); - set_errno (); - return -1; + intercept("readdir64_r", 1); + set_errno(); + return -1; } - int -closedir (DIR *dh) +closedir(DIR *dh) { - intercept ("closedir", 1); - set_errno (); - return -1; + intercept("closedir", 1); + set_errno(); + return -1; } int -__xstat (int ver, const char *path, struct stat *buf) +__xstat(int ver, const char *path, struct stat *buf) { - intercept ("__xstat\t", 2); - set_errno (); - return -1; + intercept("__xstat\t", 2); + set_errno(); + return -1; } - int -__xstat64 (int ver, const char *path, struct stat *buf) +__xstat64(int ver, const char *path, struct stat *buf) { - intercept ("__xstat64", 2); - set_errno (); - return -1; + intercept("__xstat64", 2); + set_errno(); + return -1; } int -stat (const char *path, struct stat *buf) +stat(const char *path, struct stat *buf) { - intercept ("stat", 2); - set_errno (); - return -1; + intercept("stat", 2); + set_errno(); + return -1; } int -stat64 (const char *path, struct stat *buf) +stat64(const char *path, struct stat *buf) { - intercept ("stat64", 2); - set_errno (); - return -1; + intercept("stat64", 2); + set_errno(); + return -1; } int -__fxstat (int ver, int fd, struct stat *buf) +__fxstat(int ver, int fd, struct stat *buf) { - intercept ("__fxstat\t", 2); - set_errno (); - return -1; + intercept("__fxstat\t", 2); + set_errno(); + return -1; } - int -__fxstat64 (int ver, int fd, struct stat *buf) +__fxstat64(int ver, int fd, struct stat *buf) { - intercept ("__fxstat64", 2); - set_errno (); - return -1; + intercept("__fxstat64", 2); + set_errno(); + return -1; } int -fstat (int fd, struct stat *buf) +fstat(int fd, struct stat *buf) { - intercept ("fstat", 2); - set_errno (); - return -1; + intercept("fstat", 2); + set_errno(); + return -1; } int -fstat64 (int fd , struct stat *buf) +fstat64(int fd, struct stat *buf) { - intercept ("fstat64", 2); - set_errno (); - return -1; + intercept("fstat64", 2); + set_errno(); + return -1; } int -__lxstat (int ver, const char *path, struct stat *buf) +__lxstat(int ver, const char *path, struct stat *buf) { - intercept ("__lxstat\t", 2); - set_errno (); - return -1; + intercept("__lxstat\t", 2); + set_errno(); + return -1; } int -__lxstat64 (int ver, const char *path, struct stat *buf) +__lxstat64(int ver, const char *path, struct stat *buf) { - intercept ("__lxstat64", 2); - set_errno (); - return -1; + intercept("__lxstat64", 2); + set_errno(); + return -1; } int -lstat (const char *path, struct stat *buf) +lstat(const char *path, struct stat *buf) { - intercept ("lstat", 2); - set_errno (); - return -1; + intercept("lstat", 2); + set_errno(); + return -1; } int -lstat64 (const char *path, struct stat *buf) +lstat64(const char *path, struct stat *buf) { - intercept ("lstat64", 2); - set_errno (); - return -1; + intercept("lstat64", 2); + set_errno(); + return -1; } int -statfs (const char *path, struct statfs *buf) +statfs(const char *path, struct statfs *buf) { - intercept ("statfs", 2); - set_errno (); - return -1; + intercept("statfs", 2); + set_errno(); + return -1; } - int -statfs64 (const char *path, struct statfs *buf) +statfs64(const char *path, struct statfs *buf) { - intercept ("statfs64", 2); - set_errno (); - return -1; + intercept("statfs64", 2); + set_errno(); + return -1; } int -statvfs (const char *path, struct statvfs *buf) +statvfs(const char *path, struct statvfs *buf) { - intercept ("statvfs\t", 2); - set_errno (); - return -1; + intercept("statvfs\t", 2); + set_errno(); + return -1; } - int -statvfs64 (const char *path, struct statvfs *buf) +statvfs64(const char *path, struct statvfs *buf) { - intercept ("statvfs64", 2); - set_errno (); - return -1; + intercept("statvfs64", 2); + set_errno(); + return -1; } ssize_t -getxattr (const char *path, const char *name, void *value, size_t size) +getxattr(const char *path, const char *name, void *value, size_t size) { - intercept ("getxattr", 1); - set_errno (); - return -1; + intercept("getxattr", 1); + set_errno(); + return -1; } ssize_t -lgetxattr (const char *path, const char *name, void *value, size_t size) +lgetxattr(const char *path, const char *name, void *value, size_t size) { - intercept ("lgetxattr", 1); - set_errno (); - return -1; + intercept("lgetxattr", 1); + set_errno(); + return -1; } - int -remove (const char* path) +remove(const char *path) { - intercept ("remove", 2); - set_errno (); - return -1; + intercept("remove", 2); + set_errno(); + return -1; } int -lchown (const char *path, uid_t owner, gid_t group) +lchown(const char *path, uid_t owner, gid_t group) { - intercept ("lchown", 2); - set_errno (); - return -1; + intercept("lchown", 2); + set_errno(); + return -1; } void -rewinddir (DIR *dirp) +rewinddir(DIR *dirp) { - intercept ("rewinddir", 1); - set_errno (); - return; + intercept("rewinddir", 1); + set_errno(); + return; } void -seekdir (DIR *dirp, off_t offset) +seekdir(DIR *dirp, off_t offset) { - intercept ("seekdir", 2); - set_errno (); - return; + intercept("seekdir", 2); + set_errno(); + return; } off_t -telldir (DIR *dirp) +telldir(DIR *dirp) { - intercept ("telldir", 2); - set_errno (); - return -1; + intercept("telldir", 2); + set_errno(); + return -1; } ssize_t -sendfile (int out_fd, int in_fd, off_t *offset, size_t count) +sendfile(int out_fd, int in_fd, off_t *offset, size_t count) { - intercept ("sendfile\t", 1); - set_errno (); - return -1; + intercept("sendfile\t", 1); + set_errno(); + return -1; } ssize_t -sendfile64 (int out_fd, int in_fd, off_t *offset, size_t count) +sendfile64(int out_fd, int in_fd, off_t *offset, size_t count) { - intercept ("sendfile64", 1); - set_errno (); - return -1; + intercept("sendfile64", 1); + set_errno(); + return -1; } - int -fcntl (int fd, int cmd, ...) +fcntl(int fd, int cmd, ...) { - intercept ("fcntl", 2); - set_errno (); - return -1; + intercept("fcntl", 2); + set_errno(); + return -1; } - diff --git a/extras/test/ld-preload-test/ld-preload-test.c b/extras/test/ld-preload-test/ld-preload-test.c index f98b1f4ee0e..54dde8c7d54 100644 --- a/extras/test/ld-preload-test/ld-preload-test.c +++ b/extras/test/ld-preload-test/ld-preload-test.c @@ -1,23 +1,13 @@ /* - Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com> + Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com> This file is part of GlusterFS. - GlusterFS is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - GlusterFS is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see - <http://www.gnu.org/licenses/>. + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. */ - /* * LD PRELOAD Test Tool * @@ -56,322 +46,313 @@ #include <sys/uio.h> #include <utime.h> #include <sys/time.h> -#include <attr/xattr.h> +#include <sys/xattr.h> #include <sys/sendfile.h> - -#define PRELOAD_ERRNO_VERF 6449 -inline void +#define PRELOAD_ERRNO_VERF 6449 +void check_err(int ret, char *call, int tabs) { - while (tabs > 0) { - fprintf (stdout, "\t"); - --tabs; - } - if (ret != -1) { - fprintf (stdout, "Not intercepted: %s\n", call); - return; - } - - if (errno != PRELOAD_ERRNO_VERF) { - fprintf (stdout, "Not intercepted: %s: err: %s\n", call, - strerror (errno)); - return; - } + while (tabs > 0) { + fprintf(stdout, "\t"); + --tabs; + } + if (ret != -1) { + fprintf(stdout, "Not intercepted: %s\n", call); + return; + } - fprintf (stdout, "Intercept verified: %s\n", call); + if (errno != PRELOAD_ERRNO_VERF) { + fprintf(stdout, "Not intercepted: %s: err: %s\n", call, + strerror(errno)); return; + } + + fprintf(stdout, "Intercept verified: %s\n", call); + return; } void -usage (FILE *fp) +usage(FILE *fp) { - fprintf (fp, "Usage: ld-preload-test <Options>\n"); - fprintf (fp, "Options\n"); - fprintf (fp, "\t--path\t\tPathname is used as the file/directory" - " created for the test.\n"); - + fprintf(fp, "Usage: ld-preload-test <Options>\n"); + fprintf(fp, "Options\n"); + fprintf(fp, + "\t--path\t\tPathname is used as the file/directory" + " created for the test.\n"); } - int -run_file_tests (char *testfile) +run_file_tests(char *testfile) { - int ret = -1; - struct stat buf; + int ret = -1; + struct stat buf; - assert (testfile); - fprintf (stdout, "Testing creat"); - ret = creat (testfile, S_IRWXU); - check_err (ret, "creat", 2); + assert(testfile); + fprintf(stdout, "Testing creat"); + ret = creat(testfile, S_IRWXU); + check_err(ret, "creat", 2); - fprintf (stdout, "Testing close"); - ret = close (ret); - check_err (ret, "close", 2); + fprintf(stdout, "Testing close"); + ret = close(ret); + check_err(ret, "close", 2); - fprintf (stdout, "Testing open"); - ret = open (testfile, O_RDONLY); - check_err (ret, "open", 2); + fprintf(stdout, "Testing open"); + ret = open(testfile, O_RDONLY); + check_err(ret, "open", 2); - fprintf (stdout, "Testing read"); - ret = read (0, NULL, 0); - check_err (ret, "read", 2); + fprintf(stdout, "Testing read"); + ret = read(0, NULL, 0); + check_err(ret, "read", 2); - fprintf (stdout, "Testing readv"); - ret = readv (0, NULL, 0); - check_err (ret, "readv", 2); + fprintf(stdout, "Testing readv"); + ret = readv(0, NULL, 0); + check_err(ret, "readv", 2); - fprintf (stdout, "Testing pread"); - ret = pread (0, NULL, 0, 0); - check_err (ret, "pread", 2); + fprintf(stdout, "Testing pread"); + ret = pread(0, NULL, 0, 0); + check_err(ret, "pread", 2); - fprintf (stdout, "Testing write"); - ret = write (0, NULL, 0); - check_err (ret, "write", 2); + fprintf(stdout, "Testing write"); + ret = write(0, NULL, 0); + check_err(ret, "write", 2); - fprintf (stdout, "Testing writev"); - ret = writev (0, NULL, 0); - check_err (ret, "writev", 2); + fprintf(stdout, "Testing writev"); + ret = writev(0, NULL, 0); + check_err(ret, "writev", 2); - fprintf (stdout, "Testing pwrite"); - ret = pwrite (0, NULL, 0, 0); - check_err (ret, "pwrite", 2); + fprintf(stdout, "Testing pwrite"); + ret = pwrite(0, NULL, 0, 0); + check_err(ret, "pwrite", 2); - fprintf (stdout, "Testing lseek"); - ret = lseek (0, 0, 0); - check_err (ret, "lseek", 2); + fprintf(stdout, "Testing lseek"); + ret = lseek(0, 0, 0); + check_err(ret, "lseek", 2); - fprintf (stdout, "Testing dup"); - ret = dup (0); - check_err (ret, "dup", 2); + fprintf(stdout, "Testing dup"); + ret = dup(0); + check_err(ret, "dup", 2); - fprintf (stdout, "Testing dup2"); - ret = dup2 (0, 0); - check_err (ret, "dup2", 2); + fprintf(stdout, "Testing dup2"); + ret = dup2(0, 0); + check_err(ret, "dup2", 2); - fprintf (stdout, "Testing fchmod"); - ret = fchmod (0, 0); - check_err (ret, "fchmod", 2); + fprintf(stdout, "Testing fchmod"); + ret = fchmod(0, 0); + check_err(ret, "fchmod", 2); - fprintf (stdout, "Testing fchown"); - ret = fchown (0, 0, 0); - check_err (ret, "fchown", 2); + fprintf(stdout, "Testing fchown"); + ret = fchown(0, 0, 0); + check_err(ret, "fchown", 2); - fprintf (stdout, "Testing fsync"); - ret = fsync (0); - check_err (ret, "fsync", 2); + fprintf(stdout, "Testing fsync"); + ret = fsync(0); + check_err(ret, "fsync", 2); - fprintf (stdout, "Testing ftruncate"); - ret = ftruncate (0, 0); - check_err (ret, "ftruncate", 1); + fprintf(stdout, "Testing ftruncate"); + ret = ftruncate(0, 0); + check_err(ret, "ftruncate", 1); - fprintf (stdout, "Testing fstat"); - ret = fstat (0, &buf); - check_err (ret, "fstat", 1); + fprintf(stdout, "Testing fstat"); + ret = fstat(0, &buf); + check_err(ret, "fstat", 1); - fprintf (stdout, "Testing sendfile"); - ret = sendfile (0, 0, NULL, 0); - check_err (ret, "sendfile", 1); + fprintf(stdout, "Testing sendfile"); + ret = sendfile(0, 0, NULL, 0); + check_err(ret, "sendfile", 1); - fprintf (stdout, "Testing fcntl"); - ret = fcntl (0, 0, NULL); - check_err (ret, "fcntl", 2); + fprintf(stdout, "Testing fcntl"); + ret = fcntl(0, 0, NULL); + check_err(ret, "fcntl", 2); - fprintf (stdout, "Testing close"); - ret = close (ret); - check_err (ret, "close", 2); + fprintf(stdout, "Testing close"); + ret = close(ret); + check_err(ret, "close", 2); - fprintf (stdout, "Testing remove"); - ret = remove (testfile); - check_err (ret, "remove", 2); + fprintf(stdout, "Testing remove"); + ret = remove(testfile); + check_err(ret, "remove", 2); - return ret; + return ret; } - int -run_attr_tests (char *testfile) +run_attr_tests(char *testfile) { - int ret = -1; - char *res = NULL; - struct stat buf; - struct statfs sbuf; - struct statvfs svbuf; - - assert (testfile); - - fprintf (stdout, "Testing chmod"); - ret = chmod (testfile, 0); - check_err (ret, "chmod", 2); - - fprintf (stdout, "Testing chown"); - ret = chown (testfile, 0, 0); - check_err (ret, "chown", 2); - - fprintf (stdout, "Testing link"); - ret = link (testfile, testfile); - check_err (ret, "link", 2); - - fprintf (stdout, "Testing rename"); - ret = rename (testfile, testfile); - check_err (ret, "rename", 2); - - fprintf (stdout, "Testing utimes"); - ret = utimes (testfile, NULL); - check_err (ret, "utimes", 2); - - fprintf (stdout, "Testing utime"); - ret = utime (testfile, NULL); - check_err (ret, "utime", 2); - - fprintf (stdout, "Testing unlink"); - ret = unlink (testfile); - check_err (ret, "unlink", 2); - - fprintf (stdout, "Testing symlink"); - ret = symlink (testfile, testfile); - check_err (ret, "symlink", 2); - - fprintf (stdout, "Testing readlink"); - ret = readlink (testfile, testfile, 0); - check_err (ret, "readlink", 2); - - fprintf (stdout, "Testing realpath"); - ret = 0; - res = realpath ((const char *)testfile, testfile); - if (!res) - ret = -1; - check_err (ret, "realpath", 2); - - fprintf (stdout, "Testing stat"); - ret = stat (testfile, &buf); - check_err (ret, "stat", 1); - - fprintf (stdout, "Testing lstat"); - ret = lstat (testfile, &buf); - check_err (ret, "lstat", 1); - - fprintf (stdout, "Testing statfs"); - ret = statfs (testfile, &sbuf); - check_err (ret, "statfs", 2); - - fprintf (stdout, "Testing statvfs"); - ret = statvfs (testfile, &svbuf); - check_err (ret, "statvfs", 1); - - fprintf (stdout, "Testing getxattr"); - ret = getxattr (testfile, NULL, NULL, 0); - check_err (ret, "getxattr", 2); - - fprintf (stdout, "Testing lgetxattr"); - ret = lgetxattr (testfile, NULL, NULL, 0); - check_err (ret, "lgetxattr", 1); - - fprintf (stdout, "Testing lchown"); - ret = lchown (testfile, 0, 0); - check_err (ret, "lchown", 2); - return 0; + int ret = -1; + char *res = NULL; + struct stat buf; + struct statfs sbuf; + struct statvfs svbuf; + + assert(testfile); + + fprintf(stdout, "Testing chmod"); + ret = chmod(testfile, 0); + check_err(ret, "chmod", 2); + + fprintf(stdout, "Testing chown"); + ret = chown(testfile, 0, 0); + check_err(ret, "chown", 2); + + fprintf(stdout, "Testing link"); + ret = link(testfile, testfile); + check_err(ret, "link", 2); + + fprintf(stdout, "Testing rename"); + ret = rename(testfile, testfile); + check_err(ret, "rename", 2); + + fprintf(stdout, "Testing utimes"); + ret = utimes(testfile, NULL); + check_err(ret, "utimes", 2); + + fprintf(stdout, "Testing utime"); + ret = utime(testfile, NULL); + check_err(ret, "utime", 2); + + fprintf(stdout, "Testing unlink"); + ret = unlink(testfile); + check_err(ret, "unlink", 2); + + fprintf(stdout, "Testing symlink"); + ret = symlink(testfile, testfile); + check_err(ret, "symlink", 2); + + fprintf(stdout, "Testing readlink"); + ret = readlink(testfile, testfile, 0); + check_err(ret, "readlink", 2); + + fprintf(stdout, "Testing realpath"); + ret = 0; + res = realpath((const char *)testfile, testfile); + if (!res) + ret = -1; + check_err(ret, "realpath", 2); + + fprintf(stdout, "Testing stat"); + ret = stat(testfile, &buf); + check_err(ret, "stat", 1); + + fprintf(stdout, "Testing lstat"); + ret = lstat(testfile, &buf); + check_err(ret, "lstat", 1); + + fprintf(stdout, "Testing statfs"); + ret = statfs(testfile, &sbuf); + check_err(ret, "statfs", 2); + + fprintf(stdout, "Testing statvfs"); + ret = statvfs(testfile, &svbuf); + check_err(ret, "statvfs", 1); + + fprintf(stdout, "Testing getxattr"); + ret = getxattr(testfile, NULL, NULL, 0); + check_err(ret, "getxattr", 2); + + fprintf(stdout, "Testing lgetxattr"); + ret = lgetxattr(testfile, NULL, NULL, 0); + check_err(ret, "lgetxattr", 1); + + fprintf(stdout, "Testing lchown"); + ret = lchown(testfile, 0, 0); + check_err(ret, "lchown", 2); + return 0; } - int -run_dev_tests (char *testfile) +run_dev_tests(char *testfile) { - int ret = -1; + int ret = -1; - assert (testfile); + assert(testfile); - fprintf (stdout, "Testing mknod"); - ret = mknod (testfile, 0, 0); - check_err (ret, "mknod", 2); + fprintf(stdout, "Testing mknod"); + ret = mknod(testfile, 0, 0); + check_err(ret, "mknod", 2); - fprintf (stdout, "Testing mkfifo"); - ret = mkfifo (testfile, 0); - check_err (ret, "mkfifo", 2); - return 0; + fprintf(stdout, "Testing mkfifo"); + ret = mkfifo(testfile, 0); + check_err(ret, "mkfifo", 2); + return 0; } int -run_dir_tests (char *testpath) +run_dir_tests(char *testpath) { - int ret = -1; - DIR *dh = NULL; - struct dirent *dire = NULL; - - assert (testpath); - - fprintf (stdout, "Testing mkdir"); - ret = mkdir (testpath, 0); - check_err (ret, "mkdir", 2); - - fprintf (stdout, "Testing rmdir"); - ret = rmdir (testpath); - check_err (ret, "rmdir", 2); - - fprintf (stdout, "Testing opendir"); - ret = 0; - dh = opendir (testpath); - if (!dh) - ret = -1; - check_err (ret, "opendir", 2); - - fprintf (stdout, "Testing readdir"); - ret = 0; - dire = readdir (dh); - if (!dire) - ret = -1; - check_err (ret, "readdir", 1); - - fprintf (stdout, "Testing readdir_r"); - ret = readdir_r (dh, dire, &dire); - check_err (ret, "readdir_r", 1); - - fprintf (stdout, "Testing rewinddir"); - rewinddir (dh); - check_err (-1, "rewinddir", 1); - - fprintf (stdout, "Testing seekdir"); - seekdir (dh, 0); - check_err (-1, "seekdir", 2); - - fprintf (stdout, "Testing telldir"); - ret = telldir (dh); - check_err (ret, "telldir", 2); - - fprintf (stdout, "Testing closedir"); - ret = closedir (dh); - check_err (ret, "closedir", 2); - return 0; + int ret = -1; + DIR *dh = NULL; + struct dirent *dire = NULL; + + assert(testpath); + + fprintf(stdout, "Testing mkdir"); + ret = mkdir(testpath, 0); + check_err(ret, "mkdir", 2); + + fprintf(stdout, "Testing rmdir"); + ret = rmdir(testpath); + check_err(ret, "rmdir", 2); + + fprintf(stdout, "Testing opendir"); + ret = 0; + dh = opendir(testpath); + if (!dh) + ret = -1; + check_err(ret, "opendir", 2); + + fprintf(stdout, "Testing readdir"); + ret = 0; + dire = readdir(dh); + if (!dire) + ret = -1; + check_err(ret, "readdir", 1); + + fprintf(stdout, "Testing readdir_r"); + ret = readdir_r(dh, dire, &dire); + check_err(ret, "readdir_r", 1); + + fprintf(stdout, "Testing rewinddir"); + rewinddir(dh); + check_err(-1, "rewinddir", 1); + + fprintf(stdout, "Testing seekdir"); + seekdir(dh, 0); + check_err(-1, "seekdir", 2); + + fprintf(stdout, "Testing telldir"); + ret = telldir(dh); + check_err(ret, "telldir", 2); + + fprintf(stdout, "Testing closedir"); + ret = closedir(dh); + check_err(ret, "closedir", 2); + return 0; } - - int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - char *testpath = NULL; - int x = 0; - - for (;x < argc; ++x) { - if (strcmp (argv[x], "--path") == 0) { - testpath = argv[x+1]; - continue; - } + char *testpath = NULL; + int x = 0; + for (; x < argc; ++x) { + if (strcmp(argv[x], "--path") == 0) { + testpath = argv[x + 1]; + continue; } + } - if (!testpath) { - fprintf (stderr, "--path not specified\n"); - usage (stderr); - return -1; - } + if (!testpath) { + fprintf(stderr, "--path not specified\n"); + usage(stderr); + return -1; + } - run_file_tests (testpath); - run_dir_tests (testpath); - run_attr_tests (testpath); - run_dev_tests (testpath); + run_file_tests(testpath); + run_dir_tests(testpath); + run_attr_tests(testpath); + run_dev_tests(testpath); - return 0; + return 0; } - - diff --git a/extras/test/open-fd-tests.c b/extras/test/open-fd-tests.c new file mode 100644 index 00000000000..509952b4180 --- /dev/null +++ b/extras/test/open-fd-tests.c @@ -0,0 +1,67 @@ + +#include <stdio.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/xattr.h> +#include <errno.h> +#include <string.h> + +int +main(int argc, char *argv[]) +{ + int ret = -1; + int fd = 0; + char *filename = NULL; + int loop = 0; + struct stat stbuf = { + 0, + }; + char string[1024] = { + 0, + }; + + if (argc > 1) + filename = argv[1]; + + if (!filename) + filename = "temp-fd-test-file"; + + fd = open(filename, O_RDWR | O_CREAT | O_TRUNC); + if (fd < 0) { + fd = 0; + fprintf(stderr, "open failed : %s\n", strerror(errno)); + goto out; + } + + while (loop < 1000) { + /* Use it as a mechanism to test time delays */ + memset(string, 0, 1024); + scanf("%s", string); + + ret = write(fd, string, strlen(string)); + if (ret != strlen(string)) { + fprintf(stderr, "write failed : %s (%s %d)\n", strerror(errno), + string, loop); + goto out; + } + + ret = write(fd, "\n", 1); + if (ret != 1) { + fprintf(stderr, "write failed : %s (%d)\n", strerror(errno), loop); + goto out; + } + + loop++; + } + + fprintf(stdout, "finishing the test after %d loops\n", loop); + + ret = 0; +out: + if (fd) + close(fd); + + return ret; +} diff --git a/extras/test/run.sh b/extras/test/run.sh new file mode 100755 index 00000000000..4b3839cf679 --- /dev/null +++ b/extras/test/run.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +# Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +# Running gluster sanity test which starts glusterd and runs gluster commands, and exit at the first failure. +$PWD/gluster_commands.sh + +if [ $? -ne 0 ]; then + echo "sanity failed" +else + echo "sanity passed" +fi + +# Stopping glusterd +$PWD/stop_glusterd.sh + diff --git a/extras/test/stop_glusterd.sh b/extras/test/stop_glusterd.sh new file mode 100755 index 00000000000..a2db13f4241 --- /dev/null +++ b/extras/test/stop_glusterd.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +#This script stops the glusterd running on the machine. Helpful for gluster sanity script + +killall -9 glusterd + +if [ $? -ne 0 ]; then + echo "Error: Could not kill glusterd. Either glusterd is not running or kill it manually" +else + echo "Killed glusterd" +fi diff --git a/extras/test/test-ffop.c b/extras/test/test-ffop.c new file mode 100644 index 00000000000..1d9c125db67 --- /dev/null +++ b/extras/test/test-ffop.c @@ -0,0 +1,920 @@ +#include <stdio.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/xattr.h> +#include <errno.h> +#include <string.h> +#include <dirent.h> + +int +fd_based_fops_1(char *filename); // for fd based fops after unlink +int +fd_based_fops_2(char *filename); // for fd based fops before unlink +int +dup_fd_based_fops(char *filename); // fops based on fd after dup +int +path_based_fops(char *filename); // for fops based on path +int +dir_based_fops(char *filename); // for fops which operate on directory +int +link_based_fops( + char *filename); // for fops which operate in link files (symlinks) +int +test_open_modes( + char *filename); // to test open syscall with open modes available. +int +generic_open_read_write( + char *filename, + int flag); // generic function which does open write and read. + +int +main(int argc, char *argv[]) +{ + int ret = -1; + char filename[255] = { + 0, + }; + + if (argc > 1) + strcpy(filename, argv[1]); + else + strcpy(filename, "temp-xattr-test-file"); + + ret = fd_based_fops_1(strcat(filename, "_1")); + if (ret < 0) + fprintf(stderr, "fd based file operation 1 failed\n"); + else + fprintf(stdout, "fd based file operation 1 passed\n"); + + ret = fd_based_fops_2(strcat(filename, "_2")); + if (ret < 0) + fprintf(stderr, "fd based file operation 2 failed\n"); + else + fprintf(stdout, "fd based file operation 2 passed\n"); + + ret = dup_fd_based_fops(strcat(filename, "_3")); + if (ret < 0) + fprintf(stderr, "dup fd based file operation failed\n"); + else + fprintf(stdout, "dup fd based file operation passed\n"); + + ret = path_based_fops(strcat(filename, "_4")); + if (ret < 0) + fprintf(stderr, "path based file operation failed\n"); + else + fprintf(stdout, "path based file operation passed\n"); + + ret = dir_based_fops(strcat(filename, "_5")); + if (ret < 0) + fprintf(stderr, "directory based file operation failed\n"); + else + fprintf(stdout, "directory based file operation passed\n"); + + ret = link_based_fops(strcat(filename, "_5")); + if (ret < 0) + fprintf(stderr, "link based file operation failed\n"); + else + fprintf(stdout, "link based file operation passed\n"); + + ret = test_open_modes(strcat(filename, "_5")); + if (ret < 0) + fprintf(stderr, "testing modes of 'open' call failed\n"); + else + fprintf(stdout, "testing modes of 'open' call passed\n"); + +out: + return ret; +} + +int +fd_based_fops_1(char *filename) +{ + int fd = 0; + int ret = -1; + struct stat stbuf = { + 0, + }; + char wstr[50] = { + 0, + }; + char rstr[50] = { + 0, + }; + + fd = open(filename, O_RDWR | O_CREAT); + if (fd < 0) { + fd = 0; + fprintf(stderr, "open failed : %s\n", strerror(errno)); + goto out; + } + + ret = unlink(filename); + if (ret < 0) { + fprintf(stderr, "unlink failed : %s\n", strerror(errno)); + goto out; + } + + strcpy(wstr, "This is my string\n"); + ret = write(fd, wstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "write failed: %s\n", strerror(errno)); + goto out; + } + + ret = lseek(fd, 0, SEEK_SET); + if (ret < 0) { + fprintf(stderr, "lseek failed: %s\n", strerror(errno)); + goto out; + } + + ret = read(fd, rstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "read failed: %s\n", strerror(errno)); + goto out; + } + + ret = memcmp(rstr, wstr, strlen(wstr)); + if (ret != 0) { + ret = -1; + fprintf(stderr, "read returning junk\n"); + goto out; + } + + ret = ftruncate(fd, 0); + if (ret < 0) { + fprintf(stderr, "ftruncate failed : %s\n", strerror(errno)); + goto out; + } + + ret = fstat(fd, &stbuf); + if (ret < 0) { + fprintf(stderr, "fstat failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchmod(fd, 0640); + if (ret < 0) { + fprintf(stderr, "fchmod failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchown(fd, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "fchown failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsync(fd); + if (ret < 0) { + fprintf(stderr, "fsync failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsetxattr(fd, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fdatasync(fd); + if (ret < 0) { + fprintf(stderr, "fdatasync failed : %s\n", strerror(errno)); + goto out; + } + + ret = flistxattr(fd, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "flistxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fgetxattr(fd, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fremovexattr(fd, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = 0; +out: + if (fd) + close(fd); + + return ret; +} + +int +fd_based_fops_2(char *filename) +{ + int fd = 0; + int ret = -1; + struct stat stbuf = { + 0, + }; + char wstr[50] = { + 0, + }; + char rstr[50] = { + 0, + }; + + fd = open(filename, O_RDWR | O_CREAT); + if (fd < 0) { + fd = 0; + fprintf(stderr, "open failed : %s\n", strerror(errno)); + goto out; + } + + ret = ftruncate(fd, 0); + + if (ret < 0) { + fprintf(stderr, "ftruncate failed : %s\n", strerror(errno)); + goto out; + } + + strcpy(wstr, "This is my second string\n"); + ret = write(fd, wstr, strlen(wstr)); + if (ret < 0) { + ret = -1; + fprintf(stderr, "write failed: %s\n", strerror(errno)); + goto out; + } + + lseek(fd, 0, SEEK_SET); + if (ret < 0) { + fprintf(stderr, "lseek failed: %s\n", strerror(errno)); + goto out; + } + + ret = read(fd, rstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "read failed: %s\n", strerror(errno)); + goto out; + } + + ret = memcmp(rstr, wstr, strlen(wstr)); + if (ret != 0) { + ret = -1; + fprintf(stderr, "read returning junk\n"); + goto out; + } + + ret = fstat(fd, &stbuf); + if (ret < 0) { + fprintf(stderr, "fstat failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchmod(fd, 0640); + if (ret < 0) { + fprintf(stderr, "fchmod failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchown(fd, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "fchown failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsync(fd); + if (ret < 0) { + fprintf(stderr, "fsync failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsetxattr(fd, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fdatasync(fd); + if (ret < 0) { + fprintf(stderr, "fdatasync failed : %s\n", strerror(errno)); + goto out; + } + + ret = flistxattr(fd, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "flistxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fgetxattr(fd, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fremovexattr(fd, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno)); + goto out; + } + +out: + if (fd) + close(fd); + unlink(filename); + + return ret; +} + +int +path_based_fops(char *filename) +{ + int ret = -1; + int fd = 0; + struct stat stbuf = { + 0, + }; + char newfilename[255] = { + 0, + }; + + fd = creat(filename, 0644); + if (fd < 0) { + fprintf(stderr, "creat failed: %s\n", strerror(errno)); + goto out; + } + + ret = truncate(filename, 0); + if (ret < 0) { + fprintf(stderr, "truncate failed: %s\n", strerror(errno)); + goto out; + } + + ret = stat(filename, &stbuf); + if (ret < 0) { + fprintf(stderr, "stat failed: %s\n", strerror(errno)); + goto out; + } + + ret = chmod(filename, 0640); + if (ret < 0) { + fprintf(stderr, "chmod failed: %s\n", strerror(errno)); + goto out; + } + + ret = chown(filename, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "chown failed: %s\n", strerror(errno)); + goto out; + } + + ret = setxattr(filename, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "setxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = listxattr(filename, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "listxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = getxattr(filename, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "getxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = removexattr(filename, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "removexattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = access(filename, R_OK | W_OK); + if (ret < 0) { + fprintf(stderr, "access failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(newfilename, filename); + strcat(newfilename, "_new"); + ret = rename(filename, newfilename); + if (ret < 0) { + fprintf(stderr, "rename failed: %s\n", strerror(errno)); + goto out; + } + unlink(newfilename); + +out: + if (fd) + close(fd); + + unlink(filename); + return ret; +} + +int +dup_fd_based_fops(char *filename) +{ + int fd = 0; + int newfd = 0; + int ret = -1; + struct stat stbuf = { + 0, + }; + char wstr[50] = { + 0, + }; + char rstr[50] = { + 0, + }; + + fd = open(filename, O_RDWR | O_CREAT); + if (fd < 0) { + fd = 0; + fprintf(stderr, "open failed : %s\n", strerror(errno)); + goto out; + } + + newfd = dup(fd); + if (newfd < 0) { + ret = -1; + fprintf(stderr, "dup failed: %s\n", strerror(errno)); + goto out; + } + + close(fd); + + strcpy(wstr, "This is my string\n"); + ret = write(newfd, wstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "write failed: %s\n", strerror(errno)); + goto out; + } + + ret = lseek(newfd, 0, SEEK_SET); + if (ret < 0) { + fprintf(stderr, "lseek failed: %s\n", strerror(errno)); + goto out; + } + + ret = read(newfd, rstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "read failed: %s\n", strerror(errno)); + goto out; + } + + ret = memcmp(rstr, wstr, strlen(wstr)); + if (ret != 0) { + ret = -1; + fprintf(stderr, "read returning junk\n"); + goto out; + } + + ret = ftruncate(newfd, 0); + if (ret < 0) { + fprintf(stderr, "ftruncate failed : %s\n", strerror(errno)); + goto out; + } + + ret = fstat(newfd, &stbuf); + if (ret < 0) { + fprintf(stderr, "fstat failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchmod(newfd, 0640); + if (ret < 0) { + fprintf(stderr, "fchmod failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchown(newfd, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "fchown failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsync(newfd); + if (ret < 0) { + fprintf(stderr, "fsync failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsetxattr(newfd, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fdatasync(newfd); + if (ret < 0) { + fprintf(stderr, "fdatasync failed : %s\n", strerror(errno)); + goto out; + } + + ret = flistxattr(newfd, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "flistxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fgetxattr(newfd, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fremovexattr(newfd, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = 0; +out: + if (newfd) + close(newfd); + ret = unlink(filename); + if (ret < 0) + fprintf(stderr, "unlink failed : %s\n", strerror(errno)); + + return ret; +} + +int +dir_based_fops(char *dirname) +{ + int ret = -1; + DIR *dp = NULL; + char buff[255] = { + 0, + }; + struct dirent *dbuff = { + 0, + }; + struct stat stbuff = { + 0, + }; + char newdname[255] = { + 0, + }; + char *cwd = NULL; + + ret = mkdir(dirname, 0755); + if (ret < 0) { + fprintf(stderr, "mkdir failed: %s\n", strerror(errno)); + goto out; + } + + dp = opendir(dirname); + if (dp == NULL) { + fprintf(stderr, "opendir failed: %s\n", strerror(errno)); + goto out; + } + + dbuff = readdir(dp); + if (NULL == dbuff) { + fprintf(stderr, "readdir failed: %s\n", strerror(errno)); + goto out; + } + + ret = closedir(dp); + if (ret < 0) { + fprintf(stderr, "closedir failed: %s\n", strerror(errno)); + goto out; + } + + ret = stat(dirname, &stbuff); + if (ret < 0) { + fprintf(stderr, "stat failed: %s\n", strerror(errno)); + goto out; + } + + ret = chmod(dirname, 0744); + if (ret < 0) { + fprintf(stderr, "chmod failed: %s\n", strerror(errno)); + goto out; + } + + ret = chown(dirname, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "chmod failed: %s\n", strerror(errno)); + goto out; + } + + ret = setxattr(dirname, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "setxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = listxattr(dirname, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "listxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = getxattr(dirname, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "getxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = removexattr(dirname, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "removexattr failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(newdname, dirname); + strcat(newdname, "/../"); + ret = chdir(newdname); + if (ret < 0) { + fprintf(stderr, "chdir failed: %s\n", strerror(errno)); + goto out; + } + + cwd = getcwd(buff, 255); + if (NULL == cwd) { + fprintf(stderr, "getcwd failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(newdname, dirname); + strcat(newdname, "new"); + ret = rename(dirname, newdname); + if (ret < 0) { + fprintf(stderr, "rename failed: %s\n", strerror(errno)); + goto out; + } + + ret = rmdir(newdname); + if (ret < 0) { + fprintf(stderr, "rmdir failed: %s\n", strerror(errno)); + return ret; + } + +out: + rmdir(dirname); + return ret; +} + +int +link_based_fops(char *filename) +{ + int ret = -1; + int fd = 0; + char newname[255] = { + 0, + }; + char linkname[255] = { + 0, + }; + struct stat lstbuf = { + 0, + }; + + fd = creat(filename, 0644); + if (fd < 0) { + fd = 0; + fprintf(stderr, "creat failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(newname, filename); + strcat(newname, "_hlink"); + ret = link(filename, newname); + if (ret < 0) { + fprintf(stderr, "link failed: %s\n", strerror(errno)); + goto out; + } + + ret = unlink(filename); + if (ret < 0) { + fprintf(stderr, "unlink failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(linkname, filename); + strcat(linkname, "_slink"); + ret = symlink(newname, linkname); + if (ret < 0) { + fprintf(stderr, "symlink failed: %s\n", strerror(errno)); + goto out; + } + + ret = lstat(linkname, &lstbuf); + if (ret < 0) { + fprintf(stderr, "lstbuf failed: %s\n", strerror(errno)); + goto out; + } + + ret = lchown(linkname, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "lchown failed: %s\n", strerror(errno)); + goto out; + } + + ret = lsetxattr(linkname, "trusted.lxattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "lsetxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = llistxattr(linkname, NULL, 0); + if (ret < 0) { + ret = -1; + fprintf(stderr, "llistxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = lgetxattr(linkname, "trusted.lxattr-test", NULL, 0); + if (ret < 0) { + ret = -1; + fprintf(stderr, "lgetxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = lremovexattr(linkname, "trusted.lxattr-test"); + if (ret < 0) { + fprintf(stderr, "lremovexattr failed: %s\n", strerror(errno)); + goto out; + } + +out: + if (fd) + close(fd); + unlink(linkname); + unlink(newname); +} + +int +test_open_modes(char *filename) +{ + int ret = -1; + + ret = generic_open_read_write(filename, O_CREAT | O_WRONLY); + if (3 != ret) { + fprintf(stderr, "flag O_CREAT|O_WRONLY failed: \n"); + goto out; + } + + ret = generic_open_read_write(filename, O_CREAT | O_RDWR); + if (ret != 0) { + fprintf(stderr, "flag O_CREAT|O_RDWR failed\n"); + goto out; + } + + ret = generic_open_read_write(filename, O_CREAT | O_RDONLY); + if (ret != 0) { + fprintf(stderr, "flag O_CREAT|O_RDONLY failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_WRONLY); + if (3 != ret) { + fprintf(stderr, "flag O_WRONLY failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_RDWR); + if (0 != ret) { + fprintf(stderr, "flag O_RDWR failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_RDONLY); + if (0 != ret) { + fprintf(stderr, "flag O_RDONLY failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_TRUNC | O_WRONLY); + if (3 != ret) { + fprintf(stderr, "flag O_TRUNC|O_WRONLY failed\n"); + goto out; + } + +#if 0 /* undefined behaviour, unable to reliably test */ + ret = creat (filename, 0644); + close (ret); + ret = generic_open_read_write (filename, O_TRUNC|O_RDONLY); + if (0 != ret) { + fprintf (stderr, "flag O_TRUNC|O_RDONLY failed\n"); + goto out; + } +#endif + + ret = generic_open_read_write(filename, O_CREAT | O_RDWR | O_SYNC); + if (0 != ret) { + fprintf(stderr, "flag O_CREAT|O_RDWR|O_SYNC failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_CREAT | O_EXCL); + if (0 != ret) { + fprintf(stderr, "flag O_CREAT|O_EXCL failed\n"); + goto out; + } + +out: + return ret; +} + +int +generic_open_read_write(char *filename, int flag) +{ + int fd = 0; + int ret = -1; + char wstring[50] = { + 0, + }; + char rstring[50] = { + 0, + }; + + fd = open(filename, flag); + if (fd < 0) { + if (flag == O_CREAT | O_EXCL && errno == EEXIST) { + unlink(filename); + return 0; + } else { + fd = 0; + fprintf(stderr, "open failed: %s\n", strerror(errno)); + return 1; + } + } + + strcpy(wstring, "My string to write\n"); + ret = write(fd, wstring, strlen(wstring)); + if (ret <= 0) { + if (errno != EBADF) { + fprintf(stderr, "write failed: %s\n", strerror(errno)); + close(fd); + unlink(filename); + return 2; + } + } + + ret = lseek(fd, 0, SEEK_SET); + if (ret < 0) { + close(fd); + unlink(filename); + return 4; + } + + ret = read(fd, rstring, strlen(wstring)); + if (ret < 0) { + close(fd); + unlink(filename); + return 3; + } + + /* Compare the rstring with wstring. But we do not want to return + * error when the flag is either O_RDONLY, O_CREAT|O_RDONLY or + * O_TRUNC|O_RDONLY. Because in that case we are not writing + * anything to the file.*/ + + ret = memcmp(wstring, rstring, strlen(wstring)); + if (0 != ret && !(flag == O_CREAT | O_RDONLY || flag == O_RDONLY || + flag == O_TRUNC | O_RDONLY)) { + fprintf(stderr, "read is returning junk\n"); + close(fd); + unlink(filename); + return 4; + } + + close(fd); + unlink(filename); + return 0; +} diff --git a/extras/thin-arbiter/setup-thin-arbiter.sh b/extras/thin-arbiter/setup-thin-arbiter.sh new file mode 100755 index 00000000000..0681b30ef3f --- /dev/null +++ b/extras/thin-arbiter/setup-thin-arbiter.sh @@ -0,0 +1,184 @@ +#!/bin/bash +# Copyright (c) 2018-2019 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + + +# This tool has been developed to setup thin-arbiter process on a node. +# Seting up a thin arbiter process involves following files - +# 1 - thin-arbiter.vol +# Thin-arbiter (TA) process will use the graph in this file to load the +# required translators. +# 2 - gluster-ta-volume.service (generated by gluster-ta-volume.service.in) +# TA process would be running as systemd service. +# +# TA process uses a location to save TA id files for every subvolume. +# This location can be taken as input from user. Once provided and the +# TA process is started on a node, it can not be changed using this +# script or by any other mean. The same location should be used in +# the gluster CLI when creating thin-arbiter volumes. + +MYPATH=`dirname $0` + +volloc="/var/lib/glusterd/thin-arbiter" +mkdir -p $volloc + +if [ -f /etc/glusterfs/thin-arbiter.vol ]; then + volfile=/etc/glusterfs/thin-arbiter.vol +else + volfile=$MYPATH/thin-arbiter.vol +fi + +tafile="$volloc/thin-arbiter.vol" + + +help () { + echo " " + echo ' This tool helps to setup thin-arbiter (TA) process on a node. + TA process uses a location to save TA id files for every subvolume. + This location can be taken as input from user. Once provided and the + TA process is started on a node, it can not be changed using this script + or by any other mean. The same location should be used in gluster CLI + when creating thin-arbiter volumes. + + usage: setup-thin-arbiter.sh [-s] [-h] + options: + -s - Setup thin-arbiter file path and start process + -h - Show this help message and exit +' +} + +volfile_set_brick_path () { + while read -r line + do + dir=`echo "$line" | cut -d' ' -f 2` + if [ "$dir" = "directory" ]; then + bpath=`echo "$line" | cut -d' ' -f 3` + sed -i -- 's?'$bpath'?'$1'?g' $tafile + return + fi + done < $tafile +} + +check_ta_proc () { + pro=`ps aux | grep thin-arbiter.vol | grep "volfile-id"` + if [ "${pro}" = '' ]; then + echo "" + else + curr_loc=`cat $volloc/thin-arbiter.vol | grep option | grep directory` + loc=`echo "${curr_loc##* }"` + echo "******************************************************" + echo "Error:" + echo "Thin-arbiter process is running with thin-arbiter path = $loc" + echo "Can not change TA path on this host now." + echo "$pro" + echo "******************************************************" + exit 1 + fi +} + +getpath () { + check_ta_proc + echo "******************************************************" + echo "User will be required to enter a path/folder for arbiter volume." + echo "Please note that this path will be used for ALL VOLUMES using this" + echo "node to host thin-arbiter. After setting, if a volume" + echo "has been created using this host and path then path for" + echo "thin-arbiter can not be changed " + echo "******************************************************" + echo " " + while true; + do + echo -n "Enter brick path for thin arbiter volumes: " + echo " " + read tapath + if [ "${tapath}" = '' ]; then + echo "Please enter valid path" + continue + else + echo "Entered brick path : $tapath " + echo "Please note that this brick path will be used for ALL" + echo "VOLUMES using this node to host thin-arbiter brick" + echo -n "Want to continue? (y/N): " + echo " " + read cont + + if [ "${cont}" = 'N' ] || [ "${cont}" = 'n' ]; then + exit 0 + else + break + fi + fi + done +} + +setup () { + getpath + mkdir -p $tapath/.glusterfs/indices + if [ -d $tapath/.glusterfs/indices ]; then + echo " " + else + echo "Could not create $tapath/.glusterfs/indices directory, check provided ta path." + exit 1 + fi + + cp -f --backup --suffix=_old $volfile $volloc/thin-arbiter.vol + volfile_set_brick_path "$tapath" + + echo "Directory path to be used for thin-arbiter volume is: $tapath" + echo " " + echo "========================================================" + + if [ -f /usr/lib/systemd/system/gluster-ta-volume.service ]; then + echo "Starting thin-arbiter process" + else + cp $MYPATH/../systemd/gluster-ta-volume.service /etc/systemd/system/ + echo "Starting thin-arbiter process" + chmod 0644 /etc/systemd/system/gluster-ta-volume.service + fi + + systemctl daemon-reload + systemctl enable gluster-ta-volume + systemctl stop gluster-ta-volume + systemctl start gluster-ta-volume + + if [ $? == 0 ]; then + echo "thin-arbiter process has been setup and running" + else + echo "Failed to setup thin arbiter" + exit 1 + fi + +} + +main() +{ + + if [ "$#" -ne 1 ]; then + help + exit 0 + fi + + while getopts "sh" opt; do + case $opt in + h) + help + exit 0 + ;; + s) + setup + exit 0 + ;; + *) + help + exit 0 + ;; + esac + done +} + +main "$@" diff --git a/extras/thin-arbiter/thin-arbiter.vol b/extras/thin-arbiter/thin-arbiter.vol new file mode 100644 index 00000000000..c76babc7b3c --- /dev/null +++ b/extras/thin-arbiter/thin-arbiter.vol @@ -0,0 +1,57 @@ +volume ta-posix + type storage/posix + option directory /mnt/thin-arbiter +end-volume + +volume ta-thin-arbiter + type features/thin-arbiter + subvolumes ta-posix +end-volume + +volume ta-locks + type features/locks + option notify-contention yes + subvolumes ta-thin-arbiter +end-volume + +volume ta-upcall + type features/upcall + option cache-invalidation off + subvolumes ta-locks +end-volume + +volume ta-io-threads + type performance/io-threads + subvolumes ta-upcall +end-volume + +volume ta-index + type features/index + option xattrop-pending-watchlist trusted.afr.ta- + option xattrop-dirty-watchlist trusted.afr.dirty + option index-base /mnt/thin-arbiter/.glusterfs/indices + subvolumes ta-io-threads +end-volume + +volume /mnt/thin-arbiter + type debug/io-stats + option count-fop-hits off + option latency-measurement off + option unique-id /mnt/thin-arbiter + subvolumes ta-index +end-volume + +volume ta-server + type protocol/server + option transport.listen-backlog 10 + option transport.socket.keepalive-count 9 + option transport.socket.keepalive-interval 2 + option transport.socket.keepalive-time 20 + option transport.tcp-user-timeout 0 + option transport.socket.keepalive 1 + option auth.addr./mnt/thin-arbiter.allow * + option auth-path /mnt/thin-arbiter + option transport.address-family inet + option transport-type tcp + subvolumes /mnt/thin-arbiter +end-volume diff --git a/extras/volfilter.py b/extras/volfilter.py new file mode 100644 index 00000000000..5558a1beff4 --- /dev/null +++ b/extras/volfilter.py @@ -0,0 +1,168 @@ +# Copyright (c) 2010-2011 Red Hat, Inc. +# +# This file is part of HekaFS. +# +# HekaFS is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License, version 3, as published by the Free +# Software Foundation. +# +# HekaFS is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License * along +# with HekaFS. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import print_function +import copy +import string +import sys +import types + +good_xlators = [ + "cluster/afr", + "cluster/dht", + "cluster/distribute", + "cluster/replicate", + "cluster/stripe", + "debug/io-stats", + "features/access-control", + "features/locks", + "features/marker", + "features/uidmap", + "performance/io-threads", + "protocol/client", + "protocol/server", + "storage/posix", +] + +def copy_stack (old_xl, suffix, recursive=False): + if recursive: + new_name = old_xl.name + "-" + suffix + else: + new_name = suffix + new_xl = Translator(new_name) + new_xl.type = old_xl.type + # The results with normal assignment here are . . . amusing. + new_xl.opts = copy.deepcopy(old_xl.opts) + for sv in old_xl.subvols: + new_xl.subvols.append(copy_stack(sv, suffix, True)) + # Patch up the path at the bottom. + if new_xl.type == "storage/posix": + new_xl.opts["directory"] += ("/" + suffix) + return new_xl + +def cleanup (parent, graph): + if parent.type in good_xlators: + # Temporary fix so that HekaFS volumes can use the + # SSL-enabled multi-threaded socket transport. + if parent.type == "protocol/server": + parent.type = "protocol/server2" + parent.opts["transport-type"] = "ssl" + elif parent.type == "protocol/client": + parent.type = "protocol/client2" + parent.opts["transport-type"] = "ssl" + sv = [] + for child in parent.subvols: + sv.append(cleanup(child, graph)) + parent.subvols = sv + else: + parent = cleanup(parent.subvols[0], graph) + return parent + +class Translator: + def __init__ (self, name): + self.name = name + self.type = "" + self.opts = {} + self.subvols = [] + self.dumped = False + def __repr__ (self): + return "<Translator %s>" % self.name + +def load (path): + # If it's a string, open it; otherwise, assume it's already a + # file-like object (most notably from urllib*). + if type(path) in (str,): + fp = file(path, "r") + else: + fp = path + all_xlators = {} + xlator = None + last_xlator = None + while True: + text = fp.readline() + if text == "": + break + text = text.split() + if not len(text): + continue + if text[0] == "volume": + if xlator: + raise RuntimeError("nested volume definition") + xlator = Translator(text[1]) + continue + if not xlator: + raise RuntimeError("text outside volume definition") + if text[0] == "type": + xlator.type = text[1] + continue + if text[0] == "option": + xlator.opts[text[1]] = ''.join(text[2:]) + continue + if text[0] == "subvolumes": + for sv in text[1:]: + xlator.subvols.append(all_xlators[sv]) + continue + if text[0] == "end-volume": + all_xlators[xlator.name] = xlator + last_xlator = xlator + xlator = None + continue + raise RuntimeError("unrecognized keyword %s" % text[0]) + if xlator: + raise RuntimeError("unclosed volume definition") + return all_xlators, last_xlator + +def generate (graph, last, stream=sys.stdout): + for sv in last.subvols: + if not sv.dumped: + generate(graph, sv, stream) + print("", file=stream) + sv.dumped = True + print("volume %s" % last.name, file=stream) + print(" type %s" % last.type, file=stream) + for k, v in last.opts.items(): + print(" option %s %s" % (k, v), file=stream) + if last.subvols: + print(" subvolumes %s" % ''.join( + [ sv.name for sv in last.subvols ]), file=stream) + print("end-volume", file=stream) + +def push_filter (graph, old_xl, filt_type, opts={}): + suffix = "-" + old_xl.type.split("/")[1] + if len(old_xl.name) > len(suffix): + if old_xl.name[-len(suffix):] == suffix: + old_xl.name = old_xl.name[:-len(suffix)] + new_xl = Translator(old_xl.name+suffix) + new_xl.type = old_xl.type + new_xl.opts = old_xl.opts + new_xl.subvols = old_xl.subvols + graph[new_xl.name] = new_xl + old_xl.name += ("-" + filt_type.split("/")[1]) + old_xl.type = filt_type + old_xl.opts = opts + old_xl.subvols = [new_xl] + graph[old_xl.name] = old_xl + +def delete (graph, victim): + if len(victim.subvols) != 1: + raise RuntimeError("attempt to delete non-unary translator") + for xl in graph.itervalues(): + while xl.subvols.count(victim): + i = xl.subvols.index(victim) + xl.subvols[i] = victim.subvols[0] + +if __name__ == "__main__": + graph, last = load(sys.argv[1]) + generate(graph, last) diff --git a/extras/volgen/CreateBooster.py b/extras/volgen/CreateBooster.py deleted file mode 100644 index 488426436de..00000000000 --- a/extras/volgen/CreateBooster.py +++ /dev/null @@ -1,123 +0,0 @@ -GLUSTERFS_BOOSTER_FSTAB = "booster.fstab" -GLUSTERFS_UNFS3_EXPORTS = "boosterexports" -GLUSTERFS_CIFS_CONFIG = "boostersmb.conf" -LOGDIR = "/var/log/glusterfs" -fstype = "glusterfs" - -class CreateBooster: - - def __init__ (self, options, transports): - - self.volume_name = options.volume_name - self.need_nfs = options.need_nfs - self.need_cifs = options.need_cifs - self.username = options.cifs_username - self.enable_guest = options.enable_guest - self.conf_dir = options.conf_dir - self.transports = transports - - def configure_booster_fstab (self): - - _fstab = "" - _options = "" - _options_log = "" - _options_ext = "" - - if self.conf_dir: - booster_fstab_fd = file ("%s/%s" % (str(self.conf_dir), - GLUSTERFS_BOOSTER_FSTAB), "a") - else: - booster_fstab_fd = file (GLUSTERFS_BOOSTER_FSTAB, "a") - - if self.need_nfs: - for transport in self.transports: - if self.conf_dir: - _fstab = "%s/%s-%s.vol %s" % (str(self.conf_dir), - self.volume_name, - transport, - str("/nfs/" + - self.volume_name)) - else: - _fstab = "%s-%s.vol %s" % (self.volume_name, - transport, - str("/nfs/" + - self.volume_name)) - - _options = "%s" % fstype - _options_log = "logfile=%s/%s-nfs.log" % (LOGDIR, self.volume_name) - _options_ext = "loglevel=ERROR,attr_timeout=0" - booster_fstab_fd.write ("%s %s %s,%s\n" % - (_fstab, - _options, - _options_log, - _options_ext)) - - if self.need_cifs: - for transport in self.transports: - if self.conf_dir: - _fstab = "%s/%s-%s.vol %s" % (self.conf_dir, - self.volume_name, - transport, - str("/cifs/" + - self.volume_name)) - else: - _fstab = "%s-%s.vol %s" % (self.volume_name, - transport, - str("/cifs/" + - self.volume_name)) - - _options = "%s" % fstype - _options_log = "logfile=%s/%s-cifs.log" % (LOGDIR, self.volume_name) - _options_ext = "loglevel=ERROR,attr_timeout=0" - booster_fstab_fd.write ("%s %s %s,%s\n" % - (_fstab, - _options, - _options_log, - _options_ext)) - - return - - def configure_nfs_booster (self): - - if self.conf_dir: - nfs_exports_fd = file ("%s/%s" % (str(self.conf_dir), - GLUSTERFS_UNFS3_EXPORTS), "a") - else: - nfs_exports_fd = file (GLUSTERFS_UNFS3_EXPORTS, "a") - - nfs_exports_fd.write ("%s 0.0.0.0/0(rw,no_root_squash)\n" % - str("/nfs/" + self.volume_name)) - return - - def configure_cifs_booster (self): - - if self.conf_dir: - cifs_config_fd = file ("%s/%s" % (str(self.conf_dir), - GLUSTERFS_CIFS_CONFIG), "a") - else: - cifs_config_fd = file (GLUSTERFS_CIFS_CONFIG, "a") - cifs_config_fd.write ("[%s]\n" % self.volume_name) - cifs_config_fd.write ("comment = %s volume served by Gluster\n" % - self.volume_name) - cifs_config_fd.write ("path = %s\n" % str("/cifs/" + self.volume_name)) - - if self.enable_guest: - cifs_config_fd.write ("guest ok = yes\n") - - cifs_config_fd.write ("public = yes\n") - cifs_config_fd.write ("writable = yes\n") - cifs_config_fd.write ("users = %s\n" % self.username) - cifs_config_fd.close() - return - - def configure_booster (self): - - self.configure_booster_fstab() - if self.need_nfs: - self.configure_nfs_booster() - print "Generating booster configuration for NFS reexport" - if self.need_cifs: - self.configure_cifs_booster() - print "Generating booster configuration for CIFS reexport" - - return diff --git a/extras/volgen/CreateVolfile.py b/extras/volgen/CreateVolfile.py deleted file mode 100644 index 2cca8903bb3..00000000000 --- a/extras/volgen/CreateVolfile.py +++ /dev/null @@ -1,282 +0,0 @@ -import os, sys, string -import subprocess - -num_replica = 2 -num_stripe = 4 -cache_size = "1GB" - -class CreateVolfile: - - def __init__ (self, server_dict, server, transport, - transports, options, server_array): - - self.host_dict = server_dict - self.host = server - self.volume_name = options.volume_name - self.transport = transport - self.transports = transports - self.gfs_port = options.port - self.gfs_ib_port = options.port + 1 - self.auth_parameters = options.auth_param - self.raid_type = options.raid_type - self.ib_devport = options.ib_dev - self.num_servers = len (self.host_dict.keys()) - self.conf_dir = options.conf_dir - self.host_array = server_array - self.unused = options.unused - self.debug = options.debug - - def create_mount_volfile (self): - - raid_type = self.raid_type - - if self.conf_dir: - mount_fd = file ("%s/%s-%s.vol" % (self.conf_dir, - str(self.volume_name), - str(self.transport)), "w") - else: - mount_fd = file ("%s-%s.vol" % (str(self.volume_name), - str(self.transport)), "w") - - print "Generating client volfiles.. for transport '%s'" % (self.transport) - - - cmdline = string.join (sys.argv, ' ') - - mount_fd.write ("## file auto generated by %s (mount.vol)\n" % - sys.argv[0]) - mount_fd.write ("# Cmd line:\n") - mount_fd.write ("# $ %s\n\n" % cmdline) - - if raid_type is not None: - # Used for later usage - mount_fd.write ("# RAID %d\n" % raid_type) - - mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport) - subvolumes = [] - for host in self.host_dict.keys(): - i = 1 - for exports in self.host_dict[host]: - mount_fd.write ("volume %s-%s\n" % (host,i)) - mount_fd.write (" type protocol/client\n") - mount_fd.write (" option transport-type %s\n" % - self.transport) - command = "dig %s | grep '^%s'" % (host, host) - ps = subprocess.Popen(command, - shell=True, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=True) - ipaddress = host - if ps.wait() == 0: - output = ps.communicate() - ipaddress = output[0].split()[-1] - - mount_fd.write (" option remote-host %s\n" % ipaddress) - if self.transport == 'ib-verbs': - mount_fd.write (" option transport.ib-verbs.port %d\n" % - self.ib_devport) - mount_fd.write (" option transport.remote-port %d\n" % - self.gfs_ib_port) - if self.transport == 'tcp': - mount_fd.write (" option transport.socket.nodelay on\n") - mount_fd.write (" option transport.remote-port %d\n" % - self.gfs_port) - - mount_fd.write (" option remote-subvolume brick%s\n" % - i) - mount_fd.write ("end-volume\n\n") - i += 1 - - exportlist = {} - for entry in self.host_array: - node = entry.split(':')[0] - if not exportlist.has_key(node): - exportlist[node] = 1 - else: - exportlist[node] += 1 - subvolumes.append(str(node) + '-' + str(exportlist[node])) - - # Stripe section.. if given - if raid_type is 0: - max_stripe_idx = len (subvolumes) / num_stripe - stripe_idx = 0 - index = 0 - while index < max_stripe_idx: - mount_fd.write ("volume stripe-%d\n" % index) - mount_fd.write (" type cluster/stripe\n") - if self.unused: - mount_fd.write ("# option block-size 128k\n") - mount_fd.write ("# option use-xattr no\n") - - mount_fd.write (" subvolumes %s %s %s %s\n" % - (subvolumes[stripe_idx], - subvolumes[stripe_idx+1], - subvolumes[stripe_idx+2], - subvolumes[stripe_idx+3])) - mount_fd.write ("end-volume\n\n") - stripe_idx += 4 - index +=1 - - # Replicate section - if raid_type is 1: - max_mirror_idx = len (subvolumes) / num_replica - mirror_idx = 0 - index = 0 - while index < max_mirror_idx: - mount_fd.write ("volume mirror-%d\n" % index) - mount_fd.write (" type cluster/replicate\n") - mount_fd.write (" subvolumes %s %s\n" % - (subvolumes[mirror_idx], - subvolumes[mirror_idx+1])) - mount_fd.write ("end-volume\n\n") - mirror_idx += 2 - index += 1 - - # Distribute section - if raid_type is 0: - subvolumes = [] - flag = 0 - while flag < index: - subvolumes.append ("stripe-%d" % flag) - flag += 1 - if raid_type is 1: - subvolumes = [] - flag = 0 - while flag < index: - subvolumes.append ("mirror-%d" % flag) - flag += 1 - - if len (subvolumes) > 1: - mount_fd.write ("volume distribute\n") - mount_fd.write (" type cluster/distribute\n") - if self.unused: - mount_fd.write("# option unhashed-sticky-bit yes # Used for migrating data while adding new nodes\n") - mount_fd.write("# option min-free-disk 5% # Minimum free disk available on the volume\n") - - - mount_fd.write (" subvolumes %s\n" % - string.join (subvolumes,' ')) - mount_fd.write ("end-volume\n\n") - subvolumes[0] = "distribute" - - mount_fd.write ("volume writebehind\n") - mount_fd.write (" type performance/write-behind\n") - mount_fd.write (" option cache-size 4MB\n") - if self.unused: - mount_fd.write ("# option enable-trickling-writes yes # Flush final write calls when network is free\n") - mount_fd.write ("# option enable-O_SYNC yes # Enable O_SYNC for write-behind\n") - mount_fd.write ("# option disable-for-first-nbytes 1 # Disable first nbytes with very small initial writes\n") - - mount_fd.write (" subvolumes %s\n" % subvolumes[0]) - mount_fd.write ("end-volume\n\n") - - mount_fd.write ("volume readahead\n") - mount_fd.write (" type performance/read-ahead\n") - mount_fd.write (" option page-count 4\n") - - if self.unused: - mount_fd.write ("# option force-atime-update yes # force updating atimes, default off\n") - mount_fd.write (" subvolumes writebehind\n") - mount_fd.write ("end-volume\n\n") - - mount_fd.write ("volume iocache\n") - mount_fd.write (" type performance/io-cache\n") - mount_fd.write (" option cache-size %s\n" % cache_size) - mount_fd.write (" option cache-timeout 1\n") - if self.unused: - mount_fd.write ("# option priority *.html:1,abc*:2 # Priority list for iocaching files\n") - mount_fd.write (" subvolumes readahead\n") - mount_fd.write ("end-volume\n\n") - - mount_fd.write ("volume quickread\n") - mount_fd.write (" type performance/quick-read\n") - mount_fd.write (" option cache-timeout 1\n") - mount_fd.write (" option max-file-size 64kB\n") - mount_fd.write (" subvolumes iocache\n") - mount_fd.write ("end-volume\n\n") - - mount_fd.write ("volume statprefetch\n") - mount_fd.write (" type performance/stat-prefetch\n") - mount_fd.write (" subvolumes quickread\n") - mount_fd.write ("end-volume\n\n") - - return - - def create_export_volfile (self): - - cmdline = string.join (sys.argv, ' ') - - if self.conf_dir: - exp_fd = file ("%s/%s-export.vol" % - (self.conf_dir, - str(self.host + '-' + self.volume_name)),"w") - else: - exp_fd = file ("%s-export.vol" % - (str(self.host + '-' + self.volume_name)),"w") - - print "Generating server volfiles.. for server '%s'" % (self.host) - - exp_fd.write ("## file auto generated by %s (export.vol)\n" % - sys.argv[0]) - exp_fd.write ("# Cmd line:\n") - exp_fd.write ("# $ %s\n\n" % cmdline) - total_bricks = [] - i=1 - for export in self.host_dict[self.host]: - exp_fd.write ("volume posix%d\n" % i) - exp_fd.write (" type storage/posix\n") - if self.unused: - exp_fd.write("# option o-direct enable # (default: disable) boolean type only\n") - exp_fd.write("# option export-statfs-size no # (default: yes) boolean type only\n") - exp_fd.write("# option mandate-attribute off # (default: on) boolean type only\n") - exp_fd.write("# option span-devices 8 # (default: 0) integer value\n") - exp_fd.write("# option background-unlink yes # (default: no) boolean type\n") - - exp_fd.write (" option directory %s\n" % export) - exp_fd.write ("end-volume\n\n") - - exp_fd.write ("volume locks%d\n" % i) - exp_fd.write (" type features/locks\n") - if self.unused: - exp_fd.write ("# option mandatory on # Default off, used in specific applications\n") - - exp_fd.write (" subvolumes posix%d\n" % i) - exp_fd.write ("end-volume\n\n") - - exp_fd.write ("volume brick%d\n" % i) - exp_fd.write (" type performance/io-threads\n") - exp_fd.write (" option thread-count 8\n") - if self.unused: - exp_fd.write ("# option autoscaling yes # Heuristic for autoscaling threads on demand\n") - exp_fd.write ("# option min-threads 2 # min count for thread pool\n") - exp_fd.write ("# option max-threads 64 # max count for thread pool\n") - - exp_fd.write (" subvolumes locks%d\n" % i) - exp_fd.write ("end-volume\n\n") - - total_bricks.append("brick%s" % i) - i += 1 - - for transport in self.transports: - exp_fd.write ("volume server-%s\n" % transport) - exp_fd.write (" type protocol/server\n") - exp_fd.write (" option transport-type %s\n" % transport) - for brick in total_bricks: - exp_fd.write (" option auth.addr.%s.allow %s\n" % - (brick, self.auth_parameters)) - - if transport == 'ib-verbs': - exp_fd.write (" option transport.ib-verbs.listen-port %d\n" % self.gfs_ib_port) - exp_fd.write (" option transport.ib-verbs.port %d\n" % - self.ib_devport) - if transport == 'tcp': - exp_fd.write (" option transport.socket.listen-port %d\n" % self.gfs_port) - exp_fd.write (" option transport.socket.nodelay on\n") - - exp_fd.write (" subvolumes %s\n" % - string.join(total_bricks, ' ')) - exp_fd.write ("end-volume\n\n") - - return diff --git a/extras/volgen/Makefile.am b/extras/volgen/Makefile.am deleted file mode 100644 index 73b62356ed3..00000000000 --- a/extras/volgen/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ - -volgendir = $(datadir)/glusterfs -dist_volgen_DATA = CreateVolfile.py CreateBooster.py - -dist_bin_SCRIPTS = glusterfs-volgen - -CLEANFILES = - diff --git a/extras/volgen/glusterfs-volgen.in b/extras/volgen/glusterfs-volgen.in deleted file mode 100755 index bd19d122773..00000000000 --- a/extras/volgen/glusterfs-volgen.in +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/python - -import getopt, sys, os, string -from optparse import OptionParser,OptionGroup,make_option - -prefix = "@prefix@" - -if not (prefix + "/share/glusterfs") in sys.path: - sys.path.append(prefix + "/share/glusterfs") - -from CreateVolfile import * -from CreateBooster import * - -def check_duplicate_entry(args): - """Check duplicate entries in incoming arguments""" - _tmp = [] - for server in args: - if server not in _tmp: - _tmp.append (server) - else: - print "Duplicate arguments detected (%s)" % server - raise ValueError - - return - -def args2dict(args): - - keyvalue = {} - for arg in args: - if int(arg.find(':')) == -1: - continue - first = arg.split(':')[0] - keyvalue[first] = [] - - for arg in args: - if int(arg.find(':')) == -1: - continue - first = arg.split(':')[0] - if arg.split(':')[1] not in keyvalue[first]: - if arg.split(':')[1][0] != '/': - print "Absolute export path required for %s" % arg - raise ValueError - keyvalue[first].append (arg.split(':')[1]) - - return keyvalue - -def args2array(args): - - array = [] - - for arg in args: - if int(arg.find(':')) == -1: - continue - array.append(arg) - - return array - -def generate_volume_files (): - - num_stripe = 4 - num_replica = 2 - - usage_str = "%prog: -n <VOLUMENAME> -t <TRANSPORT> -p <NUMBER> -a <AUTH> -r <TYPE>" - version_str = "%prog 3.0" - desc_str = "A tool to generate volume files for GlusterFS." - - parse = OptionParser(usage=usage_str, version=version_str, description=desc_str) - - # Basic option list - group = OptionGroup(parse, "Basic Options") - group.add_option("-n", "--name", dest="volume_name", - help="<volume-name>") - group.add_option("-t", "--transport", dest="transport_type", - default="tcp", help="tcp,ib-verbs default: tcp") - group.add_option("-p", "--port", type="int", - dest="port", default=6996, - help="<port> number") - group.add_option("-a", "--auth", dest="auth_param", default="*", - help="comma seperated ip range") - group.add_option("-r", "--raid", type="int", dest="raid_type", - help="0|1") - group.add_option("--nfs", action="store_true", dest="need_nfs", - default=False, help="booster nfs reexport") - group.add_option("--cifs", action="store_true", dest="need_cifs", - default=False, help="booster cifs reexport"), - parse.add_option_group(group) - - # CIFS option list - group = OptionGroup(parse, "CIFS Options") - group.add_option("--username", dest="cifs_username", - default="gluster", help="<cifs_username>") - group.add_option("--guest", action="store_true", - dest="enable_guest", default=False, - help="enable guest access") - parse.add_option_group(group) - # NFS option list - - # Advanced option list - group = OptionGroup(parse, "Advanced Options") - group.add_option("--ibdev", type="int", dest="ib_dev", - default=1, help="Infiniband device number <N>") - group.add_option("-c", "--conf-dir", dest="conf_dir", - help="output directory for volume files"), - parse.add_option_group(group) - - group = OptionGroup(parse, "Extra Options") - group.add_option("--unused", action="store_true", - dest="unused", default=False, - help="enable unused options") - group.add_option("--debug", action="store_true", - dest="debug", default=False, - help="add all debug modules to volumes") - - parse.add_option_group(group) - - (options, args) = parse.parse_args() - - if options.volume_name is None: - print "Error: volume name is mandatory, please provide volume name" - raise ValueError - - if options.transport_type: - transports = options.transport_type.split(',') - for transport in transports: - if (transport != "tcp" and transport != "ib-verbs"): - print "Error: --transport: option " + transport + \ - " is not valid transport type" - raise ValueError - - if options.raid_type: - if (options.raid_type != 1 and options.raid_type != 0): - print "Error: --raid: option " + str(options.raid_type) + " is not valid raid type" - raise ValueError - - - check_duplicate_entry(args) - - server_dict = {} - - server_dict = args2dict(args) - - server_array = args2array(args) - - if len (server_dict.keys()) == 0: - print "Error: no servers provided, please provide atleast one server" - raise ValueError - - if options.raid_type == 1: - if (len(server_array) % num_replica) != 0: - print "raid type (%d) and number of volumes (%d) invalid" % (options.raid_type, - len(server_array)) - raise ValueError - - if options.raid_type == 0: - if (len(server_array) % num_stripe) != 0: - print "raid type (%d) and number of volumes (%d) invalid" % (options.raid_type, - len(server_array)) - raise ValueError - - for server in server_dict.keys(): - create_exp = CreateVolfile (server_dict, server, - None, transports, - options, None) - try: - create_exp.create_export_volfile () - except IOError, (errno, strerror): - print "Got %s creating server volfiles for %s" % (strerror, server) - - for transport in transports: - create_mnt = CreateVolfile (server_dict, None, - transport, transports, - options, server_array) - try: - create_mnt.create_mount_volfile () - except IOError, (errno, strerror): - print "Got %s creating client volfiles for transport '%s'" % (strerror, transport) - - - create_booster = CreateBooster (options, transports) - try: - create_booster.configure_booster () - except IOError, (errno, strerror): - print "Got %s creating booster configuration" % (strerror) - -def main (): - - try: - generate_volume_files() - except ValueError: - sys.exit(1) - - -main() diff --git a/extras/who-wrote-glusterfs/gitdm.aliases b/extras/who-wrote-glusterfs/gitdm.aliases new file mode 100644 index 00000000000..901c12418e3 --- /dev/null +++ b/extras/who-wrote-glusterfs/gitdm.aliases @@ -0,0 +1,58 @@ +# +# This is the email aliases file, mapping secondary addresses onto a single, +# canonical address. This file should probably match the contents of .mailmap +# in the root of the git repository. +# +# Format: <alias> <real> + +amar@gluster.com amarts@redhat.com +amar@del.gluster.com amarts@redhat.com +avati@amp.gluster.com avati@redhat.com +avati@blackhole.gluster.com avati@redhat.com +avati@dev.gluster.com avati@redhat.com +avati@gluster.com avati@redhat.com +wheelear@gmail.com awheeler@redhat.com +anush@gluster.com ashetty@redhat.com +csaba@gluster.com csaba@redhat.com +csaba@lowlife.hu csaba@redhat.com +csaba@zresearch.com csaba@redhat.com +gd@samba.org gd@redhat.com +harsha@gluster.com fharshav@redhat.com +harsha@zresearch.com fharshav@redhat.com +harsha@dev.gluster.com fharshav@redhat.com +harsha@harshavardhana.net fharshav@redhat.com +jclift@redhat.com jclift@gluster.org +kkeithle@linux.keithley.org kkeithle@redhat.com +kkeithle@f16node1.kkeithle.usersys.redhat.com kkeithle@redhat.com +kaushal@gluster.com kaushal@redhat.com +kaushikbv@gluster.com kbudiger@redhat.com +krishna@gluster.com ksriniva@redhat.com +krishna@zresearch.com ksriniva@redhat.com +krishna@guest-laptop ksriniva@redhat.com +kp@gluster.com kparthas@redhat.com +me@louiszuckerman.com louiszuckerman@gmail.com +msvbhat@gmail.com vbhat@redhat.com +nullpai@gmail.com ppai@redhat.com +vishwanath@gluster.com vbhat@redhat.com +obnox@samba.org madam@redhat.com +oleksandr@natalenko.name o.natalenko@lanet.ua +patrick@puiterwijk.org puiterwijk@fedoraproject.org +pavan@dev.gluster.com pavan@gluster.com +zaitcev@yahoo.com zaitcev@kotori.zaitcev.us +pranithk@gluster.com pkarampu@redhat.com +raghavendrabhat@gluster.com raghavendra@redhat.com +raghavendra@gluster.com rgowdapp@redhat.com +raghavendra@zresearch.com rgowdapp@redhat.com +rahulcssjce@gmail.com rahulcs@redhat.com +rajesh@gluster.com rajesh@redhat.com +rajesh.amaravathi@gmail.com rajesh@redhat.com +root@ravi2.(none) ravishankar@redhat.com +sabansal@localhost.localdomain sabansal@redhat.com +shehjart@zresearch.com shehjart@gluster.com +venky@gluster.com vshankar@redhat.com +vijay@gluster.com vbellur@redhat.com +vijay@dev.gluster.com vbellur@redhat.com +vijaykumar.koppad@gmail.com vkoppad@redhat.com +vikas@zresearch.com vikas@gluster.com +shishirng@gluster.com sgowda@redhat.com +potatogim@potatogim.net potatogim@gluesys.com diff --git a/extras/who-wrote-glusterfs/gitdm.config b/extras/who-wrote-glusterfs/gitdm.config new file mode 100644 index 00000000000..e1ff2bd5bda --- /dev/null +++ b/extras/who-wrote-glusterfs/gitdm.config @@ -0,0 +1,8 @@ +# +# This is the gitdm configuration file for GlusterFS. +# See the gitdm.config in the gitdm repositofy for additional options and +# comments. +# + +EmailAliases gitdm.aliases +EmailMap gitdm.domain-map diff --git a/extras/who-wrote-glusterfs/gitdm.domain-map b/extras/who-wrote-glusterfs/gitdm.domain-map new file mode 100644 index 00000000000..7cd2bbd605b --- /dev/null +++ b/extras/who-wrote-glusterfs/gitdm.domain-map @@ -0,0 +1,29 @@ +# +# Here is a set of mappings of domain names onto employer names. +# +active.by ActiveCloud +appeartv.com Appear TV +cern.ch CERN +cmss.chinamobile.com China Mobile(Suzhou) Software Technology +datalab.es DataLab S.L. +fb.com Facebook +fedoraproject.org Fedora Project +gluster.com Red Hat +gmail.com (personal contributions) +gooddata.com GoodData +hastexo.com hastexo +horde.com (personal contributions) +ibm.com IBM +io.com IO +lanet.ua Lanet Network +linbit.com LINBIT +nectec.or.th NECTEC +netbsd.org NetBSD +netdirect.ca Net Direct +nokia.com Nokia +redhat.com Red Hat +stepping-stone.ch stepping stone GmbH +xtaotech.com XTAO Co. +yahoo.in (personal contributions) +zresearch.com Red Hat +gluesys.com Gluesys diff --git a/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh b/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh new file mode 100755 index 00000000000..70d5964c576 --- /dev/null +++ b/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Gather statistics on "Who wrote GlusterFS". The idea comes from the excellent +# articles on http://lwn.net/ named "Who wrote <linux-version>?". +# +# gitdm comes from git://git.lwn.net/gitdm.git by Jonathan Corbet. +# +# Confguration files used: +# - gitdm.config: main configuration file, pointing to the others +# - gitdm.aliases: merge users with different emailaddresses into one +# - gitdm.domain-map: map domain names from emailaddresses to companies +# + +DIRNAME=$(dirname $0) + +GITDM_REPO=git://git.lwn.net/gitdm.git +GITDM_DIR=${DIRNAME}/gitdm +GITDM_CMD="python ${GITDM_DIR}/gitdm" + +error() +{ + local ret=${?} + printf "${@}\n" > /dev/stderr + return ${ret} +} + +check_gitdm() +{ + if [ ! -e "${GITDM_DIR}/gitdm" ] + then + git clone --quiet ${GITDM_REPO} ${DIRNAME}/gitdm + fi +} + +# The first argument is the revision-range (see 'git rev-list --help'). +# REV can be empty, and the statistics will be calculated over the whole +# current branch. +REV=${1} +shift +# all remaining options are passed to gitdm, see the gitdm script for an +# explanation of the accepted options. +GITDM_OPTS=${@} + +if ! check_gitdm +then + error "Could not find 'gitdm', exiting..." + exit 1 +fi + +git log --numstat -M ${REV} | ${GITDM_CMD} -b ${DIRNAME} -n ${GITDM_OPTS} |