diff options
Diffstat (limited to 'extras')
148 files changed, 14098 insertions, 3631 deletions
diff --git a/extras/LinuxRPM/Makefile.am b/extras/LinuxRPM/Makefile.am index b82c65f02c4..f02853798c0 100644 --- a/extras/LinuxRPM/Makefile.am +++ b/extras/LinuxRPM/Makefile.am @@ -6,15 +6,22 @@ GFS_TAR = ../../glusterfs-$(VERSION).tar.gz all: @echo "To build RPMS run 'make glusterrpms'" -.PHONY: glusterrpms prep srcrpm testsrpm clean +.PHONY: glusterrpms glusterrpms_without_autogen +.PHONY: autogen prep srcrpm testsrpm clean -glusterrpms: prep srcrpm rpms +glusterrpms: autogen glusterrpms_without_autogen + +glusterrpms_without_autogen: prep srcrpm rpms -rm -rf rpmbuild +autogen: + cd ../.. && \ + rm -rf autom4te.cache && \ + ./autogen.sh && \ + ./configure --enable-gnfs --with-previous-options + prep: - if [ ! -e $(GFS_TAR) ]; then \ - $(MAKE) -C ../.. dist; \ - fi + $(MAKE) -C ../.. dist; -mkdir -p rpmbuild/BUILD -mkdir -p rpmbuild/SPECS -mkdir -p rpmbuild/RPMS @@ -29,7 +36,7 @@ srcrpm: mv rpmbuild/SRPMS/* . rpms: - rpmbuild --define '_topdir $(shell pwd)/rpmbuild' -bb rpmbuild/SPECS/glusterfs.spec + rpmbuild --define '_topdir $(shell pwd)/rpmbuild' --with gnfs -bb rpmbuild/SPECS/glusterfs.spec mv rpmbuild/RPMS/*/* . # EPEL-5 does not like new versions of rpmbuild and requires some diff --git a/extras/LinuxRPM/make_glusterrpms b/extras/LinuxRPM/make_glusterrpms new file mode 100755 index 00000000000..3156af97870 --- /dev/null +++ b/extras/LinuxRPM/make_glusterrpms @@ -0,0 +1,9 @@ +#!/bin/sh + +( + cd $(realpath $(dirname $0))/../.. || exit 1 + rm -rf autom4te.cache + ./autogen.sh || exit 1 + ./configure --with-previous-options || exit 1 +) +make -C $(realpath $(dirname $0)) glusterrpms_without_autogen diff --git a/extras/Makefile.am b/extras/Makefile.am index b3405552675..983f014cca6 100644 --- a/extras/Makefile.am +++ b/extras/Makefile.am @@ -1,26 +1,77 @@ +addonexecdir = $(GLUSTERFS_LIBEXECDIR) +addonexec_SCRIPTS = +if WITH_SERVER +addonexec_SCRIPTS += peer_add_secret_pub +if USE_SYSTEMD +addonexec_SCRIPTS += mount-shared-storage.sh +endif +endif + EditorModedir = $(docdir) EditorMode_DATA = glusterfs-mode.el glusterfs.vim SUBDIRS = init.d systemd benchmarking hook-scripts $(OCF_SUBDIR) LinuxRPM \ - $(GEOREP_EXTRAS_SUBDIR) + $(GEOREP_EXTRAS_SUBDIR) snap_scheduler firewalld cliutils python \ + ganesha confdir = $(sysconfdir)/glusterfs +if WITH_SERVER conf_DATA = glusterfs-logrotate gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf \ - logger.conf.example glusterfs-georep-logrotate group-virt.example + logger.conf.example glusterfs-georep-logrotate group-virt.example \ + group-metadata-cache group-gluster-block group-nl-cache \ + group-db-workload group-distributed-virt group-samba +endif voldir = $(sysconfdir)/glusterfs -vol_DATA = glusterd.vol +vol_DATA = thin-arbiter/thin-arbiter.vol +if WITH_SERVER +vol_DATA += glusterd.vol +endif + scriptsdir = $(datadir)/glusterfs/scripts -scripts_DATA = post-upgrade-script-for-quota.sh pre-upgrade-script-for-quota.sh +scripts_SCRIPTS = thin-arbiter/setup-thin-arbiter.sh +if WITH_SERVER +scripts_SCRIPTS += post-upgrade-script-for-quota.sh \ + pre-upgrade-script-for-quota.sh stop-all-gluster-processes.sh +if USE_SYSTEMD +scripts_SCRIPTS += control-cpu-load.sh +scripts_SCRIPTS += control-mem.sh +endif +endif -EXTRA_DIST = $(conf_DATA) specgen.scm glusterfs-mode.el glusterfs.vim \ - migrate-unify-to-distribute.sh backend-xattr-sanitize.sh backend-cleanup.sh \ - disk_usage_sync.sh clear_xattrs.sh glusterd-sysconfig glusterd.vol \ - post-upgrade-script-for-quota.sh pre-upgrade-script-for-quota.sh \ - command-completion/gluster.bash command-completion/Makefile \ - command-completion/README +EXTRA_DIST = glusterfs-logrotate gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf \ + logger.conf.example glusterfs-georep-logrotate group-virt.example \ + group-metadata-cache group-gluster-block group-nl-cache \ + group-db-workload group-samba specgen.scm glusterfs-mode.el glusterfs.vim \ + migrate-unify-to-distribute.sh backend-xattr-sanitize.sh \ + backend-cleanup.sh disk_usage_sync.sh clear_xattrs.sh \ + glusterd-sysconfig glusterd.vol post-upgrade-script-for-quota.sh \ + pre-upgrade-script-for-quota.sh command-completion/gluster.bash \ + command-completion/Makefile command-completion/README \ + stop-all-gluster-processes.sh clang-checker.sh mount-shared-storage.sh \ + control-cpu-load.sh control-mem.sh group-distributed-virt \ + thin-arbiter/thin-arbiter.vol thin-arbiter/setup-thin-arbiter.sh +if WITH_SERVER install-data-local: - $(MKDIR_P) $(DESTDIR)$(GLUSTERD_WORKDIR)/groups + if [ -n "$(tmpfilesdir)" ]; then \ + $(mkdir_p) $(DESTDIR)$(tmpfilesdir); \ + $(INSTALL_DATA) run-gluster.tmpfiles \ + $(DESTDIR)$(tmpfilesdir)/gluster.conf; \ + fi + $(mkdir_p) $(DESTDIR)$(GLUSTERD_WORKDIR)/groups $(INSTALL_DATA) $(top_srcdir)/extras/group-virt.example \ $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/virt + $(INSTALL_DATA) $(top_srcdir)/extras/group-metadata-cache \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/metadata-cache + $(INSTALL_DATA) $(top_srcdir)/extras/group-gluster-block \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/gluster-block + $(INSTALL_DATA) $(top_srcdir)/extras/group-nl-cache \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/nl-cache + $(INSTALL_DATA) $(top_srcdir)/extras/group-db-workload \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/db-workload + $(INSTALL_DATA) $(top_srcdir)/extras/group-distributed-virt \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/distributed-virt + $(INSTALL_DATA) $(top_srcdir)/extras/group-samba \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/samba +endif diff --git a/extras/backend-cleanup.sh b/extras/backend-cleanup.sh index dc504860b73..cb0a1d8871f 100644 --- a/extras/backend-cleanup.sh +++ b/extras/backend-cleanup.sh @@ -17,7 +17,7 @@ export_directory="/export/glusterfs" clean_dir() { # Clean the 'link' files on backend - find "${export_directory}" -type f -perm +01000 -exec rm -v '{}' \; + find "${export_directory}" -type f -perm /01000 -exec rm -v '{}' \; } main() diff --git a/extras/benchmarking/glfs-bm.c b/extras/benchmarking/glfs-bm.c index dc717f33c16..f7f5873f84d 100644 --- a/extras/benchmarking/glfs-bm.c +++ b/extras/benchmarking/glfs-bm.c @@ -25,365 +25,338 @@ #include <sys/time.h> struct state { - char need_op_write:1; - char need_op_read:1; + char need_op_write : 1; + char need_op_read : 1; - char need_iface_fileio:1; - char need_iface_xattr:1; + char need_iface_fileio : 1; + char need_iface_xattr : 1; - char need_mode_posix:1; + char need_mode_posix : 1; - char prefix[512]; - long int count; + char prefix[512]; + long int count; - size_t block_size; + size_t block_size; - char *specfile; + char *specfile; - long int io_size; + long int io_size; }; - -#define MEASURE(func, arg) measure (func, #func, arg) - +#define MEASURE(func, arg) measure(func, #func, arg) void -tv_difference (struct timeval *tv_stop, - struct timeval *tv_start, - struct timeval *tv_diff) +tv_difference(struct timeval *tv_stop, struct timeval *tv_start, + struct timeval *tv_diff) { - if (tv_stop->tv_usec < tv_start->tv_usec) { - tv_diff->tv_usec = (tv_stop->tv_usec + 1000000) - tv_start->tv_usec; - tv_diff->tv_sec = (tv_stop->tv_sec - 1 - tv_start->tv_sec); - } else { - tv_diff->tv_usec = tv_stop->tv_usec - tv_start->tv_usec; - tv_diff->tv_sec = tv_stop->tv_sec - tv_start->tv_sec; - } + if (tv_stop->tv_usec < tv_start->tv_usec) { + tv_diff->tv_usec = (tv_stop->tv_usec + 1000000) - tv_start->tv_usec; + tv_diff->tv_sec = (tv_stop->tv_sec - 1 - tv_start->tv_sec); + } else { + tv_diff->tv_usec = tv_stop->tv_usec - tv_start->tv_usec; + tv_diff->tv_sec = tv_stop->tv_sec - tv_start->tv_sec; + } } - void -measure (int (*func)(struct state *state), - char *func_name, struct state *state) +measure(int (*func)(struct state *state), char *func_name, struct state *state) { - struct timeval tv_start, tv_stop, tv_diff; - state->io_size = 0; - long int count; + struct timeval tv_start, tv_stop, tv_diff; + state->io_size = 0; + long int count; - gettimeofday (&tv_start, NULL); - count = func (state); - gettimeofday (&tv_stop, NULL); + gettimeofday(&tv_start, NULL); + count = func(state); + gettimeofday(&tv_stop, NULL); - tv_difference (&tv_stop, &tv_start, &tv_diff); + tv_difference(&tv_stop, &tv_start, &tv_diff); - fprintf (stdout, "%s: count=%ld, size=%ld, time=%ld:%ld\n", - func_name, count, state->io_size, - tv_diff.tv_sec, tv_diff.tv_usec); + fprintf(stdout, "%s: count=%ld, size=%ld, time=%ld:%ld\n", func_name, count, + state->io_size, tv_diff.tv_sec, tv_diff.tv_usec); } - static error_t -parse_opts (int key, char *arg, - struct argp_state *_state) +parse_opts(int key, char *arg, struct argp_state *_state) { - struct state *state = _state->input; + struct state *state = _state->input; - switch (key) - { + switch (key) { case 'o': - if (strcasecmp (arg, "read") == 0) { - state->need_op_write = 0; - state->need_op_read = 1; - } else if (strcasecmp (arg, "write") == 0) { - state->need_op_write = 1; - state->need_op_read = 0; - } else if (strcasecmp (arg, "both") == 0) { - state->need_op_write = 1; - state->need_op_read = 1; - } else { - fprintf (stderr, "unknown op: %s\n", arg); - return -1; - } - break; + if (strcasecmp(arg, "read") == 0) { + state->need_op_write = 0; + state->need_op_read = 1; + } else if (strcasecmp(arg, "write") == 0) { + state->need_op_write = 1; + state->need_op_read = 0; + } else if (strcasecmp(arg, "both") == 0) { + state->need_op_write = 1; + state->need_op_read = 1; + } else { + fprintf(stderr, "unknown op: %s\n", arg); + return -1; + } + break; case 'i': - if (strcasecmp (arg, "fileio") == 0) { - state->need_iface_fileio = 1; - state->need_iface_xattr = 0; - } else if (strcasecmp (arg, "xattr") == 0) { - state->need_iface_fileio = 0; - state->need_iface_xattr = 1; - } else if (strcasecmp (arg, "both") == 0) { - state->need_iface_fileio = 1; - state->need_iface_xattr = 1; - } else { - fprintf (stderr, "unknown interface: %s\n", arg); - return -1; - } - break; - case 'b': - { - size_t block_size = atoi (arg); - if (!block_size) { - fprintf (stderr, "incorrect size: %s\n", arg); - return -1; - } - state->block_size = block_size; - } - break; + if (strcasecmp(arg, "fileio") == 0) { + state->need_iface_fileio = 1; + state->need_iface_xattr = 0; + } else if (strcasecmp(arg, "xattr") == 0) { + state->need_iface_fileio = 0; + state->need_iface_xattr = 1; + } else if (strcasecmp(arg, "both") == 0) { + state->need_iface_fileio = 1; + state->need_iface_xattr = 1; + } else { + fprintf(stderr, "unknown interface: %s\n", arg); + return -1; + } + break; + case 'b': { + size_t block_size = atoi(arg); + if (!block_size) { + fprintf(stderr, "incorrect size: %s\n", arg); + return -1; + } + state->block_size = block_size; + } break; case 's': - state->specfile = strdup (arg); - break; + state->specfile = strdup(arg); + break; case 'p': - fprintf (stderr, "using prefix: %s\n", arg); - strncpy (state->prefix, arg, 512); - break; - case 'c': - { - long count = atol (arg); - if (!count) { - fprintf (stderr, "incorrect count: %s\n", arg); - return -1; - } - state->count = count; - } - break; + fprintf(stderr, "using prefix: %s\n", arg); + strncpy(state->prefix, arg, 512); + break; + case 'c': { + long count = atol(arg); + if (!count) { + fprintf(stderr, "incorrect count: %s\n", arg); + return -1; + } + state->count = count; + } break; case ARGP_KEY_NO_ARGS: - break; + break; case ARGP_KEY_ARG: - break; - } + break; + } - return 0; + return 0; } int -do_mode_posix_iface_fileio_write (struct state *state) +do_mode_posix_iface_fileio_write(struct state *state) { - long int i; - int ret = -1; - char block[state->block_size]; - - for (i=0; i<state->count; i++) { - int fd = -1; - char filename[512]; - - sprintf (filename, "%s.%06ld", state->prefix, i); - - fd = open (filename, O_CREAT|O_WRONLY, 00600); - if (fd == -1) { - fprintf (stderr, "open(%s) => %s\n", filename, strerror (errno)); - break; - } - ret = write (fd, block, state->block_size); - if (ret != state->block_size) { - fprintf (stderr, "write (%s) => %d/%s\n", filename, ret, - strerror (errno)); - close (fd); - break; - } - close (fd); - state->io_size += ret; + long int i; + int ret = -1; + char block[state->block_size]; + + for (i = 0; i < state->count; i++) { + int fd = -1; + char filename[512]; + + sprintf(filename, "%s.%06ld", state->prefix, i); + + fd = open(filename, O_CREAT | O_WRONLY, 00600); + if (fd == -1) { + fprintf(stderr, "open(%s) => %s\n", filename, strerror(errno)); + break; + } + ret = write(fd, block, state->block_size); + if (ret != state->block_size) { + fprintf(stderr, "write (%s) => %d/%s\n", filename, ret, + strerror(errno)); + close(fd); + break; } + close(fd); + state->io_size += ret; + } - return i; + return i; } - int -do_mode_posix_iface_fileio_read (struct state *state) +do_mode_posix_iface_fileio_read(struct state *state) { - long int i; - int ret = -1; - char block[state->block_size]; - - for (i=0; i<state->count; i++) { - int fd = -1; - char filename[512]; - - sprintf (filename, "%s.%06ld", state->prefix, i); - - fd = open (filename, O_RDONLY); - if (fd == -1) { - fprintf (stderr, "open(%s) => %s\n", filename, strerror (errno)); - break; - } - ret = read (fd, block, state->block_size); - if (ret == -1) { - fprintf (stderr, "read(%s) => %d/%s\n", filename, ret, strerror (errno)); - close (fd); - break; - } - close (fd); - state->io_size += ret; + long int i; + int ret = -1; + char block[state->block_size]; + + for (i = 0; i < state->count; i++) { + int fd = -1; + char filename[512]; + + sprintf(filename, "%s.%06ld", state->prefix, i); + + fd = open(filename, O_RDONLY); + if (fd == -1) { + fprintf(stderr, "open(%s) => %s\n", filename, strerror(errno)); + break; } + ret = read(fd, block, state->block_size); + if (ret == -1) { + fprintf(stderr, "read(%s) => %d/%s\n", filename, ret, + strerror(errno)); + close(fd); + break; + } + close(fd); + state->io_size += ret; + } - return i; + return i; } - int -do_mode_posix_iface_fileio (struct state *state) +do_mode_posix_iface_fileio(struct state *state) { - if (state->need_op_write) - MEASURE (do_mode_posix_iface_fileio_write, state); + if (state->need_op_write) + MEASURE(do_mode_posix_iface_fileio_write, state); - if (state->need_op_read) - MEASURE (do_mode_posix_iface_fileio_read, state); + if (state->need_op_read) + MEASURE(do_mode_posix_iface_fileio_read, state); - return 0; + return 0; } - int -do_mode_posix_iface_xattr_write (struct state *state) +do_mode_posix_iface_xattr_write(struct state *state) { - long int i; - int ret = -1; - char block[state->block_size]; - char *dname = NULL, *dirc = NULL; - char *bname = NULL, *basec = NULL; - - dirc = strdup (state->prefix); - basec = strdup (state->prefix); - dname = dirname (dirc); - bname = basename (basec); - - for (i=0; i<state->count; i++) { - char key[512]; - - sprintf (key, "glusterfs.file.%s.%06ld", bname, i); - - ret = lsetxattr (dname, key, block, state->block_size, 0); - - if (ret != 0) { - fprintf (stderr, "lsetxattr (%s, %s, %p) => %s\n", - dname, key, block, strerror (errno)); - break; - } - state->io_size += state->block_size; + long int i; + int ret = -1; + char block[state->block_size]; + char *dname = NULL, *dirc = NULL; + char *bname = NULL, *basec = NULL; + + dirc = strdup(state->prefix); + basec = strdup(state->prefix); + dname = dirname(dirc); + bname = basename(basec); + + for (i = 0; i < state->count; i++) { + char key[512]; + + sprintf(key, "glusterfs.file.%s.%06ld", bname, i); + + ret = lsetxattr(dname, key, block, state->block_size, 0); + + if (ret != 0) { + fprintf(stderr, "lsetxattr (%s, %s, %p) => %s\n", dname, key, block, + strerror(errno)); + break; } + state->io_size += state->block_size; + } - free (dirc); - free (basec); + free(dirc); + free(basec); - return i; + return i; } - int -do_mode_posix_iface_xattr_read (struct state *state) +do_mode_posix_iface_xattr_read(struct state *state) { - long int i; - int ret = -1; - char block[state->block_size]; - char *dname = NULL, *dirc = NULL; - char *bname = NULL, *basec = NULL; - - dirc = strdup (state->prefix); - basec = strdup (state->prefix); - dname = dirname (dirc); - bname = basename (basec); - - for (i=0; i<state->count; i++) { - char key[512]; - - sprintf (key, "glusterfs.file.%s.%06ld", bname, i); - - ret = lgetxattr (dname, key, block, state->block_size); - - if (ret < 0) { - fprintf (stderr, "lgetxattr (%s, %s, %p) => %s\n", - dname, key, block, strerror (errno)); - break; - } - state->io_size += ret; + long int i; + int ret = -1; + char block[state->block_size]; + char *dname = NULL, *dirc = NULL; + char *bname = NULL, *basec = NULL; + + dirc = strdup(state->prefix); + basec = strdup(state->prefix); + dname = dirname(dirc); + bname = basename(basec); + + for (i = 0; i < state->count; i++) { + char key[512]; + + sprintf(key, "glusterfs.file.%s.%06ld", bname, i); + + ret = lgetxattr(dname, key, block, state->block_size); + + if (ret < 0) { + fprintf(stderr, "lgetxattr (%s, %s, %p) => %s\n", dname, key, block, + strerror(errno)); + break; } + state->io_size += ret; + } - return i; + return i; } - int -do_mode_posix_iface_xattr (struct state *state) +do_mode_posix_iface_xattr(struct state *state) { - if (state->need_op_write) - MEASURE (do_mode_posix_iface_xattr_write, state); + if (state->need_op_write) + MEASURE(do_mode_posix_iface_xattr_write, state); - if (state->need_op_read) - MEASURE (do_mode_posix_iface_xattr_read, state); + if (state->need_op_read) + MEASURE(do_mode_posix_iface_xattr_read, state); - return 0; + return 0; } int -do_mode_posix (struct state *state) +do_mode_posix(struct state *state) { - if (state->need_iface_fileio) - do_mode_posix_iface_fileio (state); + if (state->need_iface_fileio) + do_mode_posix_iface_fileio(state); - if (state->need_iface_xattr) - do_mode_posix_iface_xattr (state); + if (state->need_iface_xattr) + do_mode_posix_iface_xattr(state); - return 0; + return 0; } - int -do_actions (struct state *state) +do_actions(struct state *state) { - if (state->need_mode_posix) - do_mode_posix (state); + if (state->need_mode_posix) + do_mode_posix(state); - return 0; + return 0; } static struct argp_option options[] = { - {"op", 'o', "OPERATIONS", 0, - "WRITE|READ|BOTH - defaults to BOTH"}, - {"iface", 'i', "INTERFACE", 0, - "FILEIO|XATTR|BOTH - defaults to FILEIO"}, - {"block", 'b', "BLOCKSIZE", 0, - "<NUM> - defaults to 4096"}, - {"specfile", 's', "SPECFILE", 0, - "absolute path to specfile"}, - {"prefix", 'p', "PREFIX", 0, - "filename prefix"}, - {"count", 'c', "COUNT", 0, - "number of files"}, - {0, 0, 0, 0, 0} -}; + {"op", 'o', "OPERATIONS", 0, "WRITE|READ|BOTH - defaults to BOTH"}, + {"iface", 'i', "INTERFACE", 0, "FILEIO|XATTR|BOTH - defaults to FILEIO"}, + {"block", 'b', "BLOCKSIZE", 0, "<NUM> - defaults to 4096"}, + {"specfile", 's', "SPECFILE", 0, "absolute path to specfile"}, + {"prefix", 'p', "PREFIX", 0, "filename prefix"}, + {"count", 'c', "COUNT", 0, "number of files"}, + {0, 0, 0, 0, 0}}; -static struct argp argp = { - options, - parse_opts, - "tool", - "tool to benchmark small file performance" -}; +static struct argp argp = {options, parse_opts, "tool", + "tool to benchmark small file performance"}; int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - struct state state = {0, }; + struct state state = { + 0, + }; - state.need_op_write = 1; - state.need_op_read = 1; + state.need_op_write = 1; + state.need_op_read = 1; - state.need_iface_fileio = 1; - state.need_iface_xattr = 0; + state.need_iface_fileio = 1; + state.need_iface_xattr = 0; - state.need_mode_posix = 1; + state.need_mode_posix = 1; - state.block_size = 4096; + state.block_size = 4096; - strcpy (state.prefix, "tmpfile"); - state.count = 1048576; + strcpy(state.prefix, "tmpfile"); + state.count = 1048576; - if (argp_parse (&argp, argc, argv, 0, 0, &state) != 0) { - fprintf (stderr, "argp_parse() failed\n"); - return 1; - } + if (argp_parse(&argp, argc, argv, 0, 0, &state) != 0) { + fprintf(stderr, "argp_parse() failed\n"); + return 1; + } - do_actions (&state); + do_actions(&state); - return 0; + return 0; } diff --git a/extras/benchmarking/rdd.c b/extras/benchmarking/rdd.c index a667c6a1d65..efc9d342a37 100644 --- a/extras/benchmarking/rdd.c +++ b/extras/benchmarking/rdd.c @@ -20,633 +20,586 @@ #define TWO_POWER(power) (2UL << (power)) -#define RDD_INTEGER_VALUE ((TWO_POWER ((sizeof (int) * 8))) - 1) +#define RDD_INTEGER_VALUE ((TWO_POWER((sizeof(int) * 8))) - 1) #ifndef UNIX_PATH_MAX #define UNIX_PATH_MAX 108 #endif #define UNIT_KB 1024ULL -#define UNIT_MB UNIT_KB*1024ULL -#define UNIT_GB UNIT_MB*1024ULL -#define UNIT_TB UNIT_GB*1024ULL -#define UNIT_PB UNIT_TB*1024ULL +#define UNIT_MB UNIT_KB * 1024ULL +#define UNIT_GB UNIT_MB * 1024ULL +#define UNIT_TB UNIT_GB * 1024ULL +#define UNIT_PB UNIT_TB * 1024ULL -#define UNIT_KB_STRING "KB" -#define UNIT_MB_STRING "MB" -#define UNIT_GB_STRING "GB" -#define UNIT_TB_STRING "TB" -#define UNIT_PB_STRING "PB" +#define UNIT_KB_STRING "KB" +#define UNIT_MB_STRING "MB" +#define UNIT_GB_STRING "GB" +#define UNIT_TB_STRING "TB" +#define UNIT_PB_STRING "PB" struct rdd_file { - char path[UNIX_PATH_MAX]; - struct stat st; - int fd; + char path[UNIX_PATH_MAX]; + struct stat st; + int fd; }; struct rdd_config { - long iters; - long max_ops_per_seq; - size_t max_bs; - size_t min_bs; - int thread_count; - pthread_t *threads; - pthread_barrier_t barrier; - pthread_mutex_t lock; - struct rdd_file in_file; - struct rdd_file out_file; - ssize_t file_size; + long iters; + long max_ops_per_seq; + size_t max_bs; + size_t min_bs; + int thread_count; + pthread_t *threads; + pthread_barrier_t barrier; + pthread_mutex_t lock; + struct rdd_file in_file; + struct rdd_file out_file; + ssize_t file_size; }; static struct rdd_config rdd_config; enum rdd_keys { - RDD_MIN_BS_KEY = 1, - RDD_MAX_BS_KEY, + RDD_MIN_BS_KEY = 1, + RDD_MAX_BS_KEY, }; static error_t -rdd_parse_opts (int key, char *arg, - struct argp_state *_state) +rdd_parse_opts(int key, char *arg, struct argp_state *_state) { - switch (key) { - case 'o': - { - int len = 0; - len = strlen (arg); - if (len > UNIX_PATH_MAX) { - fprintf (stderr, "output file name too long (%s)\n", - arg); - return -1; - } - - strncpy (rdd_config.out_file.path, arg, len); - } - break; - - case 'i': - { - int len = 0; - len = strlen (arg); - if (len > UNIX_PATH_MAX) { - fprintf (stderr, "input file name too long (%s)\n", - arg); - return -1; - } - - strncpy (rdd_config.in_file.path, arg, len); - rdd_config.in_file.path[len] = '\0'; - } - break; - - case 'f': - { - char *tmp = NULL; - unsigned long long fs = 0; - if (string2bytesize (arg, &fs) == -1) { - fprintf (stderr, "invalid argument for file size " - "(%s)\n", arg); - return -1; - } - - rdd_config.file_size = fs; - } - break; - - case RDD_MIN_BS_KEY: - { - char *tmp = NULL; - long bs = 0; - bs = strtol (arg, &tmp, 10); - if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) { - fprintf (stderr, "invalid argument for minimum block" - "size (%s)\n", arg); - return -1; - } - - rdd_config.min_bs = bs; - } - break; - - case RDD_MAX_BS_KEY: - { - char *tmp = NULL; - long bs = 0; - bs = strtol (arg, &tmp, 10); - if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) { - fprintf (stderr, "invalid argument for maximum block" - "size (%s)\n", arg); - return -1; - } - - rdd_config.max_bs = bs; - } - break; - - case 'r': - { - char *tmp = NULL; - long iters = 0; - iters = strtol (arg, &tmp, 10); - if ((iters == LONG_MAX) || - (iters == LONG_MIN) || - (tmp && *tmp)) { - fprintf (stderr, "invalid argument for iterations" - "(%s)\n", arg); - return -1; - } - - rdd_config.iters = iters; - } - break; - - case 'm': - { - char *tmp = NULL; - long max_ops = 0; - max_ops = strtol (arg, &tmp, 10); - if ((max_ops == LONG_MAX) || - (max_ops == LONG_MIN) || - (tmp && *tmp)) { - fprintf (stderr, "invalid argument for max-ops" - "(%s)\n", arg); - return -1; - } + switch (key) { + case 'o': { + int len = 0; + len = strlen(arg); + if (len > UNIX_PATH_MAX) { + fprintf(stderr, "output file name too long (%s)\n", arg); + return -1; + } - rdd_config.max_ops_per_seq = max_ops; - } - break; + strncpy(rdd_config.out_file.path, arg, len); + } break; - case 't': - { - char *tmp = NULL; - long threads = 0; - threads = strtol (arg, &tmp, 10); - if ((threads == LONG_MAX) || - (threads == LONG_MIN) || - (tmp && *tmp)) { - fprintf (stderr, "invalid argument for thread count" - "(%s)\n", arg); - return -1; - } + case 'i': { + int len = 0; + len = strlen(arg); + if (len > UNIX_PATH_MAX) { + fprintf(stderr, "input file name too long (%s)\n", arg); + return -1; + } + + strncpy(rdd_config.in_file.path, arg, len); + rdd_config.in_file.path[len] = '\0'; + } break; + + case 'f': { + char *tmp = NULL; + unsigned long long fs = 0; + if (string2bytesize(arg, &fs) == -1) { + fprintf(stderr, + "invalid argument for file size " + "(%s)\n", + arg); + return -1; + } + + rdd_config.file_size = fs; + } break; + + case RDD_MIN_BS_KEY: { + char *tmp = NULL; + long bs = 0; + bs = strtol(arg, &tmp, 10); + if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for minimum block" + "size (%s)\n", + arg); + return -1; + } + + rdd_config.min_bs = bs; + } break; + + case RDD_MAX_BS_KEY: { + char *tmp = NULL; + long bs = 0; + bs = strtol(arg, &tmp, 10); + if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for maximum block" + "size (%s)\n", + arg); + return -1; + } + + rdd_config.max_bs = bs; + } break; + + case 'r': { + char *tmp = NULL; + long iters = 0; + iters = strtol(arg, &tmp, 10); + if ((iters == LONG_MAX) || (iters == LONG_MIN) || (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for iterations" + "(%s)\n", + arg); + return -1; + } + + rdd_config.iters = iters; + } break; + + case 'm': { + char *tmp = NULL; + long max_ops = 0; + max_ops = strtol(arg, &tmp, 10); + if ((max_ops == LONG_MAX) || (max_ops == LONG_MIN) || + (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for max-ops" + "(%s)\n", + arg); + return -1; + } + + rdd_config.max_ops_per_seq = max_ops; + } break; + + case 't': { + char *tmp = NULL; + long threads = 0; + threads = strtol(arg, &tmp, 10); + if ((threads == LONG_MAX) || (threads == LONG_MIN) || + (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for thread count" + "(%s)\n", + arg); + return -1; + } - rdd_config.thread_count = threads; - } - break; + rdd_config.thread_count = threads; + } break; case ARGP_KEY_NO_ARGS: - break; + break; case ARGP_KEY_ARG: - break; + break; case ARGP_KEY_END: - if (_state->argc == 1) { - argp_usage (_state); - } + if (_state->argc == 1) { + argp_usage(_state); + } + } - } - - return 0; + return 0; } int -string2bytesize (const char *str, unsigned long long *n) +string2bytesize(const char *str, unsigned long long *n) { - unsigned long long value = 0ULL; - char *tail = NULL; - int old_errno = 0; - const char *s = NULL; - - if (str == NULL || n == NULL) - { - errno = EINVAL; - return -1; - } - - for (s = str; *s != '\0'; s++) - { - if (isspace (*s)) - { - continue; - } - if (*s == '-') - { - return -1; - } - break; + unsigned long long value = 0ULL; + char *tail = NULL; + int old_errno = 0; + const char *s = NULL; + + if (str == NULL || n == NULL) { + errno = EINVAL; + return -1; + } + + for (s = str; *s != '\0'; s++) { + if (isspace(*s)) { + continue; } - - old_errno = errno; - errno = 0; - value = strtoull (str, &tail, 10); - - if (errno == ERANGE || errno == EINVAL) - { - return -1; + if (*s == '-') { + return -1; } - - if (errno == 0) - { - errno = old_errno; + break; + } + + old_errno = errno; + errno = 0; + value = strtoull(str, &tail, 10); + + if (errno == ERANGE || errno == EINVAL) { + return -1; + } + + if (errno == 0) { + errno = old_errno; + } + + if (tail[0] != '\0') { + if (strcasecmp(tail, UNIT_KB_STRING) == 0) { + value *= UNIT_KB; + } else if (strcasecmp(tail, UNIT_MB_STRING) == 0) { + value *= UNIT_MB; + } else if (strcasecmp(tail, UNIT_GB_STRING) == 0) { + value *= UNIT_GB; + } else if (strcasecmp(tail, UNIT_TB_STRING) == 0) { + value *= UNIT_TB; + } else if (strcasecmp(tail, UNIT_PB_STRING) == 0) { + value *= UNIT_PB; } - if (tail[0] != '\0') - { - if (strcasecmp (tail, UNIT_KB_STRING) == 0) - { - value *= UNIT_KB; - } - else if (strcasecmp (tail, UNIT_MB_STRING) == 0) - { - value *= UNIT_MB; - } - else if (strcasecmp (tail, UNIT_GB_STRING) == 0) - { - value *= UNIT_GB; - } - else if (strcasecmp (tail, UNIT_TB_STRING) == 0) - { - value *= UNIT_TB; - } - else if (strcasecmp (tail, UNIT_PB_STRING) == 0) - { - value *= UNIT_PB; - } - - else - { - return -1; - } + else { + return -1; } + } - *n = value; + *n = value; - return 0; + return 0; } static struct argp_option rdd_options[] = { - {"if", 'i', "INPUT_FILE", 0, "input-file"}, - {"of", 'o', "OUTPUT_FILE", 0, "output-file"}, - {"threads", 't', "COUNT", 0, "number of threads to spawn (defaults to 2)"}, - {"min-bs", RDD_MIN_BS_KEY, "MIN_BLOCK_SIZE", 0, - "Minimum block size in bytes (defaults to 1024)"}, - {"max-bs", RDD_MAX_BS_KEY, "MAX_BLOCK_SIZE", 0, - "Maximum block size in bytes (defaults to 4096)"}, - {"iters", 'r', "ITERS", 0, - "Number of read-write sequences (defaults to 1000000)"}, - {"max-ops", 'm', "MAXOPS", 0, - "maximum number of read-writes to be performed in a sequence (defaults to 1)"}, - {"file-size", 'f', "FILESIZE", 0, - "the size of the file which will be created and upon it I/O will be done" - " (defaults to 100MB"}, - {0, 0, 0, 0, 0} -}; + {"if", 'i', "INPUT_FILE", 0, "input-file"}, + {"of", 'o', "OUTPUT_FILE", 0, "output-file"}, + {"threads", 't', "COUNT", 0, "number of threads to spawn (defaults to 2)"}, + {"min-bs", RDD_MIN_BS_KEY, "MIN_BLOCK_SIZE", 0, + "Minimum block size in bytes (defaults to 1024)"}, + {"max-bs", RDD_MAX_BS_KEY, "MAX_BLOCK_SIZE", 0, + "Maximum block size in bytes (defaults to 4096)"}, + {"iters", 'r', "ITERS", 0, + "Number of read-write sequences (defaults to 1000000)"}, + {"max-ops", 'm', "MAXOPS", 0, + "maximum number of read-writes to be performed in a sequence (defaults to " + "1)"}, + {"file-size", 'f', "FILESIZE", 0, + "the size of the file which will be created and upon it I/O will be done" + " (defaults to 100MB"}, + {0, 0, 0, 0, 0}}; static struct argp argp = { - rdd_options, - rdd_parse_opts, - "", - "random dd - tool to do a sequence of random block-sized continuous" - "read writes starting at a random offset" -}; - + rdd_options, rdd_parse_opts, "", + "random dd - tool to do a sequence of random block-sized continuous" + "read writes starting at a random offset"}; static void -rdd_default_config (void) +rdd_default_config(void) { - char *tmp_path = "rdd.in"; - - rdd_config.thread_count = 2; - rdd_config.iters = 1000000; - rdd_config.max_bs = 4096; - rdd_config.min_bs = 1024; - rdd_config.in_file.fd = rdd_config.out_file.fd = -1; - rdd_config.max_ops_per_seq = 1; - strncpy (rdd_config.in_file.path, tmp_path, strlen (tmp_path)); - rdd_config.file_size = 104857600; - - return; + char *tmp_path = "rdd.in"; + + rdd_config.thread_count = 2; + rdd_config.iters = 1000000; + rdd_config.max_bs = 4096; + rdd_config.min_bs = 1024; + rdd_config.in_file.fd = rdd_config.out_file.fd = -1; + rdd_config.max_ops_per_seq = 1; + strncpy(rdd_config.in_file.path, tmp_path, strlen(tmp_path)); + rdd_config.file_size = 104857600; + + return; } - static char -rdd_valid_config (void) +rdd_valid_config(void) { - char ret = 1; - int fd = -1; + char ret = 1; + int fd = -1; - fd = open (rdd_config.in_file.path, O_RDONLY); - if (fd == -1 && (errno != ENOENT)) { - fprintf (stderr, "open: (%s)", strerror (errno)); - ret = 0; - goto out; - } - close (fd); - - if (rdd_config.min_bs > rdd_config.max_bs) { - fprintf (stderr, "minimum blocksize %ld is greater than the " - "maximum blocksize %ld", rdd_config.min_bs, - rdd_config.max_bs); - ret = 0; - goto out; - } + fd = open(rdd_config.in_file.path, O_RDONLY); + if (fd == -1 && (errno != ENOENT)) { + fprintf(stderr, "open: (%s)", strerror(errno)); + ret = 0; + goto out; + } + close(fd); + + if (rdd_config.min_bs > rdd_config.max_bs) { + fprintf(stderr, + "minimum blocksize %ld is greater than the " + "maximum blocksize %ld", + rdd_config.min_bs, rdd_config.max_bs); + ret = 0; + goto out; + } - if (strlen (rdd_config.out_file.path) == 0) { - sprintf (rdd_config.out_file.path, "%s.rddout", - rdd_config.in_file.path); - } + if (strlen(rdd_config.out_file.path) == 0) { + sprintf(rdd_config.out_file.path, "%s.rddout", rdd_config.in_file.path); + } out: - return ret; + return ret; } - static void * -rdd_read_write (void *arg) +rdd_read_write(void *arg) { - int i = 0, ret = 0; - size_t bs = 0; - off_t offset = 0; - long rand = 0; - long max_ops = 0; - char *buf = NULL; - - buf = calloc (1, rdd_config.max_bs); - if (!buf) { - fprintf (stderr, "calloc failed (%s)\n", strerror (errno)); + int i = 0, ret = 0; + size_t bs = 0; + off_t offset = 0; + long rand = 0; + long max_ops = 0; + char *buf = NULL; + + buf = calloc(1, rdd_config.max_bs); + if (!buf) { + fprintf(stderr, "calloc failed (%s)\n", strerror(errno)); + ret = -1; + goto out; + } + + for (i = 0; i < rdd_config.iters; i++) { + pthread_mutex_lock(&rdd_config.lock); + { + int bytes = 0; + rand = random(); + + if (rdd_config.min_bs == rdd_config.max_bs) { + bs = rdd_config.max_bs; + } else { + bs = rdd_config.min_bs + + (rand % (rdd_config.max_bs - rdd_config.min_bs)); + } + + offset = rand % rdd_config.in_file.st.st_size; + max_ops = rand % rdd_config.max_ops_per_seq; + if (!max_ops) { + max_ops++; + } + + ret = lseek(rdd_config.in_file.fd, offset, SEEK_SET); + if (ret != offset) { + fprintf(stderr, "lseek failed (%s)\n", strerror(errno)); ret = -1; - goto out; - } + goto unlock; + } - for (i = 0; i < rdd_config.iters; i++) - { - pthread_mutex_lock (&rdd_config.lock); - { - int bytes = 0; - rand = random (); - - if (rdd_config.min_bs == rdd_config.max_bs) { - bs = rdd_config.max_bs; - } else { - bs = rdd_config.min_bs + - (rand % - (rdd_config.max_bs - - rdd_config.min_bs)); - } - - offset = rand % rdd_config.in_file.st.st_size; - max_ops = rand % rdd_config.max_ops_per_seq; - if (!max_ops) { - max_ops ++; - } - - ret = lseek (rdd_config.in_file.fd, offset, SEEK_SET); - if (ret != offset) { - fprintf (stderr, "lseek failed (%s)\n", - strerror (errno)); - ret = -1; - goto unlock; - } - - ret = lseek (rdd_config.out_file.fd, offset, SEEK_SET); - if (ret != offset) { - fprintf (stderr, "lseek failed (%s)\n", - strerror (errno)); - ret = -1; - goto unlock; - } - - while (max_ops--) - { - bytes = read (rdd_config.in_file.fd, buf, bs); - if (!bytes) { - break; - } - - if (bytes == -1) { - fprintf (stderr, "read failed (%s)\n", - strerror (errno)); - ret = -1; - goto unlock; - } - - if (write (rdd_config.out_file.fd, buf, bytes) - != bytes) { - fprintf (stderr, "write failed (%s)\n", - strerror (errno)); - ret = -1; - goto unlock; - } - } + ret = lseek(rdd_config.out_file.fd, offset, SEEK_SET); + if (ret != offset) { + fprintf(stderr, "lseek failed (%s)\n", strerror(errno)); + ret = -1; + goto unlock; + } + + while (max_ops--) { + bytes = read(rdd_config.in_file.fd, buf, bs); + if (!bytes) { + break; } - unlock: - pthread_mutex_unlock (&rdd_config.lock); - if (ret == -1) { - goto out; + + if (bytes == -1) { + fprintf(stderr, "read failed (%s)\n", strerror(errno)); + ret = -1; + goto unlock; + } + + if (write(rdd_config.out_file.fd, buf, bytes) != bytes) { + fprintf(stderr, "write failed (%s)\n", strerror(errno)); + ret = -1; + goto unlock; } - ret = 0; + } } + unlock: + pthread_mutex_unlock(&rdd_config.lock); + if (ret == -1) { + goto out; + } + ret = 0; + } out: - free (buf); - pthread_barrier_wait (&rdd_config.barrier); + free(buf); + pthread_barrier_wait(&rdd_config.barrier); - return NULL; + return NULL; } static void -cleanup (void) +cleanup(void) { - close (rdd_config.in_file.fd); - close (rdd_config.out_file.fd); - rdd_config.in_file.fd = rdd_config.out_file.fd = -1; + close(rdd_config.in_file.fd); + close(rdd_config.out_file.fd); + rdd_config.in_file.fd = rdd_config.out_file.fd = -1; } static int -check_and_create (void) +check_and_create(void) { - int ret = -1; - char buf[4096] = {0,}; - struct stat stbuf = {0,}; - int fd[2] = {-1,}; - size_t total_size = -1; - - total_size = rdd_config.file_size; - - ret = stat (rdd_config.in_file.path, &stbuf); - if (ret == -1 && (errno != ENOENT)) + int ret = -1; + char buf[4096] = { + 0, + }; + struct stat stbuf = { + 0, + }; + int fd[2] = { + -1, + }; + size_t total_size = -1; + + total_size = rdd_config.file_size; + + ret = stat(rdd_config.in_file.path, &stbuf); + if (ret == -1 && (errno != ENOENT)) + goto out; + + fd[1] = open(rdd_config.in_file.path, O_CREAT | O_WRONLY | O_TRUNC); + if (fd[1] == -1) + goto out; + + fd[0] = open("/dev/urandom", O_RDONLY); + if (fd[0] == -1) + goto out; + + while (total_size > 0) { + if (total_size >= 4096) { + ret = read(fd[0], buf, 4096); + if (ret == -1) goto out; - - fd[1] = open (rdd_config.in_file.path, O_CREAT | O_WRONLY | O_TRUNC); - if (fd[1] == -1) + ret = write(fd[1], buf, 4096); + if (ret == -1) goto out; - - fd[0] = open ("/dev/urandom", O_RDONLY); - if (fd[0] == -1) + total_size = total_size - 4096; + } else { + ret = read(fd[0], buf, total_size); + if (ret == -1) goto out; - - while (total_size > 0) { - if (total_size >= 4096) { - ret = read (fd[0], buf, 4096); - if (ret == -1) - goto out; - ret = write (fd[1], buf, 4096); - if (ret == -1) - goto out; - total_size = total_size - 4096; - } else { - ret = read (fd[0], buf, total_size); - if (ret == -1) - goto out; - ret = write (fd[1], buf, total_size); - if (ret == -1) - goto out; - total_size = total_size - total_size; - } - + ret = write(fd[1], buf, total_size); + if (ret == -1) + goto out; + total_size = total_size - total_size; } + } - ret = 0; + ret = 0; out: - if (fd[0] > 0) - close (fd[0]); - if (fd[1] > 0) - close (fd[1]); - return ret; + if (fd[0] > 0) + close(fd[0]); + if (fd[1] > 0) + close(fd[1]); + return ret; } static int -rdd_spawn_threads (void) +rdd_spawn_threads(void) { - int i = 0, ret = -1, fd = -1; - char buf[4096]; - - ret = check_and_create (); - if (ret == -1) - goto out; - - fd = open (rdd_config.in_file.path, O_RDONLY); - if (fd < 0) { - fprintf (stderr, "cannot open %s (%s)\n", - rdd_config.in_file.path, strerror (errno)); - ret = -1; - goto out; - } - ret = fstat (fd, &rdd_config.in_file.st); - if (ret != 0) { - close (fd); - fprintf (stderr, "cannot stat %s (%s)\n", - rdd_config.in_file.path, strerror (errno)); - ret = -1; - goto out; - } - rdd_config.in_file.fd = fd; - - fd = open (rdd_config.out_file.path, O_WRONLY | O_CREAT | O_TRUNC, - S_IRWXU | S_IROTH); - if (fd < 0) { - close (rdd_config.in_file.fd); - rdd_config.in_file.fd = -1; - fprintf (stderr, "cannot open %s (%s)\n", - rdd_config.out_file.path, strerror (errno)); - ret = -1; - goto out; - } - rdd_config.out_file.fd = fd; - - while ((ret = read (rdd_config.in_file.fd, buf, 4096)) > 0) { - if (write (rdd_config.out_file.fd, buf, ret) != ret) { - fprintf (stderr, "write failed (%s)\n", - strerror (errno)); - cleanup (); - ret = -1; - goto out; - } - } - - rdd_config.threads = calloc (rdd_config.thread_count, - sizeof (pthread_t)); - if (rdd_config.threads == NULL) { - fprintf (stderr, "calloc() failed (%s)\n", strerror (errno)); - - ret = -1; - cleanup (); - goto out; - } - - ret = pthread_barrier_init (&rdd_config.barrier, NULL, - rdd_config.thread_count + 1); - if (ret != 0) { - fprintf (stderr, "pthread_barrier_init() failed (%s)\n", - strerror (ret)); - - free (rdd_config.threads); - cleanup (); - ret = -1; - goto out; + int i = 0, ret = -1, fd = -1; + char buf[4096]; + + ret = check_and_create(); + if (ret == -1) + goto out; + + fd = open(rdd_config.in_file.path, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "cannot open %s (%s)\n", rdd_config.in_file.path, + strerror(errno)); + ret = -1; + goto out; + } + ret = fstat(fd, &rdd_config.in_file.st); + if (ret != 0) { + close(fd); + fprintf(stderr, "cannot stat %s (%s)\n", rdd_config.in_file.path, + strerror(errno)); + ret = -1; + goto out; + } + rdd_config.in_file.fd = fd; + + fd = open(rdd_config.out_file.path, O_WRONLY | O_CREAT | O_TRUNC, + S_IRWXU | S_IROTH); + if (fd < 0) { + close(rdd_config.in_file.fd); + rdd_config.in_file.fd = -1; + fprintf(stderr, "cannot open %s (%s)\n", rdd_config.out_file.path, + strerror(errno)); + ret = -1; + goto out; + } + rdd_config.out_file.fd = fd; + + while ((ret = read(rdd_config.in_file.fd, buf, 4096)) > 0) { + if (write(rdd_config.out_file.fd, buf, ret) != ret) { + fprintf(stderr, "write failed (%s)\n", strerror(errno)); + cleanup(); + ret = -1; + goto out; } - - ret = pthread_mutex_init (&rdd_config.lock, NULL); + } + + rdd_config.threads = calloc(rdd_config.thread_count, sizeof(pthread_t)); + if (rdd_config.threads == NULL) { + fprintf(stderr, "calloc() failed (%s)\n", strerror(errno)); + + ret = -1; + cleanup(); + goto out; + } + + ret = pthread_barrier_init(&rdd_config.barrier, NULL, + rdd_config.thread_count + 1); + if (ret != 0) { + fprintf(stderr, "pthread_barrier_init() failed (%s)\n", strerror(ret)); + + free(rdd_config.threads); + cleanup(); + ret = -1; + goto out; + } + + ret = pthread_mutex_init(&rdd_config.lock, NULL); + if (ret != 0) { + fprintf(stderr, "pthread_mutex_init() failed (%s)\n", strerror(ret)); + + free(rdd_config.threads); + pthread_barrier_destroy(&rdd_config.barrier); + cleanup(); + ret = -1; + goto out; + } + + for (i = 0; i < rdd_config.thread_count; i++) { + ret = pthread_create(&rdd_config.threads[i], NULL, rdd_read_write, + NULL); if (ret != 0) { - fprintf (stderr, "pthread_mutex_init() failed (%s)\n", - strerror (ret)); - - free (rdd_config.threads); - pthread_barrier_destroy (&rdd_config.barrier); - cleanup (); - ret = -1; - goto out; - } - - for (i = 0; i < rdd_config.thread_count; i++) - { - ret = pthread_create (&rdd_config.threads[i], NULL, - rdd_read_write, NULL); - if (ret != 0) { - fprintf (stderr, "pthread_create failed (%s)\n", - strerror (errno)); - exit (1); - } + fprintf(stderr, "pthread_create failed (%s)\n", strerror(errno)); + exit(1); } + } out: - return ret; + return ret; } static void -rdd_wait_for_completion (void) +rdd_wait_for_completion(void) { - pthread_barrier_wait (&rdd_config.barrier); + pthread_barrier_wait(&rdd_config.barrier); } - int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - int ret = -1; + int ret = -1; - rdd_default_config (); + rdd_default_config(); - ret = argp_parse (&argp, argc, argv, 0, 0, NULL); - if (ret != 0) { - ret = -1; - fprintf (stderr, "%s: argp_parse() failed\n", argv[0]); - goto err; - } + ret = argp_parse(&argp, argc, argv, 0, 0, NULL); + if (ret != 0) { + ret = -1; + fprintf(stderr, "%s: argp_parse() failed\n", argv[0]); + goto err; + } - if (!rdd_valid_config ()) { - ret = -1; - fprintf (stderr, "%s: configuration validation failed\n", - argv[0]); - goto err; - } + if (!rdd_valid_config()) { + ret = -1; + fprintf(stderr, "%s: configuration validation failed\n", argv[0]); + goto err; + } - ret = rdd_spawn_threads (); - if (ret != 0) { - fprintf (stderr, "%s: spawning threads failed\n", argv[0]); - goto err; - } + ret = rdd_spawn_threads(); + if (ret != 0) { + fprintf(stderr, "%s: spawning threads failed\n", argv[0]); + goto err; + } - rdd_wait_for_completion (); + rdd_wait_for_completion(); err: - return ret; + return ret; } diff --git a/extras/clang-checker.sh b/extras/clang-checker.sh new file mode 100755 index 00000000000..4909d3adfcd --- /dev/null +++ b/extras/clang-checker.sh @@ -0,0 +1,301 @@ +#!/usr/bin/env bash +#******************************************************************************* +# * +# Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> * +# This file is part of GlusterFS. * +# * +# This file is licensed to you under your choice of the GNU Lesser * +# General Public License, version 3 or any later version (LGPLv3 or * +# later), or the GNU General Public License, version 2 (GPLv2), in all * +# cases as published by the Free Software Foundation. * +#------------------------------------------------------------------------------* +# * +# clang-checker.sh: This script runs clang static analyzer using 'scan-build' * +# a perl wrapper. After you commit your patch i.e. right * +# before executing rfc.sh in order to push the patch to * +# repository, it is recommended that you execute * +# clang-checker.sh to perform static analysis inorder to * +# check if there are any possible bugs in the code. * +# * +# This script performs the static analysis with and * +# without HEAD commit, it runs the analyzer only in the * +# directory where changes have been made and finally diff's * +# the number of bugs using both cases (i.e. with and * +# without your commit) and gives a summary, which explain's * +# about the eligibility of your patch. * +# * +# Usage: $ cd $PATH_TO_GLUSTERFS * +# $ extras/clang-checker.sh (or) $ make clang-check * +# * +# Author: Prasanna Kumar Kalever <prasanna.kalever@redhat.com> * +# * +#******************************************************************************* + +REPORTS_DIR=$(pwd) +BASELINE_DIR=${REPORTS_DIR}/baseline +BRESULTS_DIR=${BASELINE_DIR}/results +BBACKUP_DIR=${BASELINE_DIR}/backup +TARGET_DIR=${REPORTS_DIR}/target +TRESULTS_DIR=${TARGET_DIR}/results +TBACKUP_DIR=${TARGET_DIR}/backup + +declare -A DICT_B +declare -A DICT_T +declare -A ARR +declare -A FILES + +function identify_changes () { + MODIFIED_DATA=$(git show --name-status --oneline | tail -n +2) + FLAG=0 + for i in ${MODIFIED_DATA}; do + if [ $FLAG -eq 1 ]; then + ARR+="$(dirname $i) "; + FLAG=0; + fi + if [ $i = 'M' ] || [ $i = 'A' ]; then + FLAG=1; + fi + done + + MODIFIED_DIR=$(echo "${ARR[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ') + for i in $MODIFIED_DIR; do + # run only in directories which has Makefile + if [ $(find ./$i -iname "makefile*" | wc -c) -gt 0 ]; then + # skip 'doc' and '.'(top) directory + if [ "xx$i" != "xxdoc" ] && [ "xx$i" != "xx." ]; then + FILES+="$i " + fi + fi + done + if [ -z $FILES ]; then + echo "Probably no changes made to 'c' files" + exit; + fi +} + +function check_prerequisites () { + if ! type "clang" 2> /dev/null; then + echo -e "\ntry after installing clang and scan-build..." + echo "useful info at http://clang-analyzer.llvm.org/installation.html\n" + echo -e "hint: 'dnf -y install clang-analyzer.noarch'\n" + exit 1; + elif ! type "scan-build" 2> /dev/null; then + echo -e "\ntry after installing scan-build..." + echo "useful info at http://clang-analyzer.llvm.org/installation.html" + echo -e "hint: 'dnf -y install clang-analyzer.noarch'\n" + exit 1; + fi +} + +function force_terminate () { + echo -e "\nreceived a signal to force terminate ..\n" + git am --abort 2> /dev/null + git am ${PATCH_NAME} + rm -f ${REPORTS_DIR}/${PATCH_NAME} + exit 1; +} + +function run_scanbuild () { + local CLANG=$(which clang) + local SCAN_BUILD=$(which scan-build) + local ORIG_COMMIT=$(git rev-parse --verify HEAD^) + PATCH_NAME=$(git format-patch $ORIG_COMMIT) + + echo -e "\n| Performing clang analysis on:" \ + "$(git log --pretty=format:"%h - '%s' by %an" -1) ... |\n" + echo -e "Changes are identified in '${FILES[@]}' directorie[s]\n" + + if [ -d "${BRESULTS_DIR}" ]; then + mkdir -p ${BBACKUP_DIR} ${TBACKUP_DIR} + mv ${BRESULTS_DIR} \ + ${BBACKUP_DIR}/results_$(ls -l ${BBACKUP_DIR} | wc -l) + mv ${TRESULTS_DIR} \ + ${TBACKUP_DIR}/results_$(ls -l ${TBACKUP_DIR} | wc -l) + fi + for DIR in ${FILES[@]}; do + mkdir -p ${BRESULTS_DIR}/$(echo ${DIR} | sed 's/\//_/g') + mkdir -p ${TRESULTS_DIR}/$(echo ${DIR} | sed 's/\//_/g') + done + # get nproc info + case $(uname -s) in + 'Linux') + local NPROC=$(getconf _NPROCESSORS_ONLN) + ;; + 'NetBSD') + local NPROC=$(getconf NPROCESSORS_ONLN) + ;; + esac + + trap force_terminate INT TERM QUIT EXIT + + git reset --hard HEAD^ + + # build complete source code for sake of dependencies + echo -e "\n# make -j${NPROC} ..." + make -j${NPROC} 1>/dev/null + + for DIR in ${FILES[@]}; do + if [ $(find ./$i -iname "makefile*" | wc -c) -gt 0 ]; then + make clean -C ${DIR} 1>/dev/null + echo -e "\n| Analyzing ${DIR} without commit ... |\n" + # run only in directory where changes are made + ${SCAN_BUILD} -o ${BRESULTS_DIR}/$(echo ${DIR} | sed 's/\//_/g') \ + --use-analyzer=${CLANG} make -j${NPROC} -C ${DIR} + fi + done + + echo -e "\n| Analyzing without commit complete ... |\n" + + git am ${PATCH_NAME} + trap - INT TERM QUIT EXIT + + # In case commit has changes to configure stuff ? + echo -e "\n# make clean ..." + make clean 1>/dev/null + echo -e "\n# ./autogen.sh && ./configure --with-previous-options ..." + ${REPORTS_DIR}/autogen.sh 2>/dev/null + ${REPORTS_DIR}/configure --with-previous-options 1>/dev/null + echo -e "\n# make -j${NPROC} ..." + make -j${NPROC} 1>/dev/null + + for DIR in ${FILES[@]}; do + if [ $(find ./$i -iname "makefile*" | wc -c) -gt 0 ]; then + make clean -C ${DIR} 1>/dev/null + echo -e "\n| Analyzing ${DIR} with commit ... |\n" + # run only in directory where changes are made + ${SCAN_BUILD} -o ${TRESULTS_DIR}/$(echo ${DIR} | sed 's/\//_/g') \ + --use-analyzer=${CLANG} make -j${NPROC} -C ${DIR} + fi + done + + echo -e "\n| Analyzing with commit complete ... |\n" + + rm -f ${REPORTS_DIR}/${PATCH_NAME} +} + +function count_for_baseline () { + for DIR in ${FILES[@]}; do + HTMLS_DIR=${BRESULTS_DIR}/$(echo ${DIR} | + sed 's/\//_/g')/$(ls ${BRESULTS_DIR}/$(echo ${DIR} | + sed 's/\//_/g')/); + + local NAMES_OF_BUGS_B=$(grep -n "SUMM_DESC" ${HTMLS_DIR}/index.html | + cut -d"<" -f3 | cut -d">" -f2 | + sed 's/[^a-zA-Z0]/_/g' | tr '\n' ' ') + local NO_OF_BUGS_B=$(grep -n "SUMM_DESC" ${HTMLS_DIR}/index.html | + cut -d"<" -f5 | cut -d">" -f2 | tr '\n' ' ') + local count_B=0; + + read -a BUG_NAME_B <<<$NAMES_OF_BUGS_B + read -a BUG_COUNT_B <<<$NO_OF_BUGS_B + for i in ${BUG_NAME_B[@]}; + do + if [ ! -z ${DICT_B[$i]} ]; then + DICT_B[$i]=$(expr ${BUG_COUNT_B[count_B]} + ${DICT_B[$i]}); + else + DICT_B+=([$i]=${BUG_COUNT_B[count_B]}); + fi + count_B=$(expr $count_B + 1) + done + done + + echo -e "\nBASELINE BUGS LIST (before applying patch):" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + for key_B in ${!DICT_B[@]}; do + echo "${key_B} --> ${DICT_B[${key_B}]}" | sed 's/_/ /g' | tr -s ' ' + done +} + +function count_for_target () { + for DIR in ${FILES[@]}; do + HTMLS_DIR=${TRESULTS_DIR}/$(echo ${DIR} | + sed 's/\//_/g')/$(ls ${TRESULTS_DIR}/$(echo ${DIR} | + sed 's/\//_/g')/); + + local NAME_OF_BUGS_T=$(grep -n "SUMM_DESC" ${HTMLS_DIR}/index.html | + cut -d"<" -f3 | cut -d">" -f2 | + sed 's/[^a-zA-Z0]/_/g'| tr '\n' ' ') + local NO_OF_BUGS_T=$(grep -n "SUMM_DESC" ${HTMLS_DIR}/index.html | + cut -d"<" -f5 | cut -d">" -f2 | tr '\n' ' ') + local count_T=0; + + read -a BUG_NAME_T <<<$NAME_OF_BUGS_T + read -a BUG_COUNT_T <<<$NO_OF_BUGS_T + + for i in ${BUG_NAME_T[@]}; + do + if [ ! -z ${DICT_T[$i]} ]; then + DICT_T[$i]=$(expr ${BUG_COUNT_T[count_T]} + ${DICT_T[$i]}); + else + DICT_T+=([$i]=${BUG_COUNT_T[count_T]}); + fi + count_T=$(expr $count_T + 1) + done + done + + echo -e "\nTARGET BUGS LIST (after applying patch):" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + for key_T in ${!DICT_T[@]}; do + echo "${key_T} --> ${DICT_T[${key_T}]}" | sed 's/_/ /g' | tr -s ' ' + done +} + +function array_contains () { + local SEEKING=$1; shift + local IN=1 + for ELEMENT; do + if [[ $ELEMENT == $SEEKING ]]; then + IN=0 + break + fi + done + return $IN +} + +function main () { + echo -e "\n================ Clang analyzer in progress ================\n" + check_prerequisites + identify_changes + run_scanbuild + clear + count_for_baseline + count_for_target + echo -e "\nSUMMARY OF CLANG-ANALYZER:" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" + + FLAG=0 + for BUG in ${!DICT_T[@]}; do + array_contains $BUG "${!DICT_B[@]}" + if [ $? -eq 1 ]; then + echo "New ${DICT_T[${BUG}]} Bug[s] introduced: $(echo $BUG | + sed 's/_/ /g' | + tr -s ' ')" + FLAG=1 + else + if [ ${BUG} != "All_Bugs" ]; then + if [ ${DICT_B[${BUG}]} -lt \ + ${DICT_T[${BUG}]} ]; then + echo "Extra $(expr ${DICT_T[${BUG}]} - \ + ${DICT_B[${BUG}]}) Bug[s] Introduced in: $(echo $BUG | + sed 's/_/ /g' | tr -s ' ')" + FLAG=1 + fi + fi + fi + done + + echo + if [ $FLAG -eq 0 ]; then + echo -e "Patch Value given by Clang analyzer '+1'\n" + else + echo -e "Patch Value given by Clang analyzer '-1'\n" + fi + echo -e "\nExplore complete results at:" + find ${BRESULTS_DIR}/ -iname "index.html" + find ${TRESULTS_DIR}/ -iname "index.html" + echo -e "\n================= Done with Clang Analysis =================\n" + + exit ${FLAG} +} + +main diff --git a/extras/cliutils/Makefile.am b/extras/cliutils/Makefile.am new file mode 100644 index 00000000000..7039703e275 --- /dev/null +++ b/extras/cliutils/Makefile.am @@ -0,0 +1,4 @@ +EXTRA_DIST= cliutils.py __init__.py + +cliutilsdir = @BUILD_PYTHON_SITE_PACKAGES@/gluster/cliutils +cliutils_PYTHON = cliutils.py __init__.py diff --git a/extras/cliutils/README.md b/extras/cliutils/README.md new file mode 100644 index 00000000000..309beb1ca25 --- /dev/null +++ b/extras/cliutils/README.md @@ -0,0 +1,233 @@ +# CLI utility for creating Cluster aware CLI tools for Gluster +cliutils is a Python library which provides wrapper around `gluster system:: +execute` command to extend the functionalities of Gluster. + +Example use cases: +- Start a service in all peer nodes of Cluster +- Collect the status of a service from all peer nodes +- Collect the config values from each peer nodes and display latest + config based on version. +- Copy a file present in GLUSTERD_WORKDIR from one peer node to all + other peer nodes.(Geo-replication create push-pem is using this to + distribute the SSH public keys from all master nodes to all slave + nodes) +- Generate pem keys in all peer nodes and collect all the public keys + to one place(Geo-replication gsec_create is doing this) +- Provide Config sync CLIs for new features like `gluster-eventsapi`, + `gluster-restapi`, `gluster-mountbroker` etc. + +## Introduction + +If a executable file present in `$GLUSTER_LIBEXEC` directory in all +peer nodes(Filename startswith `peer_`) then it can be executed by +running `gluster system:: execute` command from any one peer node. + +- This command will not copy any executables to peer nodes, Script + should exist in all peer nodes to use this infrastructure. Raises + error in case script not exists in any one of the peer node. +- Filename should start with `peer_` and should exist in + `$GLUSTER_LIBEXEC` directory. +- This command can not be called from outside the cluster. + +To understand the functionality, create a executable file `peer_hello` +under $GLUSTER_LIBEXEC directory and copy to all peer nodes. + + #!/usr/bin/env bash + echo "Hello from $(gluster system:: uuid get)" + +Now run the following command from any one gluster node, + + gluster system:: execute hello + +**Note:** Gluster will not copy the executable script to all nodes, + copy `peer_hello` script to all peer nodes to use `gluster system:: + execute` infrastructure. + +It will run `peer_hello` executable in all peer nodes and shows the +output from each node(Below example shows output from my two nodes +cluster) + + Hello from UUID: e7a3c5c8-e7ad-47ad-aa9c-c13907c4da84 + Hello from UUID: c680fc0a-01f9-4c93-a062-df91cc02e40f + +## cliutils +A Python wrapper around `gluster system:: execute` command is created +to address the following issues + +- If a node is down in the cluster, `system:: execute` just skips it + and runs only in up nodes. +- `system:: execute` commands are not user friendly +- It captures only stdout, so handling errors is tricky. + +**Advantages of cliutils:** + +- Single executable file will act as node component as well as User CLI. +- `execute_in_peers` utility function will merge the `gluster system:: + execute` output with `gluster peer status` to identify offline nodes. +- Easy CLI Arguments handling. +- If node component returns non zero return value then, `gluster + system:: execute` will fail to aggregate the output from other + nodes. `node_output_ok` or `node_output_notok` utility functions + returns zero both in case of success or error, but returns json + with ok: true or ok:false respectively. +- Easy to iterate on the node outputs. +- Better error handling - Geo-rep CLIs `gluster system:: execute + mountbroker`, `gluster system:: execute gsec_create` and `gluster + system:: add_secret_pub` are suffering from error handling. These + tools are not notifying user if any failures during execute or if a node + is down during execute. + +### Hello World +Create a file in `$LIBEXEC/glusterfs/peer_message.py` with following +content. + + #!/usr/bin/python3 + from gluster.cliutils import Cmd, runcli, execute_in_peers, node_output_ok + + class NodeHello(Cmd): + name = "node-hello" + + def run(self, args): + node_output_ok("Hello") + + class Hello(Cmd): + name = "hello" + + def run(self, args): + out = execute_in_peers("node-hello") + for row in out: + print ("{0} from {1}".format(row.output, row.hostname)) + + if __name__ == "__main__": + runcli() + +When we run `python peer_message.py`, it will have two subcommands, +"node-hello" and "hello". This file should be copied to +`$LIBEXEC/glusterfs` directory in all peer nodes. User will call +subcommand "hello" from any one peer node, which internally call +`gluster system:: execute message.py node-hello`(This runs in all peer +nodes and collect the outputs) + +For node component do not print the output directly, use +`node_output_ok` or `node_output_notok` functions. `node_output_ok` +additionally collects the node UUID and prints in JSON +format. `execute_in_peers` function will collect this output and +merges with `peers list` so that we don't miss the node information if +that node is offline. + +If you observed already, function `args` is optional, if you don't +have arguments then no need to create a function. When we run the +file, we will have two subcommands. For example, + + python peer_message.py hello + python peer_message.py node-hello + +First subcommand calls second subcommand in all peer nodes. Basically +`execute_in_peers(NAME, ARGS)` will be converted into + + CMD_NAME = FILENAME without "peers_" + gluster system:: execute <CMD_NAME> <SUBCOMMAND> <ARGS> + +In our example, + + filename = "peer_message.py" + cmd_name = "message.py" + gluster system:: execute ${cmd_name} node-hello + +Now create symlink in `/usr/bin` or `/usr/sbin` directory depending on +the usecase.(Optional step for usability) + + ln -s /usr/libexec/glusterfs/peer_message.py /usr/bin/gluster-message + +Now users can use `gluster-message` instead of calling +`/usr/libexec/glusterfs/peer_message.py` + + gluster-message hello + +### Showing CLI output as Table + +Following example uses prettytable library, which can be installed +using `pip install prettytable` or `dnf install python-prettytable` + + #!/usr/bin/python3 + from prettytable import PrettyTable + from gluster.cliutils import Cmd, runcli, execute_in_peers, node_output_ok + + class NodeHello(Cmd): + name = "node-hello" + + def run(self, args): + node_output_ok("Hello") + + class Hello(Cmd): + name = "hello" + + def run(self, args): + out = execute_in_peers("node-hello") + # Initialize the CLI table + table = PrettyTable(["ID", "NODE", "NODE STATUS", "MESSAGE"]) + table.align["NODE STATUS"] = "r" + for row in out: + table.add_row([row.nodeid, + row.hostname, + "UP" if row.node_up else "DOWN", + row.output if row.ok else row.error]) + + print table + + if __name__ == "__main__": + runcli() + + +Example output, + + +--------------------------------------+-----------+-------------+---------+ + | ID | NODE | NODE STATUS | MESSAGE | + +--------------------------------------+-----------+-------------+---------+ + | e7a3c5c8-e7ad-47ad-aa9c-c13907c4da84 | localhost | UP | Hello | + | bb57a4c4-86eb-4af5-865d-932148c2759b | vm2 | UP | Hello | + | f69b918f-1ffa-4fe5-b554-ee10f051294e | vm3 | DOWN | N/A | + +--------------------------------------+-----------+-------------+---------+ + +## How to package in Gluster +If the project is created in `$GLUSTER_SRC/tools/message` + +Add "message" to SUBDIRS list in `$GLUSTER_SRC/tools/Makefile.am` + +and then create a `Makefile.am` in `$GLUSTER_SRC/tools/message` +directory with following content. + + EXTRA_DIST = peer_message.py + + peertoolsdir = $(libexecdir)/glusterfs/ + peertools_SCRIPTS = peer_message.py + + install-exec-hook: + $(mkdir_p) $(DESTDIR)$(bindir) + rm -f $(DESTDIR)$(bindir)/gluster-message + ln -s $(libexecdir)/glusterfs/peer_message.py \ + $(DESTDIR)$(bindir)/gluster-message + + uninstall-hook: + rm -f $(DESTDIR)$(bindir)/gluster-message + +Thats all. Add following files in `glusterfs.spec.in` if packaging is +required.(Under `%files` section) + + %{_libexecdir}/glusterfs/peer_message.py* + %{_bindir}/gluster-message + +## Who is using cliutils +- gluster-mountbroker http://review.gluster.org/14544 +- gluster-eventsapi http://review.gluster.org/14248 +- gluster-georep-sshkey http://review.gluster.org/14732 +- gluster-restapi https://github.com/gluster/restapi + +## Limitations/TODOs +- Not yet possible to create CLI without any subcommand, For example + `gluster-message` without any arguments +- Hiding node subcommands in `--help`(`gluster-message --help` will + show all subcommands including node subcommands) +- Only positional arguments supported for node arguments, Optional + arguments can be used for other commands. +- API documentation diff --git a/extras/cliutils/__init__.py b/extras/cliutils/__init__.py new file mode 100644 index 00000000000..8765cc85099 --- /dev/null +++ b/extras/cliutils/__init__.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Reexporting the utility funcs and classes +from .cliutils import (runcli, + sync_file_to_peers, + execute_in_peers, + execute, + node_output_ok, + node_output_notok, + output_error, + oknotok, + yesno, + get_node_uuid, + Cmd, + GlusterCmdException, + set_common_args_func) + + +# This will be useful when `from cliutils import *` +__all__ = ["runcli", + "sync_file_to_peers", + "execute_in_peers", + "execute", + "node_output_ok", + "node_output_notok", + "output_error", + "oknotok", + "yesno", + "get_node_uuid", + "Cmd", + "GlusterCmdException", + "set_common_args_func"] diff --git a/extras/cliutils/cliutils.py b/extras/cliutils/cliutils.py new file mode 100644 index 00000000000..55fbaf56704 --- /dev/null +++ b/extras/cliutils/cliutils.py @@ -0,0 +1,237 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +from argparse import ArgumentParser, RawDescriptionHelpFormatter +import inspect +import subprocess +import os +import xml.etree.cElementTree as etree +import json +import sys + +MY_UUID = None +parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter, + description=__doc__) +subparsers = parser.add_subparsers(dest="mode") + +subcommands = {} +cache_data = {} +ParseError = etree.ParseError if hasattr(etree, 'ParseError') else SyntaxError +_common_args_func = lambda p: True + + +class GlusterCmdException(Exception): + def __init__(self, message): + self.message = message + try: + # Python 3 + super().__init__(message) + except TypeError: + # Python 2 + super(GlusterCmdException, self).__init__(message) + + +def get_node_uuid(): + # Caches the Node UUID in global variable, + # Executes gluster system:: uuid get command only if + # calling this function for first time + global MY_UUID + if MY_UUID is not None: + return MY_UUID + + cmd = ["gluster", "system::", "uuid", "get", "--xml"] + rc, out, err = execute(cmd) + + if rc != 0: + return None + + tree = etree.fromstring(out) + uuid_el = tree.find("uuidGenerate/uuid") + MY_UUID = uuid_el.text + return MY_UUID + + +def yesno(flag): + return "Yes" if flag else "No" + + +def oknotok(flag): + return "OK" if flag else "NOT OK" + + +def output_error(message, errcode=1): + print (message, file=sys.stderr) + sys.exit(errcode) + + +def node_output_ok(message=""): + # Prints Success JSON output and exits with returncode zero + out = {"ok": True, "nodeid": get_node_uuid(), "output": message} + print (json.dumps(out)) + sys.exit(0) + + +def node_output_notok(message): + # Prints Error JSON output and exits with returncode zero + out = {"ok": False, "nodeid": get_node_uuid(), "error": message} + print (json.dumps(out)) + sys.exit(0) + + +def execute(cmd): + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + out, err = p.communicate() + return p.returncode, out, err + + +def get_pool_list(): + cmd = ["gluster", "--mode=script", "pool", "list", "--xml"] + rc, out, err = execute(cmd) + if rc != 0: + output_error("Failed to get Pool Info: {0}".format(err)) + + tree = etree.fromstring(out) + + pool = [] + try: + for p in tree.findall('peerStatus/peer'): + pool.append({"nodeid": p.find("uuid").text, + "hostname": p.find("hostname").text, + "connected": (True if p.find("connected").text == "1" + else False)}) + except (ParseError, AttributeError, ValueError) as e: + output_error("Failed to parse Pool Info: {0}".format(e)) + + return pool + + +class NodeOutput(object): + def __init__(self, **kwargs): + self.nodeid = kwargs.get("nodeid", "") + self.hostname = kwargs.get("hostname", "") + self.node_up = kwargs.get("node_up", False) + self.ok = kwargs.get("ok", False) + self.output = kwargs.get("output", "N/A") + self.error = kwargs.get("error", "N/A") + + +def execute_in_peers(name, args=[]): + # Get the file name of Caller function, If the file name is peer_example.py + # then Gluster peer command will be gluster system:: execute example.py + # Command name is without peer_ + frame = inspect.stack()[1] + module = inspect.getmodule(frame[0]) + actual_file = module.__file__ + # If file is symlink then find actual file + if os.path.islink(actual_file): + actual_file = os.readlink(actual_file) + + # Get the name of file without peer_ + cmd_name = os.path.basename(actual_file).replace("peer_", "") + cmd = ["gluster", "system::", "execute", cmd_name, name] + args + rc, out, err = execute(cmd) + if rc != 0: + raise GlusterCmdException((rc, out, err, " ".join(cmd))) + + out = out.strip().splitlines() + + # JSON decode each line and construct one object with node id as key + all_nodes_data = {} + for node_data in out: + data = json.loads(node_data) + all_nodes_data[data["nodeid"]] = { + "nodeid": data.get("nodeid"), + "ok": data.get("ok"), + "output": data.get("output", ""), + "error": data.get("error", "")} + + # gluster pool list + pool_list = get_pool_list() + + data_out = [] + # Iterate pool_list and merge all_nodes_data collected above + # If a peer node is down then set node_up = False + for p in pool_list: + p_data = all_nodes_data.get(p.get("nodeid"), None) + row_data = NodeOutput(node_up=False, + hostname=p.get("hostname"), + nodeid=p.get("nodeid"), + ok=False) + + if p_data is not None: + # Node is UP + row_data.node_up = True + row_data.ok = p_data.get("ok") + row_data.output = p_data.get("output") + row_data.error = p_data.get("error") + + data_out.append(row_data) + + return data_out + + +def sync_file_to_peers(fname): + # Copy file from current node to all peer nodes, fname + # is path after GLUSTERD_WORKDIR + cmd = ["gluster", "system::", "copy", "file", fname] + rc, out, err = execute(cmd) + if rc != 0: + raise GlusterCmdException((rc, out, err)) + + +class Cmd(object): + name = "" + + def run(self, args): + # Must required method. Raise NotImplementedError if derived class + # not implemented this method + raise NotImplementedError("\"run(self, args)\" method is " + "not implemented by \"{0}\"".format( + self.__class__.__name__)) + + +def runcli(): + # Get list of Classes derived from class "Cmd" and create + # a subcommand as specified in the Class name. Call the args + # method by passing subcommand parser, Derived class can add + # arguments to the subcommand parser. + metavar_data = [] + for c in Cmd.__subclasses__(): + cls = c() + if getattr(cls, "name", "") == "": + raise NotImplementedError("\"name\" is not added " + "to \"{0}\"".format( + cls.__class__.__name__)) + + # Do not show in help message if subcommand starts with node- + if not cls.name.startswith("node-"): + metavar_data.append(cls.name) + + p = subparsers.add_parser(cls.name) + args_func = getattr(cls, "args", None) + if args_func is not None: + args_func(p) + + # Apply common args if any + _common_args_func(p) + + # A dict to save subcommands, key is name of the subcommand + subcommands[cls.name] = cls + + # Hide node commands in Help message + subparsers.metavar = "{" + ",".join(metavar_data) + "}" + + # Get all parsed arguments + args = parser.parse_args() + + # Get the subcommand to execute + cls = subcommands.get(args.mode, None) + + # Run + if cls is not None: + cls.run(args) + + +def set_common_args_func(func): + global _common_args_func + _common_args_func = func diff --git a/extras/collect-system-stats.sh b/extras/collect-system-stats.sh new file mode 100755 index 00000000000..865e70bbc11 --- /dev/null +++ b/extras/collect-system-stats.sh @@ -0,0 +1,52 @@ +#!/bin/bash +################################################################################ +# Usage: collect-system-stats.sh <delay-in-seconds> +# This script starts sar/top/iostat/vmstat processes which collect system stats +# with the interval <delay-in-seconds> given as argument to the script. When +# the script is stopped either by entering any input or Ctrl+C the list of +# files where output is captured will be printed on the screen which can be +# observed to find any problems/bottlenecks. +############################################################################### + +function stop_processes { + echo "Stopping the monitoring processes" + echo "sar pid:$sar_pid", "top pid: $top_pid", "iostat pid: $iostat_pid", "vmstat pid: $vmstat_pid" + kill "$sar_pid" "$top_pid" "$iostat_pid" "$vmstat_pid" + echo "Files created: ${timestamp}-network.out, ${timestamp}-top.out, ${timestamp}-iostat.out, ${timestamp}-vmstat.out" +} + +function check_dependent_commands_exist() +{ + declare -a arr=("sar" "top" "iostat" "vmstat") + for i in "${arr[@]}" + do + if ! command -v "$i" > /dev/null 2>&1 + then + echo "ERROR: '$i' command is not found" + exit 1 + fi + done + +} + +case "$1" in + ''|*[!0-9]*) echo "Usage: $0 <delay-between-successive-metrics-collection-in-seconds>"; exit 1 ;; + *) interval="$1" ;; +esac + +timestamp=$(date +"%s") + +check_dependent_commands_exist +sar -n DEV "$interval" > "${timestamp}"-network.out & +sar_pid="$!" +top -bHd "$interval" > "${timestamp}"-top.out & +top_pid="$!" +iostat -Ntkdx "$interval" > "${timestamp}"-iostat.out & +iostat_pid="$!" +vmstat -t "$interval" > "${timestamp}"-vmstat.out & +vmstat_pid="$!" +echo "Started sar, vmstat, iostat, top for collecting stats" + + +trap stop_processes EXIT +read -r -p "Press anything and ENTER to exit"; diff --git a/extras/command-completion/gluster.bash b/extras/command-completion/gluster.bash index be0591c7211..73d16098875 100644 --- a/extras/command-completion/gluster.bash +++ b/extras/command-completion/gluster.bash @@ -26,28 +26,28 @@ GLUSTER_TOP_SUBOPTIONS2=" " GLUSTER_TOP_OPTIONS=" {open - [ $TOP_SUBOPTIONS1 ] + [ $GLUSTER_TOP_SUBOPTIONS1 ] }, {read - [ $TOP_SUBOPTIONS1 ] + [ $GLUSTER_TOP_SUBOPTIONS1 ] }, {write - [ $TOP_SUBOPTIONS1 ] + [ $GLUSTER_TOP_SUBOPTIONS1 ] }, {opendir - [ $TOP_SUBOPTIONS1 ] + [ $GLUSTER_TOP_SUBOPTIONS1 ] }, {readdir - [ $TOP_SUBOPTIONS1 ] + [ $GLUSTER_TOP_SUBOPTIONS1 ] }, {clear - [ $TOP_SUBOPTIONS1 ] + [ $GLUSTER_TOP_SUBOPTIONS1 ] }, {read-perf - [ $TOP_SUBOPTIONS2 ] + [ $GLUSTER_TOP_SUBOPTIONS2 ] }, {write-perf - [ $TOP_SUBOPTIONS2 ] + [ $GLUSTER_TOP_SUBOPTIONS2 ] } " @@ -282,16 +282,16 @@ _gluster_throw () { exit } -declare FINAL_LIST='' -declare LIST='' -declare -i TOP=0 +declare GLUSTER_FINAL_LIST='' +declare GLUSTER_LIST='' +declare -i GLUSTER_TOP=0 _gluster_push () { - TOP=$((TOP + 1)) - return $TOP + GLUSTER_TOP=$((GLUSTER_TOP + 1)) + return $GLUSTER_TOP } _gluster_pop () { - TOP=$((TOP - 1)) - return $TOP + GLUSTER_TOP=$((GLUSTER_TOP - 1)) + return $GLUSTER_TOP } _gluster_goto_end () @@ -333,7 +333,7 @@ _gluster_form_list () top=$? read -r key if [ "X$cur_word" == "X" -o "${cur_word:0:1}" == "${key:0:1}" -o "${key:0:1}" == "_" ]; then - LIST="$LIST $key" + GLUSTER_LIST="$GLUSTER_LIST $key" fi _gluster_goto_end $top @@ -452,10 +452,10 @@ _gluster_parse () elif [ "$token" == '{' ]; then read -r tmp_token - LIST="$tmp_token" + GLUSTER_LIST="$tmp_token" fi - echo $LIST + echo $GLUSTER_LIST } _gluster_handle_list () @@ -479,12 +479,12 @@ _gluster_handle_list () _gluster_completion () { - FINAL_LIST=`echo $GLUSTER_COMMAND_TREE | \ - egrep -ao --color=never "([A-Za-z0-9_-.]+)|[[:space:]]+|." | \ + GLUSTER_FINAL_LIST=`echo $GLUSTER_COMMAND_TREE | \ + egrep -ao --color=never "([A-Za-z0-9_.-]+)|[[:space:]]+|." | \ egrep -v --color=never "^[[:space:]]*$" | \ _gluster_parse` - ARG="FINAL_LIST" + ARG="GLUSTER_FINAL_LIST" _gluster_handle_list $ARG ${COMP_WORDS[COMP_CWORD]} return } diff --git a/extras/control-cpu-load.sh b/extras/control-cpu-load.sh new file mode 100755 index 00000000000..52dcf62fd9f --- /dev/null +++ b/extras/control-cpu-load.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +USAGE="This script provides a utility to control CPU utilization for any +gluster daemon.In this, we use cgroup framework to configure CPU quota +for a process(like selfheal daemon). Before running this script, make +sure that daemon is running.Every time daemon restarts, it is required +to rerun this command to set CPU quota on new daemon process id. +User can enter any value between 10 to 100 for CPU quota. +Recommended value of quota period is 25. 25 means, kernel will allocate +25 ms period to this group of tasks in every 100 ms period. This 25ms +could be considered as the maximum percentage of CPU quota daemon can take. +This value will be reflected on CPU usage of "top" command.If provided pid +is the only process and no other process is in competition to get CPU, more + than 25% could be allocated to daemon to speed up the process." + +if [ $# -ge 1 ]; then + case $1 in + -h|--help) echo " " "$USAGE" | sed -r -e 's/^[ ]+//g' + exit 0; + ;; + *) echo "Please Provide correct input for script." + echo "For help correct options are -h or --help." + exit 1; + ;; + esac +fi + +DIR_EXIST=0 +LOC="/sys/fs/cgroup/cpu,cpuacct/system.slice/glusterd.service" +echo "Enter gluster daemon pid for which you want to control CPU." +read daemon_pid + +if expr ${daemon_pid} + 0 > /dev/null 2>&1 ;then + CHECK_PID=$(pgrep -f gluster | grep ${daemon_pid}) + if [ -z "${CHECK_PID}" ]; then + echo "No daemon is running or pid ${daemon_pid} does not match." + echo "with running gluster processes." + exit 1 + fi +else + echo "Entered daemon_pid is not numeric so Rerun the script." + exit 1 +fi + + +if [ -f ${LOC}/tasks ];then + CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/tasks) + if [ ${CHECK_CGROUP} ]; then + echo "pid ${daemon_pid} is attached with glusterd.service cgroup." + fi +fi + +cgroup_name=cgroup_gluster_${daemon_pid} +if [ -f ${LOC}/${cgroup_name}/tasks ]; then + CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/${cgroup_name}/tasks) + if [ ${CHECK_CGROUP} ]; then + val=`cat ${LOC}/${cgroup_name}/cpu.cfs_quota_us` + qval=$((val / 1000)) + echo "pid ${daemon_pid} is already attached ${cgroup_name} with quota value ${qval}." + echo "Press n if you don't want to reassign ${daemon_pid} with new quota value." + DIR_EXIST=1 + else + echo "pid ${daemon_pid} is not attached with ${cgroup_name}." + fi +fi + +read -p "If you want to continue the script to attach ${daemon_pid} with new ${cgroup_name} cgroup Press (y/n)?" choice +case "$choice" in + y|Y ) echo "yes";; + n|N ) echo "no";exit;; + * ) echo "invalid";exit;; +esac + +systemctl set-property glusterd.service CPUShares=1024 + +if [ ${DIR_EXIST} -eq 0 ];then + echo "Creating child cgroup directory '${cgroup_name} cgroup' for glusterd.service." + mkdir -p ${LOC}/${cgroup_name} + if [ ! -f ${LOC}/${cgroup_name}/tasks ];then + echo "Not able to create ${cgroup_name} directory so exit." + exit 1 + fi +fi + +echo "Enter quota value in range [10,100]: " + +read quota_value +if expr ${quota_value} + 0 > /dev/null 2>&1 ;then + if [ ${quota_value} -lt 10 ] || [ ${quota_value} -gt 100 ]; then + echo "Entered quota value is not correct,it should be in the range ." + echo "10-100. Ideal value is 25." + echo "Rerun the sript with correct value." + exit 1 + else + echo "Entered quota value is $quota_value" + fi +else + echo "Entered quota value is not numeric so Rerun the script." + exit 1 +fi + +quota_value=$((quota_value * 1000)) +echo "Setting $quota_value to cpu.cfs_quota_us for gluster_cgroup." +echo ${quota_value} > ${LOC}/${cgroup_name}/cpu.cfs_quota_us + +if ps -T -p ${daemon_pid} | grep gluster > /dev/null; then + for thid in `ps -T -p ${daemon_pid} | grep -v SPID | awk -F " " '{print $2}'`; + do + echo ${thid} > ${LOC}/${cgroup_name}/tasks ; + done + if cat /proc/${daemon_pid}/cgroup | grep -w ${cgroup_name} > /dev/null; then + echo "Tasks are attached successfully specific to ${daemon_pid} to ${cgroup_name}." + else + echo "Tasks are not attached successfully." + fi +fi diff --git a/extras/control-mem.sh b/extras/control-mem.sh new file mode 100755 index 00000000000..91b36f8107a --- /dev/null +++ b/extras/control-mem.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +USAGE="This commands provides a utility to control MEMORY utilization for any +gluster daemon.In this, we use cgroup framework to configure MEMORY limit for +a process. Before running this script, make sure that daemon is running.Every +time daemon restarts, it is required to rerun this command to set memory limit +(in bytes) on new daemon process id.User can enter any value between 100 +(in Mega bytes) to 8000000000000 for Memory limit in Mega bytes. +Memory limit value is depends on how much maximum memory user wants to restrict +for specific daemon process.If a process will try to consume memore more than +configured value then cgroup will hang/sleep this task and to resume the task +rerun the script with new increase memory limit value ." + +if [ $# -ge 1 ]; then + case $1 in + -h|--help) echo " " "$USAGE" | sed -r -e 's/^[ ]+//g' + exit 0; + ;; + *) echo "Please Provide correct input for script." + echo "For help correct options are -h of --help." + exit 1; + ;; + esac +fi + +DIR_EXIST=0 +LOC="/sys/fs/cgroup/memory/system.slice/glusterd.service" +echo "Enter Any gluster daemon pid for that you want to control MEMORY." +read daemon_pid + +if expr ${daemon_pid} + 0 > /dev/null 2>&1 ;then + CHECK_PID=$(pgrep -f gluster | grep ${daemon_pid}) + if [ -z "${CHECK_PID}" ]; then + echo "No daemon is running or pid ${daemon_pid} does not match." + echo "with running gluster processes." + exit 1 + fi +else + echo "Entered daemon_pid is not numeric so Rerun the script." + exit 1 +fi + + +if [ -f ${LOC}/tasks ]; then + CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/tasks) + if [ ${CHECK_CGROUP} ] ;then + echo "pid ${daemon_pid} is attached with default glusterd.service cgroup." + fi +fi + +cgroup_name=cgroup_gluster_${daemon_pid} +if [ -f ${LOC}/${cgroup_name}/tasks ];then + CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/${cgroup_name}/tasks) + if [ ${CHECK_CGROUP} ]; then + val=`cat ${LOC}/${cgroup_name}/memory.limit_in_bytes` + mval=$((val / 1024 / 1024)) + echo "pid ${daemon_pid} is already attached ${cgroup_name} with mem value ${mval}." + echo "Press n if you don't want to reassign ${daemon_pid} with new mem value." + DIR_EXIST=1 + else + echo "pid ${daemon_pid} is not attached with ${cgroup_name}." + fi +fi + +read -p "If you want to continue the script to attach daeomon with new cgroup. Press (y/n)?" choice +case "$choice" in + y|Y ) echo "yes";; + n|N ) echo "no";exit;; + * ) echo "invalid";exit;; +esac + +systemctl set-property glusterd.service CPUShares=1024 + +if [ ${DIR_EXIST} -eq 0 ];then + echo "Creating child cgroup directory '${cgroup_name} cgroup' for glusterd.service." + mkdir -p ${LOC}/${cgroup_name} + if [ ! -f ${LOC}/${cgroup_name}/tasks ];then + echo "Not able to create ${LOC}/${cgroup_name} directory so exit." + exit 1 + fi +fi + +echo "Enter Memory value in Mega bytes [100,8000000000000]: " + +read mem_value +if expr ${mem_value} + 0 > /dev/null 2>&1 ;then + if [ ${mem_value} -lt 100 ] || [ ${mem_value} -gt 8000000000000 ]; then + echo "Entered memory value is not correct,it should be in the range ." + echo "100-8000000000000, Rerun the script with correct value ." + exit 1 + else + echo "Entered memory limit value is ${mem_value}." + fi +else + echo "Entered memory value is not numeric so Rerun the script." + exit 1 +fi + +mem_value=$(($mem_value * 1024 * 1024)) +if [ ${DIR_EXIST} -eq 0 ];then + echo "Setting ${mem_value} to memory.limit_in_bytes for ${LOC}/${cgroup_name}." + echo ${mem_value} > ${LOC}/${cgroup_name}/memory.limit_in_bytes + #Set memory value to memory.memsw.limit_in_bytes + echo ${mem_value} > ${LOC}/${cgroup_name}/memory.memsw.limit_in_bytes + # disable oom_control so that kernel will not send kill signal to the + # task once limit has reached + echo 1 > ${LOC}/${cgroup_name}/memory.oom_control +else + #Increase mem_value to memory.memsw.limit_in_bytes + echo ${mem_value} > ${LOC}/${cgroup_name}/memory.memsw.limit_in_bytes + echo "Increase ${mem_value} to memory.limit_in_bytes for ${LOC}/${cgroup_name}." + echo ${mem_value} > ${LOC}/${cgroup_name}/memory.limit_in_bytes + # disable oom_control so that kernel will not send kill signal to the + # task once limit has reached + echo 1 > ${LOC}/${cgroup_name}/memory.oom_control +fi + +if ps -T -p ${daemon_pid} | grep gluster > /dev/null; then + for thid in `ps -T -p ${daemon_pid} | grep -v SPID | awk -F " " '{print $2}'`; + do + echo ${thid} > ${LOC}/${cgroup_name}/tasks ; + done + if cat /proc/${daemon_pid}/cgroup | grep -iw ${cgroup_name} > /dev/null; then + echo "Tasks are attached successfully specific to ${daemon_pid} to ${cgroup_name}." + else + echo "Tasks are not attached successfully." + fi +fi diff --git a/extras/create_new_xlator/README.md b/extras/create_new_xlator/README.md new file mode 100644 index 00000000000..fdc1ba00812 --- /dev/null +++ b/extras/create_new_xlator/README.md @@ -0,0 +1,24 @@ +####This document explains how to create a template for your new xlator. + +`$ python ./generate_xlator.py <XLATOR_DIRECTORY> <XLATOR_NAME> <FOP_PREFIX>` + * XLATOR_DIRECTORY: Directory path where the new xlator folder will reside + * XLATOR_NAME: Name of the xlator you wish to create + * FOP_PREFIX: This is the fop prefix that you wish to prefix every fop definition in your xlator, fop prefix is generally different than xlator name, if the xlator name is too long. + +Eg: `python ./generate_xlator.py /home/u1/glusterfs/xlators/features compression cmpr` +This command will create the following files with some initial contents like copyright, fops definition etc. +Note that there shouldn't be a "/" specified at the end of the <XLATOR_DIRECTORY> + `* /home/u1/glusterfs/xlators/features/compression/Makefile.am + * /home/u1/glusterfs/xlators/features/compression/src/Makefile.am + * /home/u1/glusterfs/xlators/features/compression/src/compression.c + * /home/u1/glusterfs/xlators/features/compression/src/compression.h + * /home/u1/glusterfs/xlators/features/compression/src/compression-mem-types.h + * /home/u1/glusterfs/xlators/features/compression/src/compression-messages.h` + +By default all the fops and functions are generated, if you wish to not implement certain fops and functions, comment those lines (by adding '#' at the start of the line) in libglusterfs/src/generate_xlator.py + +Few other manual steps required to get the new xlator completely functional: +* Change configure.ac +* Change `<XLATOR_DIRECTORY>/Makefile.am` to include the new xlator directory. + Eg: `/home/u1/glusterfs/xlators/features/Makefile.am` +* Change vol file or glusterd volgen to include the new xlator in volfile diff --git a/extras/create_new_xlator/generate_xlator.py b/extras/create_new_xlator/generate_xlator.py new file mode 100755 index 00000000000..983868c04db --- /dev/null +++ b/extras/create_new_xlator/generate_xlator.py @@ -0,0 +1,208 @@ +#!/usr/bin/python3 + +from __future__ import print_function +import os +import re +import sys +import string +import time +path = os.path.abspath(os.path.dirname(__file__)) + '/../../libglusterfs/src' +sys.path.append(path) +from generator import ops, xlator_cbks, xlator_dumpops + +MAKEFILE_FMT = """ +xlator_LTLIBRARIES = @XL_NAME@.la +xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/@XL_TYPE@ +@XL_NAME_NO_HYPHEN@_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) +@XL_NAME_NO_HYPHEN@_la_SOURCES = @XL_NAME@.c +@XL_NAME_NO_HYPHEN@_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la +noinst_HEADERS = @XL_NAME@.h @XL_NAME@-mem-types.h @XL_NAME@-messages.h +AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ + -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src +AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS) +CLEANFILES = +""" + +fop_subs = {} +cbk_subs = {} +fn_subs = {} + + +def get_error_arg(type_str): + if type_str.find(" *") != -1: + return "NULL" + return "-1" + + +def get_param(names, types): + # Convert two separate tuples to one of (name, type) sub-tuples. + as_tuples = list(zip(types, names)) + # Convert each sub-tuple into a "type name" string. + as_strings = [' '.join(item) for item in as_tuples] + # Join all of those into one big string. + return ',\n\t'.join(as_strings) + + +def generate(tmpl, name, table): + w_arg_names = [a[1] for a in table[name] if a[0] == 'fop-arg'] + w_arg_types = [a[2] for a in table[name] if a[0] == 'fop-arg'] + u_arg_names = [a[1] for a in table[name] if a[0] == 'cbk-arg'] + u_arg_types = [a[2] for a in table[name] if a[0] == 'cbk-arg'] + fn_arg_names = [a[1] for a in table[name] if a[0] == 'fn-arg'] + fn_arg_types = [a[2] for a in table[name] if a[0] == 'fn-arg'] + ret_type = [a[1] for a in table[name] if a[0] == 'ret-val'] + ret_var = [a[2] for a in table[name] if a[0] == 'ret-val'] + + sdict = {} + #Parameters are (t1, var1), (t2, var2)... + #Args are (var1, var2,...) + sdict["@WIND_ARGS@"] = ', '.join(w_arg_names) + sdict["@UNWIND_ARGS@"] = ', '.join(u_arg_names) + sdict["@ERROR_ARGS@"] = ', '.join(list(map(get_error_arg, u_arg_types))) + sdict["@WIND_PARAMS@"] = get_param(w_arg_names, w_arg_types) + sdict["@UNWIND_PARAMS@"] = get_param(u_arg_names, u_arg_types) + sdict["@FUNC_PARAMS@"] = get_param(fn_arg_names, fn_arg_types) + sdict["@NAME@"] = name + sdict["@FOP_PREFIX@"] = fop_prefix + sdict["@RET_TYPE@"] = ''.join(ret_type) + sdict["@RET_VAR@"] = ''.join(ret_var) + + for old, new in sdict.items(): + tmpl = tmpl.replace(old, new) + # TBD: reindent/reformat the result for maximum readability. + return tmpl + + +def gen_xlator(): + xl = open(src_dir_path+"/"+xl_name+".c", 'w+') + + print(COPYRIGHT, file=xl) + print(fragments["INCLUDE_IN_SRC_FILE"].replace("@XL_NAME@", + xl_name), file=xl) + + #Generate cbks and fops + for fop in ops: + print(generate(fragments["CBK_TEMPLATE"], fop, ops), file=xl) + print(generate(fragments["FOP_TEMPLATE"], fop, ops), file=xl) + + for cbk in xlator_cbks: + print(generate(fragments["FUNC_TEMPLATE"], cbk, + xlator_cbks), file=xl) + + for dops in xlator_dumpops: + print(generate(fragments["FUNC_TEMPLATE"], dops, + xlator_dumpops), file=xl) + + #Generate fop table + print("struct xlator_fops fops = {", file=xl) + for fop in ops: + print(" .{0:20} = {1}_{2},".format(fop, fop_prefix, fop), file=xl) + print("};", file=xl) + + #Generate xlator_cbks table + print("struct xlator_cbks cbks = {", file=xl) + for cbk in xlator_cbks: + print(" .{0:20} = {1}_{2},".format(cbk, fop_prefix, cbk), file=xl) + print("};", file=xl) + + #Generate xlator_dumpops table + print("struct xlator_dumpops dumpops = {", file=xl) + for dops in xlator_dumpops: + print(" .{0:20} = {1}_{2},".format(dops, fop_prefix, dops), file=xl) + print("};", file=xl) + + xlator_methods = fragments["XLATOR_METHODS"].replace("@XL_NAME@", xl_name) + xlator_methods = xlator_methods.replace("@FOP_PREFIX@", fop_prefix) + print(xlator_methods, file=xl) + + xl.close() + + +def create_dir_struct(): + if not os.path.exists(dir_path+"/src"): + os.makedirs(dir_path+"/src") + + +def gen_header_files(): + upname = xl_name_no_hyphen.upper() + h = open(src_dir_path+"/"+xl_name+".h", 'w+') + print(COPYRIGHT, file=h) + txt = fragments["HEADER_FMT"].replace("@HFL_NAME@", upname) + txt = txt.replace("@XL_NAME@", xl_name) + print(txt, file=h) + h.close() + + h = open(src_dir_path+"/"+xl_name+"-mem-types.h", 'w+') + print(COPYRIGHT, file=h) + txt = fragments["MEM_HEADER_FMT"].replace("@HFL_NAME@", upname+"_MEM_TYPES") + txt = txt.replace("@FOP_PREFIX@", fop_prefix) + print(txt, file=h) + h.close() + + h = open(src_dir_path+"/"+xl_name+"-messages.h", 'w+') + print(COPYRIGHT, file=h) + txt = fragments["MSG_HEADER_FMT"].replace("@HFL_NAME@", upname+"_MESSAGES") + txt = txt.replace("@FOP_PREFIX@", fop_prefix.upper()) + print(txt, file=h) + h.close() + + +def gen_makefiles(): + m = open(dir_path+"/Makefile.am", 'w+') + print("SUBDIRS = src\n\nCLEANFILES =", file=m) + m.close() + + m = open(src_dir_path+"/Makefile.am", 'w+') + txt = MAKEFILE_FMT.replace("@XL_NAME@", xl_name) + txt = txt.replace("@XL_NAME_NO_HYPHEN@", xl_name_no_hyphen) + txt = txt.replace("@XL_TYPE@", xlator_type) + print(txt, file=m) + m.close() + +def get_copyright (): + return fragments["CP"].replace("@CURRENT_YEAR@", + time.strftime("%Y")) + +def load_fragments (): + pragma_re = re.compile('pragma fragment (.*)') + cur_symbol = None + cur_value = "" + result = {} + basepath = os.path.abspath(os.path.dirname(__file__)) + fragpath = basepath + "/new-xlator.c.tmpl" + for line in open(fragpath, "r").readlines(): + m = pragma_re.search(line) + if m: + if cur_symbol: + result[cur_symbol] = cur_value + cur_symbol = m.group(1) + cur_value = "" + else: + cur_value += line + if cur_symbol: + result[cur_symbol] = cur_value + return result + +if __name__ == '__main__': + + if len(sys.argv) < 3: + print("USAGE: ./gen_xlator <XLATOR_DIR> <XLATOR_NAME> <FOP_PREFIX>") + sys.exit(0) + + xl_name = sys.argv[2] + xl_name_no_hyphen = xl_name.replace("-", "_") + if sys.argv[1].endswith('/'): + dir_path = sys.argv[1] + xl_name + else: + dir_path = sys.argv[1] + "/" + xl_name + xlator_type = os.path.basename(sys.argv[1]) + fop_prefix = sys.argv[3] + src_dir_path = dir_path + "/src" + + fragments = load_fragments() + + COPYRIGHT = get_copyright() + create_dir_struct() + gen_xlator() + gen_header_files() + gen_makefiles() diff --git a/extras/create_new_xlator/new-xlator.c.tmpl b/extras/create_new_xlator/new-xlator.c.tmpl new file mode 100644 index 00000000000..fe9735bfcf1 --- /dev/null +++ b/extras/create_new_xlator/new-xlator.c.tmpl @@ -0,0 +1,151 @@ +#pragma fragment CBK_TEMPLATE +int32_t @FOP_PREFIX@_@NAME@_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, @UNWIND_PARAMS@) +{ + STACK_UNWIND_STRICT(@NAME@, frame, op_ret, op_errno, @UNWIND_ARGS@); + return 0; +} + +#pragma fragment COMMENT +If you are generating the leaf xlators, remove the STACK_WIND and replace the + @ERROR_ARGS@ to @UNWIND_ARGS@ if necessary + +#pragma fragment FOP_TEMPLATE + int32_t @FOP_PREFIX@_@NAME@(call_frame_t *frame, xlator_t *this, @WIND_PARAMS@) +{ + STACK_WIND(frame, @FOP_PREFIX@_@NAME@_cbk, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->@NAME@, @WIND_ARGS@); + return 0; +err: + STACK_UNWIND_STRICT(@NAME@, frame, -1, errno, @ERROR_ARGS@); + return 0; +} + +#pragma fragment FUNC_TEMPLATE +@RET_TYPE@ @FOP_PREFIX@_@NAME@(@FUNC_PARAMS@) +{ + return @RET_VAR@; +} + +#pragma fragment CP +/* + * Copyright (c) @CURRENT_YEAR@ Red Hat, Inc. <http://www.redhat.com> + * This file is part of GlusterFS. + * + * This file is licensed to you under your choice of the GNU Lesser + * General Public License, version 3 or any later version (LGPLv3 or + * later), or the GNU General Public License, version 2 (GPLv2), in all + * cases as published by the Free Software Foundation. + */ + +#pragma fragment INCLUDE_IN_SRC_FILE +#include "@XL_NAME@.h" + +#pragma fragment XLATOR_METHODS + +static int32_t @FOP_PREFIX@_init(xlator_t *this) +{ + return 0; +} + +static void @FOP_PREFIX@_fini(xlator_t *this) +{ + return; +} + +static int32_t @FOP_PREFIX@_reconfigure(xlator_t *this, dict_t *dict) +{ + return 0; +} + +static int @FOP_PREFIX@_notify(xlator_t *this, int event, void *data, ...) +{ + return default_notify(this, event, data); +} + +static int32_t @FOP_PREFIX@_mem_acct_init(xlator_t *this) +{ + int ret = -1; + + ret = xlator_mem_acct_init(this, gf_@FOP_PREFIX@_mt_end + 1); + return ret; +} + +static int32_t @FOP_PREFIX@_dump_metrics(xlator_t *this, int fd) +{ + return 0; +} + +struct volume_options @FOP_PREFIX@_options[] = { + /*{ .key = {""}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "", + .op_version = {GD_OP_VERSION_}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT, + .tags = {""}, + .description = "", + .category = GF_EXPERIMENTAL, + }, + { .key = {NULL} }, + */ +}; + +xlator_api_t xlator_api = { + .init = @FOP_PREFIX@_init, + .fini = @FOP_PREFIX@_fini, + .notify = @FOP_PREFIX@_notify, + .reconfigure = @FOP_PREFIX@_reconfigure, + .mem_acct_init = @FOP_PREFIX@_mem_acct_init, + .dump_metrics = @FOP_PREFIX@_dump_metrics, + .op_version = {GD_OP_VERSION_}, + .dumpops = &@FOP_PREFIX@_dumpops, + .fops = &@FOP_PREFIX@_fops, + .cbks = &@FOP_PREFIX @_cbks, + .options = @FOP_PREFIX@_options, + .identifier = "@XL_NAME@", + .category = GF_EXPERIMENTAL, +}; +#pragma fragment HEADER_FMT +#ifndef __ @HFL_NAME@_H__ +#define __ @HFL_NAME@_H__ + +#include "@XL_NAME@-mem-types.h" +#include "@XL_NAME@-messages.h" +#include <glusterfs/glusterfs.h> +#include <glusterfs/xlator.h> +#include <glusterfs/defaults.h> + +#endif /* __@HFL_NAME@_H__ */ + +#pragma fragment MEM_HEADER_FMT +#ifndef __ @HFL_NAME@_H__ +#define __ @HFL_NAME@_H__ + +#include <glusterfs/mem-types.h> + +enum gf_mdc_mem_types_ { + gf_@FOP_PREFIX@_mt_ = gf_common_mt_end + 1, + gf_@FOP_PREFIX@_mt_end +}; + +#endif /* __@HFL_NAME@_H__ */ + +#pragma fragment MSG_HEADER_FMT +#ifndef __@HFL_NAME@_H__ +#define __@HFL_NAME@_H__ + +#include <glusterfs/glfs-message-id.h> + +/* To add new message IDs, append new identifiers at the end of the list. + * + * Never remove a message ID. If it's not used anymore, you can rename it or + * leave it as it is, but not delete it. This is to prevent reutilization of + * IDs by other messages. + * + * The component name must match one of the entries defined in + * glfs-message-id.h. + */ + +GLFS_MSGID(@FOP_PREFIX@, @FOP_PREFIX@_MSG_NO_MEMORY); + +#endif /* __@HFL_NAME@_H__ */ diff --git a/extras/devel-tools/devel-vagrant/Vagrantfile b/extras/devel-tools/devel-vagrant/Vagrantfile new file mode 100644 index 00000000000..78dc29bdc68 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/Vagrantfile @@ -0,0 +1,165 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# +# Author: Rajesh Joseph (rjoseph@redhat.com) +# Author: Christopher Blum (cblum@redhat.com) +# + + +#Variables +box_name = "gluster-dev-fedora" +box_url = "http://download.gluster.org/pub/gluster/glusterfs/vagrant/gluster-dev-fedora/boxes/gluster-dev-fedora.json" +node_count = 0 +disk_count = -1 +node_name = "Node" +ipbase="192.168.99." +source_path = "/source/glusterfs" +target_path = "/mnt/src" + +if ARGV[0] == "up" + environment = open('vagrant_env.conf', 'w') + + print "\n\e[1;37mEnter Node (or VM) Count? Default: 1 \e[32m" + while node_count < 1 or node_count > 99 + node_count = $stdin.gets.strip.to_i + if node_count == 0 # The user pressed enter without input or we cannot parse the input to a number + node_count = 1 + elsif node_count < 1 + print "\e[31mWe need at least 1 VM ;) Try again \e[32m" + elsif node_count > 99 + print "\e[31mWe don't support more than 99 VMs - Try again \e[32m" + end + end + + print "\e[1;37mEnter per Node Disc (Brick) Count? Default: 2 \e[32m" + + while disk_count < 1 + disk_count = $stdin.gets.strip.to_i + if disk_count == 0 # The user pressed enter without input or we cannot parse the input to a number + disk_count = 2 + elsif disk_count < 1 + print "\e[31mWe need at least 1 disk ;) Try again \e[32m" + end + end + + print "\e[1;37mEnter GlusterFS source location? Default: \"#{source_path}\" : \e[32m" + tmploc = $stdin.gets.strip.to_s + if tmploc != "" + source_path = "#{tmploc}" + end + + environment.puts("# BEWARE: Do NOT modify ANY settings in here or your vagrant environment will be messed up") + environment.puts(node_count.to_s) + environment.puts(disk_count.to_s) + environment.puts(source_path) + + print "\e[32m\nOK I will provision #{node_count} VMs for you and each one will have #{disk_count} disks for bricks\e[37m\n\n" + system "sleep 1" +else # So that we destroy and can connect to all VMs... + environment = open('vagrant_env.conf', 'r') + + environment.readline # Skip the comment on top + node_count = environment.readline.to_i + disk_count = environment.readline.to_i + source_path = environment.readline.gsub(/\s+/, "") + + if ARGV[0] != "ssh-config" + puts "Detected settings from previous vagrant up:" + puts " We deployed #{node_count} VMs with each #{disk_count} disks" + puts "" + end +end + +environment.close + +$ansivar = Hash.new{ |hash,key| hash[key] = [] } +$devnamecreated = false + +# +# Function to create and attach disks +# +def attachDisks(numDisk, provider) + suffix = "bcdefghijklmn".split("") + for i in 1..numDisk.to_i + devname = "vd" + (suffix[i-1]).to_s + if $devnamecreated == false + $ansivar["device"].push "#{devname}" + end + provider.storage :file, + :size => '1G', + :device => "vd" + (suffix[i-1]).to_s, + :type => "qcow2", + :bus => "virtio", + :cache => "default" + end + $devnamecreated = true +end + + +$ansivar["src_path"].push "#{source_path}" +$ansivar["trg_path"].push "#{target_path}" + +groups = Hash.new{ |hash,key| hash[key] = [] } + +groups["origin"].push "#{node_name}1" +groups["all"].push "#{node_name}1" + +(2..node_count).each do |num| + $ansivar["peer_nodes"].push "#{node_name}#{num}" + groups["all"].push "#{node_name}#{num}" +end + +hostsFile = "\n" +(1..node_count).each do |num| + hostsFile += "#{ipbase}#{( 100 + num).to_s} #{node_name}#{num.to_s}\n" +end + +Vagrant.configure("2") do |config| + (1..node_count).each do |num| + config.vm.define "#{node_name}#{num}" do |node| + ip_addr = "#{ipbase}#{(100 + num).to_s}" + node.vm.network "private_network", ip: "#{ip_addr}" + node.vm.box = box_name + node.vm.box_url = box_url + node.vm.hostname = "#{node_name}#{num}" + node.ssh.insert_key = false + node.vm.synced_folder "#{source_path}", "#{target_path}", type: "nfs" + + # Define basic config for VM, memory, cpu, storage pool + node.vm.provider "libvirt" do |virt| + virt.storage_pool_name = "default" + virt.memory = 1024 + virt.cpus = 1 + + attachDisks( disk_count, virt ) + end + + node.vm.post_up_message = "\e[37mBuilding of this VM is finished \n" + "You can access it now with: \n" + "vagrant ssh #{node_name}#{num.to_s}\n\n" + "#{target_path} directory in VM #{node_name}#{num.to_s}" + "is synced with Host machine. \nSo any changes done in this" + "directory will be reflected in the host machine as well\n" + "Beware of this when you delete content from this directory\e[32m" + + node.vm.provision :shell, path: "bootstrap.sh" + + node.vm.provision "shell", inline: <<-SHELL + echo '#{hostsFile}' | sudo tee -a /etc/hosts + SHELL + + if num == node_count + # Let's provision + node.vm.provision "ansible" do |setup| + setup.verbose = "v" + setup.playbook = "ansible/setup.yml" + setup.limit = "all" + setup.sudo = "true" + setup.groups = groups + setup.extra_vars = $ansivar + end + end + + end + end +end diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/cluster/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/cluster/tasks/main.yml new file mode 100644 index 00000000000..3306c7a3dc2 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/cluster/tasks/main.yml @@ -0,0 +1,5 @@ +--- +- name: gluster peer probe + shell: gluster peer probe {{ item }} + with_items: "{{ peer_nodes | default([]) }}" + diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/compile-gluster/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/compile-gluster/tasks/main.yml new file mode 100644 index 00000000000..6ee258c7780 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/compile-gluster/tasks/main.yml @@ -0,0 +1,29 @@ +--- +- name: autogen.sh + shell: chdir={{ item }} ./autogen.sh + with_items: "{{ trg_path }}" + +- name: configure + shell: chdir={{ item }} CFLAGS="-g -O0 -Werror -Wall -Wno-error=cpp -Wno-error=maybe-uninitialized" \ + ./configure \ + --prefix=/usr \ + --exec-prefix=/usr \ + --bindir=/usr/bin \ + --sbindir=/usr/sbin \ + --sysconfdir=/etc \ + --datadir=/usr/share \ + --includedir=/usr/include \ + --libdir=/usr/lib64 \ + --libexecdir=/usr/libexec \ + --localstatedir=/var \ + --sharedstatedir=/var/lib \ + --mandir=/usr/share/man \ + --infodir=/usr/share/info \ + --libdir=/usr/lib64 \ + --enable-debug + with_items: "{{ trg_path }}" + +- name: make install + shell: chdir={{ item }} make install + with_items: "{{ trg_path }}" + diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/install-pkgs/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/install-pkgs/tasks/main.yml new file mode 100644 index 00000000000..3944054dd25 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/install-pkgs/tasks/main.yml @@ -0,0 +1,72 @@ +--- +- name: install deltarpm + dnf: name=deltarpm state=present + +- name: update system + shell: dnf update -y + +- name: install other packages + dnf: name={{ item }} state=present + with_items: + - attr + - autoconf + - automake + - bison + - cifs-utils + - cscope + - ctags + - dbench + - dos2unix + - e2fsprogs + - findutils + - flex + - fuse-devel + - fuse-libs + - gcc + - gdb + - git + - glib2-devel + - hostname + - libacl-devel + - libaio-devel + - libattr-devel + - libibverbs-devel + - librdmacm-devel + - libtool + - libxml2-devel + - lvm2-devel + - make + - man-db + - mock + - net-tools + - nfs-utils + - openssh-server + - openssl-devel + - perl-Test-Harness + - pkgconfig + - procps-ng + - psmisc + - python-devel + - python-eventlet + - python-netifaces + - python-paste-deploy + - python-setuptools + - python-simplejson + - python-sphinx + - python-webob + - pyxattr + - readline-devel + - rpm-build + - screen + - strace + - supervisor + - systemtap-sdt-devel + - sqlite-devel + - samba* + - userspace-rcu-devel + - vim + - wget + - which + - xfsprogs + - yajl-devel + diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/iptables/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/iptables/tasks/main.yml new file mode 100644 index 00000000000..768cb0e8668 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/iptables/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: disable iptables, need to add specific rules later + shell: iptables -F diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/prepare-brick/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/prepare-brick/tasks/main.yml new file mode 100644 index 00000000000..a3a6c463468 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/prepare-brick/tasks/main.yml @@ -0,0 +1,30 @@ +--- +- name: Create physical device + shell: pvcreate /dev/{{ item }} + with_items: "{{ device }}" + +- name: Create volume group + shell: vgcreate vg{{ item }} /dev/{{ item }} + with_items: "{{ device }}" + +- name: Create thin pool + shell: lvcreate -L 950M -T vg{{ item }}/thinpool + with_items: "{{ device }}" + +- name: Create thin volume + shell: lvcreate -V900M -T vg{{ item }}/thinpool -n thinp1 + with_items: "{{ device }}" + +- name: Format backend + filesystem: fstype=xfs dev=/dev/vg{{ item }}/thinp1 + with_items: "{{ device }}" + +- name: Create mount directory + file: path=/bricks/br{{ item }} state=directory recurse=yes + with_items: "{{ device }}" + +- name: Add entry to fstab and mount + mount: name=/bricks/br{{ item }} src=/dev/vg{{ item }}/thinp1 fstype=xfs state=mounted + with_items: "{{ device }}" + + diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/selinux/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/selinux/tasks/main.yml new file mode 100644 index 00000000000..c9ba9618428 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/selinux/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: Allow gfapi in Samba to bind to other ports than well known smb ports + seboolean: name=samba_load_libgfapi state=yes persistent=yes diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/service/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/service/tasks/main.yml new file mode 100644 index 00000000000..ef78a125ae8 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/roles/service/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: disable kernel nfs + service: name=nfs-server enabled=no + +- name: stop kernel nfs + service: name=nfs-server state=stopped + +- name: enable rpcbind + service: name=rpcbind enabled=yes + +- name: start rpcbind + service: name=rpcbind state=started + +- name: enable glusterd + service: name=glusterd enabled=yes + +- name: start glusterd + service: name=glusterd state=started + + + diff --git a/extras/devel-tools/devel-vagrant/ansible/setup.yml b/extras/devel-tools/devel-vagrant/ansible/setup.yml new file mode 100644 index 00000000000..c26bd7d6051 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/ansible/setup.yml @@ -0,0 +1,23 @@ +--- +- hosts: all + become: yes + become_method: sudo + roles: + - install-pkgs + - prepare-brick + - selinux + - iptables + +- hosts: all + become: yes + become_method: sudo + serial: 1 + roles: + - compile-gluster + - service + +- hosts: origin + become: yes + become_method: sudo + roles: + - cluster diff --git a/extras/devel-tools/devel-vagrant/bootstrap.sh b/extras/devel-tools/devel-vagrant/bootstrap.sh new file mode 100644 index 00000000000..bbf9fa2c063 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/bootstrap.sh @@ -0,0 +1,3 @@ +dnf install -y nfs-utils +dnf install -y vim +dnf install -y python2 python2-dnf libselinux-python libsemanage-python diff --git a/extras/devel-tools/devel-vagrant/up.sh b/extras/devel-tools/devel-vagrant/up.sh new file mode 100755 index 00000000000..35cbe79d835 --- /dev/null +++ b/extras/devel-tools/devel-vagrant/up.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +vagrant up --no-provision $@ +vagrant provision diff --git a/extras/devel-tools/gdb_macros b/extras/devel-tools/gdb_macros new file mode 100644 index 00000000000..aae4ccb3b37 --- /dev/null +++ b/extras/devel-tools/gdb_macros @@ -0,0 +1,84 @@ + +# +# gdb_macros is a gdb script file which can assist +# developer in debugging. This script provides +# following functions for debugging gluster processes +# effectively: +# +# pdict : This function will iterate through all the +# dictionary (dict_t) members and print the +# key-value pair. +# +# plist : This function will print address of each member +# of gluster list (list_head). +# +# gdb script should be loaded in gdb before using these +# functions. gdb script can be loaded on-demand or on gdb +# start-up. Use the following command on gdb prompt for +# loading it on-demand. +# source <script filename> +# e.g. +# (gdb) source /mnt/gdb_macros +# +# To automatically load gdb script on startup use .gdbinit +# file. This file is automatically loaded by gdb on start. +# Edit (or create) ~/.gdbinit file and add the following +# entry: +# source /mnt/gdb_macros +# + + + +# This is an internal helper function +define _print_dict_data + set $data = ($arg0) + set $len = $data->len - 1 + set $i = 0 + set $ishex = 0 + while ($i < $len) + set $ch = $data->data [$i] + + # Print only alpha-numeric values as char + # and remaining as hex value. This is done + # in this way because using %s screws up + # the display if the string has non-displayable + # characters + if ($ishex == 0) && ($ch > 31) && ($ch < 127) + printf "%c", $ch + else + printf "%x", ($ch & 0xFF) + set $ishex = 1 + end + set $i = $i + 1 + end +end + + +define pdict + set $cnt = 1 + set $temp = *($arg0->members) + while ($temp) + printf "[%d](%s::",$cnt, $temp->key + _print_dict_data ($temp)->value + printf ")\n" + set $temp = $temp->next + set $cnt = $cnt + 1 + end + printf "Done\n" +end + + +define plist + set $node = &($arg0) + set $head = $node + + printf "[0x%lx]", $node + set $node = $node->next + + while ($node != $head) + printf "--> [0x%lx]", $node + set $node = $node->next + end + printf "\n" +end + diff --git a/extras/devel-tools/print-backtrace.sh b/extras/devel-tools/print-backtrace.sh new file mode 100755 index 00000000000..33fbae288bc --- /dev/null +++ b/extras/devel-tools/print-backtrace.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# sample unresolved backtrace lines picked up from a brick log that should go +# into a backtrace file eg. bt-file.txt: +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3ec81)[0x7fe4bc271c81] +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3eecd)[0x7fe4bc271ecd] +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x404cb)[0x7fe4bc2734cb] +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d2b6)[0x7fe4bc2702b6] +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d323)[0x7fe4bc270323] +# +# following is the output of the script for the above backtrace lines: +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3ec81)[0x7fe4bc271c81] __afr_selfheal_data_finalize_source inlined at /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:684 in __afr_selfheal_data_prepare /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:603 +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3eecd)[0x7fe4bc271ecd] __afr_selfheal_data /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:740 +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x404cb)[0x7fe4bc2734cb] afr_selfheal_data /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:883 +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d2b6)[0x7fe4bc2702b6] afr_selfheal_do /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-common.c:1968 +# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d323)[0x7fe4bc270323] afr_selfheal /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-common.c:2015 +# +# Usage with debuginfo RPM: +# print-backtrace.sh $HOME/Downloads/glusterfs-debuginfo-3.8.4-10.el7.x86_64.rpm bt-file.txt +# +# Usage with source install: +# print-packtrace.sh none bt-file.txt + +function version_compare() { test $(echo $1|awk -F '.' '{print $1 $2 $3}') -gt $(echo $2|awk -F '.' '{print $1 $2 $3}'); } + +function Usage() +{ + echo -e "Usage:\n\t$0 { none | <debuginfo-rpm> } <backtrace-file>" + echo "none: implies we don't have a debuginfo rpm but want to resolve" + echo " against a source install which already has the debuginfo" + echo " NOTE: in this case you should have configured the build" + echo " with --enable-debug and the linker options should" + echo " have the option -rdynamic" +} + +debuginfo_rpm=$1 +backtrace_file=$2 + +if [ ! $debuginfo_rpm ] || [ ! $backtrace_file ]; then + Usage + exit 1 +fi + +if [ $debuginfo_rpm != "none" ]; then + if [ ! -f $debuginfo_rpm ]; then + echo "no such rpm file: $debuginfo_rpm" + exit 1 + fi +fi + +if [ ! -f $backtrace_file ]; then + echo "no such backtrace file: $backtrace_file" + exit 1 +fi + +if [ "$debuginfo_rpm" != "none" ]; then + if ! file $debuginfo_rpm | grep RPM >/dev/null 2>&1 ; then + echo "file does not look like an rpm: $debuginfo_rpm" + exit 1 + fi +fi + +cpio_version=$(cpio --version|grep cpio|cut -f 2 -d ')'|sed -e 's/^[[:space:]]*//') +rpm_name="" +debuginfo_path="" +debuginfo_extension="" + +if [ $debuginfo_rpm != "none" ]; then + # extract the gluster debuginfo rpm to resolve the symbols against + rpm_name=$(basename $debuginfo_rpm '.rpm') + if [ -d $rpm_name ]; then + echo "directory already exists: $rpm_name" + echo "please remove/move it and reattempt" + exit 1 + fi + mkdir -p $rpm_name + if version_compare $cpio_version "2.11"; then + rpm2cpio $debuginfo_rpm | cpio --quiet --extract --make-directories --preserve-modification-time --directory=$rpm_name + ret=$? + else + current_dir="$PWD" + cd $rpm_name + rpm2cpio $debuginfo_rpm | cpio --quiet --extract --make-directories --preserve-modification-time + ret=$? + cd $current_dir + fi + if [ $ret -eq 1 ]; then + echo "failed to extract rpm $debuginfo_rpm to $PWD/$rpm_name directory" + rm -rf $rpm_name + exit 1 + fi + debuginfo_path="$PWD/$rpm_name/usr/lib/debug" + debuginfo_extension=".debug" +else + debuginfo_path="" + debuginfo_extension="" +fi + +# NOTE: backtrace file should contain only the lines which need to be resolved +for bt in $(cat $backtrace_file) +do + libname=$(echo $bt | cut -f 1 -d '(') + addr=$(echo $bt | cut -f 2 -d '(' | cut -f 1 -d ')') + libpath=${debuginfo_path}${libname}${debuginfo_extension} + if [ ! -f $libpath ]; then + continue + fi + newbt=( $(eu-addr2line --functions --exe=$libpath $addr) ) + echo "$bt ${newbt[*]}" +done + +# remove the temporary directory +if [ -d $rpm_name ]; then + rm -rf $rpm_name +fi + diff --git a/extras/devel-tools/strace-brick.sh b/extras/devel-tools/strace-brick.sh new file mode 100755 index 00000000000..a140729111c --- /dev/null +++ b/extras/devel-tools/strace-brick.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Usage: +# nice -n -19 strace-brick.sh glusterfsd 50 + +brick_process_name=$1 +min_watch_cpu=$2 +if [ ! $brick_process_name ]; then + brick_process_name=glusterfsd +fi + +if [ ! $min_watch_cpu ]; then + min_watch_cpu=50 +fi + +echo "min_watch_cpu: $min_watch_cpu" + +break=false + +while ! $break; +do + mypids=( $(pgrep $brick_process_name) ) + echo "mypids: ${mypids[*]}" + + pid_args=$(echo ${mypids[*]} | sed -e 's/ / -p /g;s/^/-p /') + echo "pid_args: $pid_args" + + pcpu=( $(ps $pid_args -o pcpu -h ) ) + echo "pcpu: ${pcpu[*]}" + + wait_longer=false + + for i in $( seq 0 $((${#pcpu[*]} - 1)) ) + do + echo "i: $i" + echo "mypids[$i]: ${mypids[$i]}" + + int_pcpu=$(echo ${pcpu[$i]} | cut -f 1 -d '.') + echo "int_pcpu: $int_pcpu" + if [ ! $int_pcpu ] || [ ! $min_watch_cpu ]; then + break=true + echo "breaking" + fi + if [ $int_pcpu -ge $min_watch_cpu ]; then + wait_longer=true + mydirname="${brick_process_name}-${mypids[$i]}-$(date --utc +'%Y%m%d-%H%M%S.%N')" + $(mkdir $mydirname && cd $mydirname && timeout --kill-after=5 --signal=KILL 60 nice -n -19 strace -p ${mypids[$i]} -ff -tt -T -o $brick_process_name) & + fi + done + + if $wait_longer; then + sleep 90 + else + sleep 15 + fi +done diff --git a/extras/distributed-testing/README b/extras/distributed-testing/README new file mode 100644 index 00000000000..928d943f211 --- /dev/null +++ b/extras/distributed-testing/README @@ -0,0 +1,28 @@ +PROBLEM + +The testing methodology of Gluster is extremely slow. It takes a very long time (6+ hrs) to run the basic tests on a single machine. It takes about 20+ hours to run code analysis version of tests like valgrind, asan, tsan etc. + +SOLUTION + +The fundamental problem is that the tests cannot be parallelized on a single machine. The natural solution is to run these tests on a cluster of machines. In a nutshell, apply map-reduce to run unit tests. + +WORK @ Facebook + +At Facebook we have applied the map-reduce approach to testing and have observed 10X improvements. + +The solution supports the following + +Distribute tests across machines, collect results/logs +Share worker pool across different testers +Try failure 3 times on 3 different machines before calling it a failure +Support running asan, valgrind, asan-noleaks +Self management of worker pools. The clients will manage the worker pool including version update, no manual maintenance required +WORK + +Port the code from gluster-fb-3.8 to gluster master + +HOW TO RUN + +./extras/distributed-testing/distributed-test.sh --hosts '<h1> <h2> <h3>' + +All hosts should have no password for ssh via root. This can be achieved with keys setup on the client and the server machines. diff --git a/extras/distributed-testing/distributed-test-build-env b/extras/distributed-testing/distributed-test-build-env new file mode 100644 index 00000000000..cd68ff717da --- /dev/null +++ b/extras/distributed-testing/distributed-test-build-env @@ -0,0 +1,20 @@ +#!/bin/bash + +GF_CONF_OPTS="--localstatedir=/var --sysconfdir /var/lib --prefix /usr --libdir /usr/lib64 \ + --enable-bd-xlator=yes --enable-debug --enable-gnfs" + +if [ -x /usr/lib/rpm/redhat/dist.sh ]; then + REDHAT_MAJOR=$(/usr/lib/rpm/redhat/dist.sh --distnum) +else + REDHAT_MAJOR=0 +fi + +ASAN_ENABLED=${ASAN_ENABLED:=0} +if [ "$ASAN_ENABLED" -eq "1" ]; then + GF_CONF_OPTS="$GF_CONF_OPTS --with-asan" +fi + +GF_CONF_OPTS="$GF_CONF_OPTS --with-systemd" +export GF_CONF_OPTS + +export CFLAGS="-O0 -ggdb -fPIC -Wall" diff --git a/extras/distributed-testing/distributed-test-build.sh b/extras/distributed-testing/distributed-test-build.sh new file mode 100755 index 00000000000..e8910d8425c --- /dev/null +++ b/extras/distributed-testing/distributed-test-build.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +EXTRA_CONFIGURE_ARGS="$@" +ASAN_REQUESTED=false +for arg in $EXTRA_CONFIGURE_ARGS; do + if [ $arg == "--with-asan" ]; then + echo "Requested ASAN, cleaning build first." + make -j distclean || true + touch .with_asan + ASAN_REQUESTED=true + fi +done + +if [ $ASAN_REQUESTED == false ]; then + if [ -f .with_asan ]; then + echo "Previous build was with ASAN, cleaning build first." + make -j distclean || true + rm -v .with_asan + fi +fi + +source extras/distributed-testing/distributed-test-build-env +./autogen.sh +./configure $GF_CONF_OPTS $EXTRA_CONFIGURE_ARGS +make -j diff --git a/extras/distributed-testing/distributed-test-env b/extras/distributed-testing/distributed-test-env new file mode 100644 index 00000000000..36fdd82e5dd --- /dev/null +++ b/extras/distributed-testing/distributed-test-env @@ -0,0 +1,48 @@ +#!/bin/bash + +SMOKE_TESTS="\ + tests/basic/*.t\ + tests/basic/afr/*.t\ + tests/basic/distribute/*.t\ + tests/bugs/fb*.t\ + tests/features/brick-min-free-space.t\ +" + +KNOWN_FLAKY_TESTS="\ +" + +BROKEN_TESTS="\ + tests/features/lock_revocation.t\ + tests/features/recon.t\ + tests/features/fdl-overflow.t\ + tests/features/fdl.t\ + tests/features/ipc.t\ + tests/bugs/distribute/bug-1247563.t\ + tests/bugs/distribute/bug-1543279.t\ + tests/bugs/distribute/bug-1066798.t\ + tests/bugs/ec/bug-1304988.t\ + tests/bugs/unclassified/bug-1357397.t\ + tests/bugs/quota/bug-1235182.t\ + tests/bugs/fuse/bug-1309462.t\ + tests/bugs/glusterd/bug-1238706-daemons-stop-on-peer-cleanup.t\ + tests/bugs/stripe/bug-1002207.t\ + tests/bugs/stripe/bug-1111454.t\ + tests/bugs/snapshot/bug-1140162-file-snapshot-features-encrypt-opts-validation.t\ + tests/bugs/write-behind/bug-1279730.t\ + tests/bugs/gfapi/bug-1093594.t\ + tests/bugs/replicate/bug-1473026.t\ + tests/bugs/replicate/bug-802417.t\ + tests/basic/inode-leak.t\ + tests/basic/distribute/force-migration.t\ + tests/basic/ec/heal-info.t\ + tests/basic/ec/ec-seek.t\ + tests/basic/jbr/jbr-volgen.t\ + tests/basic/jbr/jbr.t\ + tests/basic/afr/tarissue.t\ + tests/basic/tier/tierd_check.t\ + tests/basic/gfapi/bug1291259.t\ +" + +SMOKE_TESTS=$(echo $SMOKE_TESTS | tr -s ' ' ' ') +KNOWN_FLAKY_TESTS=$(echo $KNOWN_FLAKY_TESTS | tr -s ' ' ' ') +BROKEN_TESTS=$(echo $BROKEN_TESTS | tr -s ' ' ' ') diff --git a/extras/distributed-testing/distributed-test-runner.py b/extras/distributed-testing/distributed-test-runner.py new file mode 100755 index 00000000000..5a07e2feab1 --- /dev/null +++ b/extras/distributed-testing/distributed-test-runner.py @@ -0,0 +1,859 @@ +#!/usr/bin/python3 + +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +import re +import sys +import fcntl +import base64 +import threading +import socket +import os +import shlex +import argparse +import subprocess +import time +import SimpleXMLRPCServer +import xmlrpclib +import md5 +import httplib +import uuid + +DEFAULT_PORT = 9999 +TEST_TIMEOUT_S = 15 * 60 +CLIENT_CONNECT_TIMEOUT_S = 10 +CLIENT_TIMEOUT_S = 60 +PATCH_FILE_UID = str(uuid.uuid4()) +SSH_TIMEOUT_S = 10 +MAX_ATTEMPTS = 3 +ADDRESS_FAMILY = 'IPv4' + + +def socket_instance(address_family): + if address_family.upper() == 'ipv4'.upper(): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + elif address_family.upper() == 'ipv6'.upper(): + return socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + else: + Log.error("Invalid IP address family") + sys.exit(1) + + +def patch_file(): + return "/tmp/%s-patch.tar.gz" % PATCH_FILE_UID + +# .............................................................................. +# SimpleXMLRPCServer IPvX Wrapper +# .............................................................................. + + +class GeneralXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer): + def __init__(self, addr): + SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, addr) + + def server_bind(self): + if self.socket: + self.socket.close() + self.socket = socket_instance(args.address_family) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + SimpleXMLRPCServer.SimpleXMLRPCServer.server_bind(self) + + +class HTTPConnection(httplib.HTTPConnection): + def __init__(self, host): + self.host = host + httplib.HTTPConnection.__init__(self, host) + + def connect(self): + old_timeout = socket.getdefaulttimeout() + self.sock = socket.create_connection((self.host, self.port), + timeout=CLIENT_CONNECT_TIMEOUT_S) + self.sock.settimeout(old_timeout) + + +class IPTransport(xmlrpclib.Transport): + def __init__(self, *args, **kwargs): + xmlrpclib.Transport.__init__(self, *args, **kwargs) + + def make_connection(self, host): + return HTTPConnection(host) + + +# .............................................................................. +# Common +# .............................................................................. + + +class Timer: + def __init__(self): + self.start = time.time() + + def elapsed_s(self): + return int(time.time() - self.start) + + def reset(self): + ret = self.elapsed_s() + self.start = time.time() + return ret + + +def encode(buf): + return base64.b16encode(buf) + + +def decode(buf): + return base64.b16decode(buf) + + +def get_file_content(path): + with open(path, "r") as f: + return f.read() + + +def write_to_file(path, data): + with open(path, "w") as f: + f.write(data) + + +def failsafe(fn, args=()): + try: + return (True, fn(*args)) + except (xmlrpclib.Fault, xmlrpclib.ProtocolError, xmlrpclib.ResponseError, + Exception) as err: + Log.debug(str(err)) + return (False, None) + + +class LogLevel: + DEBUG = 2 + ERROR = 1 + CLI = 0 + + +class Log: + LOGLEVEL = LogLevel.ERROR + + @staticmethod + def _normalize(msg): + return msg[:100] + + @staticmethod + def debug(msg): + if Log.LOGLEVEL >= LogLevel.DEBUG: + sys.stdout.write("<debug> %s\n" % Log._normalize(msg)) + sys.stdout.flush() + + @staticmethod + def error(msg): + sys.stderr.write("<error> %s\n" % Log._normalize(msg)) + + @staticmethod + def header(msg): + sys.stderr.write("* %s *\n" % Log._normalize(msg)) + + @staticmethod + def cli(msg): + sys.stderr.write("%s\n" % msg) + + +class Shell: + def __init__(self, cwd=None, logpath=None): + self.cwd = cwd + self.shell = True + self.redirect = open(os.devnull if not logpath else logpath, "wr+") + + def __del__(self): + self.redirect.close() + + def cd(self, cwd): + Log.debug("cd %s" % cwd) + self.cwd = cwd + + def truncate(self): + self.redirect.truncate(0) + + def read_logs(self): + self.redirect.seek(0) + return self.redirect.read() + + def check_call(self, cmd): + status = self.call(cmd) + if status: + raise Exception("Error running command %s. status=%s" + % (cmd, status)) + + def call(self, cmd): + if isinstance(cmd, list): + return self._calls(cmd) + + return self._call(cmd) + + def ssh(self, hostname, cmd, id_rsa=None): + flags = "" if not id_rsa else "-i " + id_rsa + return self.call("timeout %s ssh %s root@%s \"%s\"" % + (SSH_TIMEOUT_S, flags, hostname, cmd)) + + def scp(self, hostname, src, dest, id_rsa=None): + flags = "" if not id_rsa else "-i " + id_rsa + return self.call("timeout %s scp %s %s root@%s:%s" % + (SSH_TIMEOUT_S, flags, src, hostname, dest)) + + def output(self, cmd, cwd=None): + Log.debug("%s> %s" % (cwd, cmd)) + return subprocess.check_output(shlex.split(cmd), cwd=self.cwd) + + def _calls(self, cmds): + Log.debug("Running commands. %s" % cmds) + for c in cmds: + status = self.call(c) + if status: + Log.error("Commands failed with %s" % status) + return status + return 0 + + def _call(self, cmd): + if not self.shell: + cmd = shlex.split(cmd) + + Log.debug("%s> %s" % (self.cwd, cmd)) + + status = subprocess.call(cmd, cwd=self.cwd, shell=self.shell, + stdout=self.redirect, stderr=self.redirect) + + Log.debug("return %s" % status) + return status + + +# .............................................................................. +# Server role +# .............................................................................. + +class TestServer: + def __init__(self, port, scratchdir): + self.port = port + self.scratchdir = scratchdir + self.shell = Shell() + self.rpc = None + self.pidf = None + + self.shell.check_call("mkdir -p %s" % self.scratchdir) + self._process_lock() + + def __del__(self): + if self.pidf: + self.pidf.close() + + def init(self): + Log.debug("Starting xmlrpc server on port %s" % self.port) + self.rpc = GeneralXMLRPCServer(("", self.port)) + self.rpc.register_instance(Handlers(self.scratchdir)) + + def serve(self): + (status, _) = failsafe(self.rpc.serve_forever) + Log.cli("== End ==") + + def _process_lock(self): + pid_filename = os.path.basename(__file__).replace("/", "-") + pid_filepath = "%s/%s.pid" % (self.scratchdir, pid_filename) + self.pidf = open(pid_filepath, "w") + try: + fcntl.lockf(self.pidf, fcntl.LOCK_EX | fcntl.LOCK_NB) + # We have the lock, kick anybody listening on this port + self.shell.call("kill $(lsof -t -i:%s)" % self.port) + except IOError: + Log.error("Another process instance is running") + sys.exit(0) + +# +# Server Handler +# + + +handler_lock = threading.Lock() +handler_serving_since = Timer() + + +def synchronized(func): + def decorator(*args, **kws): + handler_lock.acquire() + h = args[0] + try: + h.shell.truncate() + ret = func(*args, **kws) + return ret + except Exception() as err: + Log.error(str(err)) + Log.error(decode(h._log_content())) + raise + finally: + handler_lock.release() + handler_serving_since.reset() + + return decorator + + +class Handlers: + def __init__(self, scratchdir): + self.client_id = None + self.scratchdir = scratchdir + self.gluster_root = "%s/glusterfs" % self.scratchdir + self.shell = Shell(logpath="%s/test-handlers.log" % self.scratchdir) + + def hello(self, id): + if not handler_lock.acquire(False): + return False + try: + return self._hello_locked(id) + finally: + handler_lock.release() + + def _hello_locked(self, id): + if handler_serving_since.elapsed_s() > CLIENT_TIMEOUT_S: + Log.debug("Disconnected client %s" % self.client_id) + self.client_id = None + + if not self.client_id: + self.client_id = id + handler_serving_since.reset() + return True + + return (id == self.client_id) + + @synchronized + def ping(self, id=None): + if id: + return id == self.client_id + return True + + @synchronized + def bye(self, id): + assert id == self.client_id + self.client_id = None + handler_serving_since.reset() + return True + + @synchronized + def cleanup(self, id): + assert id == self.client_id + self.shell.cd(self.gluster_root) + self.shell.check_call("PATH=.:$PATH; sudo ./clean_gfs_devserver.sh") + return True + + @synchronized + def copy(self, id, name, content): + with open("%s/%s" % (self.scratchdir, name), "w+") as f: + f.write(decode(content)) + return True + + @synchronized + def copygzip(self, id, content): + assert id == self.client_id + gzipfile = "%s/tmp.tar.gz" % self.scratchdir + tarfile = "%s/tmp.tar" % self.scratchdir + self.shell.check_call("rm -f %s" % gzipfile) + self.shell.check_call("rm -f %s" % tarfile) + write_to_file(gzipfile, decode(content)) + + self.shell.cd(self.scratchdir) + self.shell.check_call("rm -r -f %s" % self.gluster_root) + self.shell.check_call("mkdir -p %s" % self.gluster_root) + + self.shell.cd(self.gluster_root) + cmds = [ + "gunzip -f -q %s" % gzipfile, + "tar -xvf %s" % tarfile + ] + return self.shell.call(cmds) == 0 + + @synchronized + def build(self, id, asan=False): + assert id == self.client_id + self.shell.cd(self.gluster_root) + self.shell.call("make clean") + env = "ASAN_ENABLED=1" if asan else "" + return self.shell.call( + "%s ./extras/distributed-testing/distributed-test-build.sh" % env) == 0 + + @synchronized + def install(self, id): + assert id == self.client_id + self.shell.cd(self.gluster_root) + return self.shell.call("make install") == 0 + + @synchronized + def prove(self, id, test, timeout, valgrind="no", asan_noleaks=True): + assert id == self.client_id + self.shell.cd(self.gluster_root) + env = "DEBUG=1 " + if valgrind == "memcheck" or valgrind == "yes": + cmd = "valgrind" + cmd += " --tool=memcheck --leak-check=full --track-origins=yes" + cmd += " --show-leak-kinds=all -v prove -v" + elif valgrind == "drd": + cmd = "valgrind" + cmd += " --tool=drd -v prove -v" + elif asan_noleaks: + cmd = "prove -v" + env += "ASAN_OPTIONS=detect_leaks=0 " + else: + cmd = "prove -v" + + status = self.shell.call( + "%s timeout %s %s %s" % (env, timeout, cmd, test)) + + if status != 0: + return (False, self._log_content()) + return (True, "") + + def _log_content(self): + return encode(self.shell.read_logs()) + +# .............................................................................. +# Cli role +# .............................................................................. + + +class RPCConnection((threading.Thread)): + def __init__(self, host, port, path, cb): + threading.Thread.__init__(self) + self.host = host + self.port = port + self.path = path + self.shell = Shell() + self.cb = cb + self.stop = False + self.proxy = None + self.logid = "%s:%s" % (self.host, self.port) + + def connect(self): + (status, ret) = failsafe(self._connect) + return (status and ret) + + def _connect(self): + url = "http://%s:%s" % (self.host, self.port) + self.proxy = xmlrpclib.ServerProxy(url, transport=IPTransport()) + return self.proxy.hello(self.cb.id) + + def disconnect(self): + self.stop = True + + def ping(self): + return self.proxy.ping() + + def init(self): + return self._copy() and self._compile_and_install() + + def run(self): + (status, ret) = failsafe(self.init) + if not status: + self.cb.note_lost_connection(self) + return + elif not ret: + self.cb.note_setup_failed(self) + return + + while not self.stop: + (status, ret) = failsafe(self._run) + if not status or not ret: + self.cb.note_lost_connection(self) + break + time.sleep(0) + + failsafe(self.proxy.bye, (self.cb.id,)) + Log.debug("%s connection thread stopped" % self.host) + + def _run(self): + test = self.cb.next_test() + (status, _) = failsafe(self._execute_next, (test,)) + if not status: + self.cb.note_retry(test) + return False + return True + + def _execute_next(self, test): + if not test: + time.sleep(1) + return + + (status, error) = self.proxy.prove(self.cb.id, test, + self.cb.test_timeout, + self.cb.valgrind, + self.cb.asan_noleaks) + if status: + self.cb.note_done(test) + else: + self.cb.note_error(test, error) + + def _compile_and_install(self): + Log.debug("<%s> Build " % self.logid) + asan = self.cb.asan or self.cb.asan_noleaks + return (self.proxy.build(self.cb.id, asan) and + self.proxy.install(self.cb.id)) + + def _copy(self): + return self._copy_gzip() + + def _copy_gzip(self): + Log.cli("<%s> copying and compiling %s to remote" % + (self.logid, self.path)) + data = encode(get_file_content(patch_file())) + Log.debug("GZIP size = %s B" % len(data)) + return self.proxy.copygzip(self.cb.id, data) + + +class RPCConnectionPool: + def __init__(self, gluster_path, hosts, n, id_rsa): + self.gluster_path = gluster_path + self.hosts = hosts + self.conns = [] + self.faulty = [] + self.n = int(len(hosts) / 2) + 1 if not n else n + self.id_rsa = id_rsa + self.stop = False + self.scanner = threading.Thread(target=self._scan_hosts_loop) + self.kicker = threading.Thread(target=self._kick_hosts_loop) + self.shell = Shell() + self.since_start = Timer() + + self.shell.check_call("rm -f %s" % patch_file()) + self.shell.check_call("tar -zcvf %s ." % patch_file()) + self.id = md5.new(get_file_content(patch_file())).hexdigest() + Log.cli("client UID %s" % self.id) + Log.cli("patch UID %s" % PATCH_FILE_UID) + + def __del__(self): + self.shell.check_call("rm -f %s" % patch_file()) + + def pool_status(self): + elapsed_m = int(self.since_start.elapsed_s() / 60) + return "%s/%s connected, %smin elapsed" % (len(self.conns), self.n, + elapsed_m) + + def connect(self): + Log.debug("Starting scanner") + self.scanner.start() + self.kicker.start() + + def disconnect(self): + self.stop = True + for conn in self.conns: + conn.disconnect() + + def note_lost_connection(self, conn): + Log.cli("lost connection to %s" % conn.host) + self.conns.remove(conn) + self.hosts.append((conn.host, conn.port)) + + def note_setup_failed(self, conn): + Log.error("Setup failed on %s:%s" % (conn.host, conn.port)) + self.conns.remove(conn) + self.faulty.append((conn.host, conn.port)) + + def _scan_hosts_loop(self): + Log.debug("Scanner thread started") + while not self.stop: + failsafe(self._scan_hosts) + time.sleep(5) + + def _scan_hosts(self): + if len(self.hosts) == 0 and len(self.conns) == 0: + Log.error("no more hosts available to loadbalance") + sys.exit(1) + + for (host, port) in self.hosts: + if (len(self.conns) >= self.n) or self.stop: + break + self._scan_host(host, port) + + def _scan_host(self, host, port): + Log.debug("scanning %s:%s" % (host, port)) + c = RPCConnection(host, port, self.gluster_path, self) + (status, result) = failsafe(c.connect) + if status and result: + self.hosts.remove((host, port)) + Log.debug("Connected to %s:%s" % (host, port)) + self.conns.append(c) + c.start() + Log.debug("%s / %s connected" % (len(self.conns), self.n)) + else: + Log.debug("Failed to connect to %s:%s" % (host, port)) + + def _kick_hosts_loop(self): + Log.debug("Kick thread started") + while not self.stop: + time.sleep(10) + failsafe(self._kick_hosts) + + Log.debug("Kick thread stopped") + + def _is_pingable(self, host, port): + c = RPCConnection(host, port, self.gluster_path, self) + failsafe(c.connect) + (status, result) = failsafe(c.ping) + return status and result + + def _kick_hosts(self): + # Do not kick hosts if we have the optimal number of connections + if (len(self.conns) >= self.n) or self.stop: + Log.debug("Skip kicking hosts") + return + + # Check and if dead kick all hosts + for (host, port) in self.hosts: + if self.stop: + Log.debug("Break kicking hosts") + break + + if self._is_pingable(host, port): + Log.debug("Host=%s is alive. Won't kick" % host) + continue + + Log.debug("Kicking %s" % host) + mypath = sys.argv[0] + myname = os.path.basename(mypath) + destpath = "/tmp/%s" % myname + sh = Shell() + sh.scp(host, mypath, destpath, self.id_rsa) + sh.ssh(host, "nohup %s --server &>> %s.log &" % + (destpath, destpath), self.id_rsa) + + def join(self): + self.scanner.join() + self.kicker.join() + for c in self.conns: + c.join() + + +# .............................................................................. +# test role +# .............................................................................. + +class TestRunner(RPCConnectionPool): + def __init__(self, gluster_path, hosts, n, tests, flaky_tests, valgrind, + asan, asan_noleaks, id_rsa, test_timeout): + RPCConnectionPool.__init__(self, gluster_path, self._parse_hosts(hosts), + n, id_rsa) + self.flaky_tests = flaky_tests.split(" ") + self.pending = [] + self.done = [] + self.error = [] + self.retry = {} + self.error_logs = [] + self.stats_timer = Timer() + self.valgrind = valgrind + self.asan = asan + self.asan_noleaks = asan_noleaks + self.test_timeout = test_timeout + + self.tests = self._get_tests(tests) + + Log.debug("tests: %s" % self.tests) + + def _get_tests(self, tests): + if not tests or tests == "all": + return self._not_flaky(self._all()) + elif tests == "flaky": + return self.flaky_tests + else: + return self._not_flaky(tests.strip().split(" ")) + + def run(self): + self.connect() + self.join() + return len(self.error) + + def _pretty_print(self, data): + if isinstance(data, list): + str = "" + for i in data: + str = "%s %s" % (str, i) + return str + return "%s" % data + + def print_result(self): + Log.cli("== RESULTS ==") + Log.cli("SUCCESS : %s" % len(self.done)) + Log.cli("ERRORS : %s" % len(self.error)) + Log.cli("== ERRORS ==") + Log.cli(self._pretty_print(self.error)) + Log.cli("== LOGS ==") + Log.cli(self._pretty_print(self.error_logs)) + Log.cli("== END ==") + + def next_test(self): + if len(self.tests): + test = self.tests.pop() + self.pending.append(test) + return test + + if not len(self.pending): + self.disconnect() + + return None + + def _pct_completed(self): + total = len(self.tests) + len(self.pending) + len(self.done) + total += len(self.error) + completed = len(self.done) + len(self.error) + return 0 if not total else int(completed / total * 100) + + def note_done(self, test): + Log.cli("%s PASS (%s%% done) (%s)" % (test, self._pct_completed(), + self.pool_status())) + self.pending.remove(test) + self.done.append(test) + if test in self.retry: + del self.retry[test] + + def note_error(self, test, errstr): + Log.cli("%s FAIL" % test) + self.pending.remove(test) + if test not in self.retry: + self.retry[test] = 1 + + if errstr: + path = "%s/%s-%s.log" % ("/tmp", test.replace("/", "-"), + self.retry[test]) + failsafe(write_to_file, (path, decode(errstr),)) + self.error_logs.append(path) + + if self.retry[test] < MAX_ATTEMPTS: + self.retry[test] += 1 + Log.debug("retry test %s attempt %s" % (test, self.retry[test])) + self.tests.append(test) + else: + Log.debug("giveup attempt test %s" % test) + del self.retry[test] + self.error.append(test) + + def note_retry(self, test): + Log.cli("retry %s on another host" % test) + self.pending.remove(test) + self.tests.append(test) + + # + # test classifications + # + def _all(self): + return self._list_tests(["tests"], recursive=True) + + def _not_flaky(self, tests): + for t in self.flaky_tests: + if t in tests: + tests.remove(t) + return tests + + def _list_tests(self, prefixes, recursive=False, ignore_ifnotexist=False): + tests = [] + for prefix in prefixes: + real_path = "%s/%s" % (self.gluster_path, prefix) + if not os.path.exists(real_path) and ignore_ifnotexist: + continue + for f in os.listdir(real_path): + if os.path.isdir(real_path + "/" + f): + if recursive: + tests += self._list_tests([prefix + "/" + f], recursive) + else: + if re.match(r".*\.t$", f): + tests += [prefix + "/" + f] + return tests + + def _parse_hosts(self, hosts): + ret = [] + for h in args.hosts.split(" "): + ret.append((h, DEFAULT_PORT)) + Log.debug(ret) + return ret + +# .............................................................................. +# Roles entry point +# .............................................................................. + + +def run_as_server(args): + if not args.server_path: + Log.error("please provide server path") + return 1 + + server = TestServer(args.port, args.server_path) + server.init() + server.serve() + return 0 + + +def run_as_tester(args): + Log.header("GLUSTER TEST CLI") + + Log.debug("args = %s" % args) + + tests = TestRunner(args.gluster_path, args.hosts, args.n, args.tests, + args.flaky_tests, valgrind=args.valgrind, + asan=args.asan, asan_noleaks=args.asan_noleaks, + id_rsa=args.id_rsa, test_timeout=args.test_timeout) + result = tests.run() + tests.print_result() + return result + +# .............................................................................. +# main +# .............................................................................. + + +def main(args): + if args.v: + Log.LOGLEVEL = LogLevel.DEBUG + + if args.server and args.tester: + Log.error("Invalid arguments. More than one role specified") + sys.exit(1) + + if args.server: + sys.exit(run_as_server(args)) + elif args.tester: + sys.exit(run_as_tester(args)) + else: + Log.error("please specify a mode for CI") + parser.print_help() + sys.exit(1) + + +parser = argparse.ArgumentParser(description="Gluster CI") + +# server role +parser.add_argument("--server", help="start server", action="store_true") +parser.add_argument("--server_path", help="server scratch space", + default="/tmp/gluster-test") +parser.add_argument("--host", help="server address to listen", default="") +parser.add_argument("--port", help="server port to listen", + type=int, default=DEFAULT_PORT) +# test role +parser.add_argument("--tester", help="start tester", action="store_true") +parser.add_argument("--valgrind[=memcheck,drd]", + help="run tests with valgrind tool 'memcheck' or 'drd'", + default="no") +parser.add_argument("--asan", help="test with asan enabled", + action="store_true") +parser.add_argument("--asan-noleaks", help="test with asan but no mem leaks", + action="store_true") +parser.add_argument("--tests", help="all/flaky/list of tests", default=None) +parser.add_argument("--flaky_tests", help="list of flaky tests", default=None) +parser.add_argument("--n", help="max number of machines to use", type=int, + default=0) +parser.add_argument("--hosts", help="list of worker machines") +parser.add_argument("--gluster_path", help="gluster path to test", + default=os.getcwd()) +parser.add_argument("--id-rsa", help="private key to use for ssh", + default=None) +parser.add_argument("--test-timeout", + help="test timeout in sec (default 15min)", + default=TEST_TIMEOUT_S) +# general +parser.add_argument("-v", help="verbose", action="store_true") +parser.add_argument("--address_family", help="IPv6 or IPv4 to use", + default=ADDRESS_FAMILY) + +args = parser.parse_args() + +main(args) diff --git a/extras/distributed-testing/distributed-test.sh b/extras/distributed-testing/distributed-test.sh new file mode 100755 index 00000000000..8f1e0310f33 --- /dev/null +++ b/extras/distributed-testing/distributed-test.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +source ./extras/distributed-testing/distributed-test-env + +N=0 +TESTS='all' +FLAKY=$KNOWN_FLAKY_TESTS +BROKEN=$BROKEN_TESTS +TEST_TIMEOUT_S=900 +ADDRESS_FAMILY='IPv4' + +FLAGS="" + +function print_env { + echo "Settings:" + echo "N=$N" + echo -e "-------\nHOSTS\n$HOSTS\n-------" + echo -e "TESTS\n$TESTS\n-------" + echo -e "SKIP\n$FLAKY $BROKEN\n-------" + echo -e "TEST_TIMEOUT_S=$TEST_TIMEOUT_S s\n" +} + +function cleanup { + rm -f /tmp/test*.log +} + +function usage { + echo "Usage: $0 [-h or --help] [-v or --verbose] + [--all] [--flaky] [--smoke] [--broken] + [--valgrind] [--asan] [--asan-noleaks] + [--hosts <hosts>] [-n <parallelism>] + [--tests <tests>] + [--id-rsa <ssh private key>] + [--address_family <IPv4 or IPv6>] + " +} + +function parse_args () { + args=`getopt \ + -o hvn: \ + --long help,verbose,address_family:,valgrind,asan,asan-noleaks,all,\ +smoke,flaky,broken,hosts:,tests:,id-rsa:,test-timeout: \ + -n 'fb-remote-test.sh' -- "$@"` + + if [ $? != 0 ]; then + echo "Error parsing getopt" + exit 1 + fi + + eval set -- "$args" + + while true; do + case "$1" in + -h | --help) usage ; exit 1 ;; + -v | --verbose) FLAGS="$FLAGS -v" ; shift ;; + --address_family) ADDRESS_FAMILY=$2; shift 2 ;; + --valgrind) FLAGS="$FLAGS --valgrind" ; shift ;; + --asan-noleaks) FLAGS="$FLAGS --asan-noleaks"; shift ;; + --asan) FLAGS="$FLAGS --asan" ; shift ;; + --hosts) HOSTS=$2; shift 2 ;; + --tests) TESTS=$2; FLAKY= ; BROKEN= ; shift 2 ;; + --test-timeout) TEST_TIMEOUT_S=$2; shift 2 ;; + --all) TESTS='all' ; shift 1 ;; + --flaky) TESTS=$FLAKY; FLAKY= ; shift 1 ;; + --smoke) TESTS=$SMOKE_TESTS; shift 1 ;; + --broken) TESTS=$BROKEN_TESTS; FLAKY= ; BROKEN= ; shift 1 ;; + --id-rsa) FLAGS="$FLAGS --id-rsa $2" ; shift 2 ;; + -n) N=$2; shift 2 ;; + *) break ;; + esac + done + run_tests_args="$@" +} + +function main { + parse_args "$@" + + if [ -z "$HOSTS" ]; then + echo "Please provide hosts to run the tests in" + exit -1 + fi + + print_env + + cleanup + + "extras/distributed-testing/distributed-test-runner.py" $FLAGS --tester \ + --n "$N" --hosts "$HOSTS" --tests "$TESTS" \ + --flaky_tests "$FLAKY $BROKEN" --test-timeout "$TEST_TIMEOUT_S" \ + --address_family "$ADDRESS_FAMILY" + + exit $? +} + +main "$@" diff --git a/extras/ec-heal-script/README.md b/extras/ec-heal-script/README.md new file mode 100644 index 00000000000..aaefd6681f6 --- /dev/null +++ b/extras/ec-heal-script/README.md @@ -0,0 +1,69 @@ +# gluster-heal-scripts +Scripts to correct extended attributes of fragments of files to make them healble. + +Following are the guidelines/suggestions to use these scripts. + +1 - Passwordless ssh should be setup for all the nodes of the cluster. + +2 - Scripts should be executed from one of these nodes. + +3 - Make sure NO "IO" is going on for the files for which we are running +these two scripts. + +4 - There should be no heal going on for the file for which xattrs are being +set by correct_pending_heals.sh. Disable the self heal while running this script. + +5 - All the bricks of the volume should be UP to identify good and bad fragments +and to decide if an entry is healble or not. + +6 - If correct_pending_heals.sh is stopped in the middle while it was processing +healble entries, it is suggested to re-run gfid_needing_heal_parallel.sh to create +latest list of healble and non healble entries and "potential_heal" "can_not_heal" files. + +7 - Based on the number of entries, these files might take time to get and set the +stats and xattrs of entries. + +8 - A backup of the fragments will be taken on <brick path>/.glusterfs/correct_pending_heals + directory with a file name same as gfid. + +9 - Once the correctness of the file gets verified by user, these backup should be removed. + +10 - Make sure we have enough space on bricks to take these backups. + +11 - At the end this will create two files - + 1 - modified_and_backedup_files - Contains list of files which have been modified and should be healed. + 2 - can_not_heal - Contains list of files which can not be healed. + +12 - It is suggested that the integrity of the data of files, which were modified and healed, + should be checked by the user. + + +Usage: + +Following are the sequence of steps to use these scripts - + +1 - ./gfid_needing_heal_parallel.sh <volume name> + + Execute gfid_needing_heal_parallel.sh with volume name to create list of files which could + be healed and can not be healed. It creates "potential_heal" and "can_not_heal" files. + During execution, it also displays the list of files on consol with the verdict. + +2 - ./correct_pending_heals.sh + + Execute correct_pending_heals.sh without any argument. This script processes entries present + in "heal" file. It asks user to enter how many files we want to process in one attempt. + Once the count is provided, this script will fetch the entries one by one from "potential_heal" file and takes necessary action. + If at this point also a file can not be healed, it will be pushed to "can_not_heal" file. + If a file can be healed, this script will modify the xattrs of that file fragments and create an entry in "modified_and_backedup_files" file + +3 - At the end, all the entries of "potential_heal" will be processed and based on the processing only two files will be left. + + 1 - modified_and_backedup_files - Contains list of files which have been modified and should be healed. + 2 - can_not_heal - Contains list of files which can not be healed. + +Logs and other files - + +1 - modified_and_backedup_files - It contains all the files which could be healed and the location of backup of each fragments. +2 - can_not_heal - It contains all the files which can not be healed. +3 - potential_heal - List of files which could be healed and should be processed by "correct_pending_heals.sh" +4 - /var/log/glusterfs/ec-heal-script.log - It contains logs of both the files. diff --git a/extras/ec-heal-script/correct_pending_heals.sh b/extras/ec-heal-script/correct_pending_heals.sh new file mode 100755 index 00000000000..c9f19dd7c89 --- /dev/null +++ b/extras/ec-heal-script/correct_pending_heals.sh @@ -0,0 +1,415 @@ +#!/bin/bash +# Copyright (c) 2019-2020 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +# This script finally resets the xattrs of all the fragments of a file +# which can be healed as per gfid_needing_heal_parallel.sh. +# gfid_needing_heal_parallel.sh will produce two files, potential_heal and can_not_heal. +# This script takes potential_heal as input and resets xattrs of all the fragments +# of those files present in this file and which could be healed as per +# trusted.ec.size xattar of the file else it will place the entry in can_not_heal +# file. Those entries which must be healed will be place in must_heal file +# after setting xattrs so that user can track those files. + + +MOD_BACKUP_FILES="modified_and_backedup_files" +CAN_NOT_HEAL="can_not_heal" +LOG_DIR="/var/log/glusterfs" +LOG_FILE="$LOG_DIR/ec-heal-script.log" +LINE_SEP="===================================================" + +function heal_log() +{ + echo "$1" >> "$LOG_FILE" +} + +function desc () +{ + echo "" + echo "This script finally resets the xattrs of all the fragments of a file +which can be healed as per gfid_needing_heal_parallel.sh. +gfid_needing_heal_parallel.sh will produce two files, potential_heal and can_not_heal. +This script takes potential_heal as input and resets xattrs of all the fragments +of those files present in this file and which could be healed as per +trusted.ec.size xattar of the file else it will place the entry in can_not_heal +file. Those entries which must be healed will be place in must_heal file +after setting xattrs so that user can track those files." +} + +function _init () +{ + if [ $# -ne 0 ] + then + echo "usage: $0" + desc + exit 2 + fi + + if [ ! -f "potential_heal" ] + then + echo "Nothing to correct. File "potential_heal" does not exist" + echo "" + desc + exit 2 + fi +} + +function total_file_size_in_hex() +{ + local frag_size=$1 + local size=0 + local hex_size="" + + size=$((frag_size * 4)) + hex_size=$(printf '0x%016x' $size) + echo "$hex_size" +} + +function backup_file_fragment() +{ + local file_host=$1 + local file_entry=$2 + local gfid_actual_paths=$3 + local brick_root="" + local temp="" + local backup_dir="" + local cmd="" + local gfid="" + + brick_root=$(echo "$file_entry" | cut -d "#" -f 1) + temp=$(echo "$(basename "$BASH_SOURCE")" | cut -d '.' -f 1) + backup_dir=$(echo "${brick_root}/.glusterfs/${temp}") + file_entry=${file_entry//#} + + gfid=$(echo "${gfid_actual_paths}" | cut -d '|' -f 1 | cut -d '/' -f 5) + echo "${file_host}:${backup_dir}/${gfid}" >> "$MOD_BACKUP_FILES" + + cmd="mkdir -p ${backup_dir} && yes | cp -af ${file_entry} ${backup_dir}/${gfid} 2>/dev/null" + ssh -n "${file_host}" "${cmd}" +} + +function set_frag_xattr () +{ + local file_host=$1 + local file_entry=$2 + local good=$3 + local cmd1="" + local cmd2="" + local cmd="" + local version="0x00000000000000010000000000000001" + local dirty="0x00000000000000010000000000000001" + + if [[ $good -eq 0 ]] + then + version="0x00000000000000000000000000000000" + fi + + cmd1=" setfattr -n trusted.ec.version -v ${version} ${file_entry} &&" + cmd2=" setfattr -n trusted.ec.dirty -v ${dirty} ${file_entry}" + cmd=${cmd1}${cmd2} + ssh -n "${file_host}" "${cmd}" +} + +function set_version_dirty_xattr () +{ + local file_paths=$1 + local good=$2 + local gfid_actual_paths=$3 + local file_entry="" + local file_host="" + local bpath="" + + for bpath in ${file_paths//,/ } + do + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + backup_file_fragment "$file_host" "$file_entry" "$gfid_actual_paths" + file_entry=${file_entry//#} + set_frag_xattr "$file_host" "$file_entry" "$good" + done +} + +function match_size_xattr_quorum () +{ + local file_paths=$1 + local file_entry="" + local file_host="" + local cmd="" + local size_xattr="" + local bpath="" + declare -A xattr_count + + for bpath in ${file_paths//,/ } + do + size_xattr="" + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + file_entry=${file_entry//#} + + cmd="getfattr -n trusted.ec.size -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.size" | cut -d '=' -f 2" + size_xattr=$(ssh -n "${file_host}" "${cmd}") + if [[ -n $size_xattr ]] + then + count=$((xattr_count["$size_xattr"] + 1)) + xattr_count["$size_xattr"]=${count} + if [[ $count -ge 4 ]] + then + echo "${size_xattr}" + return + fi + fi + done + echo "False" +} + +function match_version_xattr () +{ + local file_paths=$1 + local file_entry="" + local file_host="" + local cmd="" + local version="" + local bpath="" + declare -A ver_count + + for bpath in ${file_paths//,/ } + do + version="" + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + file_entry=${file_entry//#} + + cmd="getfattr -n trusted.ec.version -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.version" | cut -d '=' -f 2" + version=$(ssh -n "${file_host}" "${cmd}") + ver_count["$version"]=$((ver_count["$version"] + 1)) + done + for key in "${ver_count[@]}" + do + if [[ $key -ge 4 ]] + then + echo "True" + return + else + echo "False" + return + fi + done +} + +function match_stat_size_with_xattr () +{ + local bpath=$1 + local size=$2 + local file_stat=$3 + local xattr=$4 + local file_entry="" + local file_host="" + local cmd="" + local stat_output="" + local hex_size="" + + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + + file_entry=${file_entry//#} + cmd="stat --format=%F:%B:%s $file_entry 2>/dev/null" + stat_output=$(ssh -n "${file_host}" "${cmd}") + echo "$stat_output" | grep -w "${file_stat}" > /dev/null + + if [[ $? -eq 0 ]] + then + cmd="getfattr -n trusted.ec.size -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.size" | cut -d '=' -f 2" + hex_size=$(ssh -n "${file_host}" "${cmd}") + + if [[ -z $hex_size || "$hex_size" != "$xattr" ]] + then + echo "False" + return + fi + size_diff=$(printf '%d' $(( size - hex_size ))) + if [[ $size_diff -gt 2047 ]] + then + echo "False" + return + else + echo "True" + return + fi + else + echo "False" + return + fi +} + +function find_file_paths () +{ + local bpath=$1 + local file_entry="" + local file_host="" + local cmd="" + local brick_root="" + local gfid="" + local actual_path="" + local gfid_path="" + + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + brick_root=$(echo "$file_entry" | cut -d "#" -f 1) + + gfid=$(echo "${file_entry}" | grep ".glusterfs") + if [[ -n "$gfid" ]] + then + gfid_path=$(echo "$file_entry" | cut -d "#" -f 2) + file_entry=${file_entry//#} + cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep -v '.glusterfs' " + actual_path=$(ssh -n "${file_host}" "${cmd}") + #removing absolute path so that user can refer this from mount point + actual_path=${actual_path#"$brick_root"} + else + actual_path=$(echo "$file_entry" | cut -d "#" -f 2) + file_entry=${file_entry//#} + cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep '.glusterfs' " + gfid_path=$(ssh -n "${file_host}" "${cmd}") + gfid_path=${gfid_path#"$brick_root"} + fi + + echo "${gfid_path}|${actual_path}" +} + +function log_can_not_heal () +{ + local gfid_actual_paths=$1 + local file_paths=$2 + file_paths=${file_paths//#} + + echo "${LINE_SEP}" >> "$CAN_NOT_HEAL" + echo "Can Not Heal : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$CAN_NOT_HEAL" + for bpath in ${file_paths//,/ } + do + echo "${bpath}" >> "$CAN_NOT_HEAL" + done +} + +function check_all_frag_and_set_xattr () +{ + local file_paths=$1 + local total_size=$2 + local file_stat=$3 + local bpath="" + local healthy_count=0 + local match="False" + local matching_bricks="" + local bad_bricks="" + local gfid_actual_paths="" + + for bpath in ${file_paths//,/ } + do + if [[ -n "$gfid_actual_paths" ]] + then + break + fi + gfid_actual_paths=$(find_file_paths "$bpath") + done + + match=$(match_size_xattr_quorum "$file_paths") + +# echo "${match} : $bpath" >> "$MOD_BACKUP_FILES" + + if [[ "$match" != "False" ]] + then + xattr="$match" + for bpath in ${file_paths//,/ } + do + match="False" + match=$(match_stat_size_with_xattr "$bpath" "$total_size" "$file_stat" "$xattr") + if [[ "$match" == "True" ]] + then + matching_bricks="${bpath},${matching_bricks}" + healthy_count=$((healthy_count + 1)) + else + bad_bricks="${bpath},${bad_bricks}" + fi + done + fi + + if [[ $healthy_count -ge 4 ]] + then + match="True" + echo "${LINE_SEP}" >> "$MOD_BACKUP_FILES" + echo "Modified : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$MOD_BACKUP_FILES" + set_version_dirty_xattr "$matching_bricks" 1 "$gfid_actual_paths" + set_version_dirty_xattr "$bad_bricks" 0 "$gfid_actual_paths" + else + log_can_not_heal "$gfid_actual_paths" "${file_paths}" + fi + + echo "$match" +} +function set_xattr() +{ + local count=$1 + local heal_entry="" + local file_stat="" + local frag_size="" + local total_size="" + local file_paths="" + local num="" + local can_heal_count=0 + + heal_log "Started $(basename $BASH_SOURCE) on $(date) " + + while read -r heal_entry + do + heal_log "$LINE_SEP" + heal_log "${heal_entry}" + + file_stat=$(echo "$heal_entry" | cut -d "|" -f 1) + frag_size=$(echo "$file_stat" | rev | cut -d ":" -f 1 | rev) + total_size="$(total_file_size_in_hex "$frag_size")" + file_paths=$(echo "$heal_entry" | cut -d "|" -f 2) + match=$(check_all_frag_and_set_xattr "$file_paths" "$total_size" "$file_stat") + if [[ "$match" == "True" ]] + then + can_heal_count=$((can_heal_count + 1)) + fi + + sed -i '1d' potential_heal + count=$((count - 1)) + if [ $count == 0 ] + then + num=$(cat potential_heal | wc -l) + heal_log "$LINE_SEP" + heal_log "${1} : Processed" + heal_log "${can_heal_count} : Modified to Heal" + heal_log "$((${1} - can_heal_count)) : Moved to can_not_heal." + heal_log "${num} : Pending as Potential Heal" + exit 0 + fi + + done < potential_heal +} + +function main () +{ + local count=0 + + read -p "Number of files to correct: [choose between 1-1000] (0 for All):" count + if [[ $count -lt 0 || $count -gt 1000 ]] + then + echo "Provide correct value:" + exit 2 + fi + + if [[ $count -eq 0 ]] + then + count=$(cat potential_heal | wc -l) + fi + set_xattr "$count" +} + +_init "$@" && main "$@" diff --git a/extras/ec-heal-script/gfid_needing_heal_parallel.sh b/extras/ec-heal-script/gfid_needing_heal_parallel.sh new file mode 100755 index 00000000000..d7f53c97c33 --- /dev/null +++ b/extras/ec-heal-script/gfid_needing_heal_parallel.sh @@ -0,0 +1,278 @@ +#!/bin/bash +# Copyright (c) 2019-2020 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +# This script provides a list of all the files which can be healed or not healed. +# It also generates two files, potential_heal and can_not_heal, which contains the information +# of all theose files. These files could be used by correct_pending_heals.sh to correct +# the fragmnets so that files could be healed by shd. + +CAN_NOT_HEAL="can_not_heal" +CAN_HEAL="potential_heal" +LINE_SEP="===================================================" +LOG_DIR="/var/log/glusterfs" +LOG_FILE="$LOG_DIR/ec-heal-script.log" + +function heal_log() +{ + echo "$1" >> "$LOG_FILE" +} + +function _init () +{ + if [ $# -ne 1 ]; then + echo "usage: $0 <gluster volume name>"; + echo "This script provides a list of all the files which can be healed or not healed. +It also generates two files, potential_heal and can_not_heal, which contains the information +of all theose files. These files could be used by correct_pending_heals.sh to correct +the fragmnets so that files could be healed by shd." + exit 2; + fi + + volume=$1; +} + +function get_pending_entries () +{ + local volume_name=$1 + + gluster volume heal "$volume_name" info | grep -v ":/" | grep -v "Number of entries" | grep -v "Status:" | sort -u | sed '/^$/d' +} + +function get_entry_path_on_brick() +{ + local path="$1" + local gfid_string="" + if [[ "${path:0:1}" == "/" ]]; + then + echo "$path" + else + gfid_string="$(echo "$path" | cut -f2 -d':' | cut -f1 -d '>')" + echo "/.glusterfs/${gfid_string:0:2}/${gfid_string:2:2}/$gfid_string" + fi +} + +function run_command_on_server() +{ + local subvolume="$1" + local host="$2" + local cmd="$3" + local output + output=$(ssh -n "${host}" "${cmd}") + if [ -n "$output" ] + then + echo "$subvolume:$output" + fi +} + +function get_entry_path_all_bricks () +{ + local entry="$1" + local bricks="$2" + local cmd="" + for brick in $bricks + do + echo "${brick}#$(get_entry_path_on_brick "$entry")" + done | tr '\n' ',' +} + +function get_stat_for_entry_from_all_bricks () +{ + local entry="$1" + local bricks="$2" + local subvolume=0 + local host="" + local bpath="" + local cmd="" + + for brick in $bricks + do + if [[ "$((subvolume % 6))" == "0" ]] + then + subvolume=$((subvolume+1)) + fi + host=$(echo "$brick" | cut -f1 -d':') + bpath=$(echo "$brick" | cut -f2 -d':') + + cmd="stat --format=%F:%B:%s $bpath$(get_entry_path_on_brick "$entry") 2>/dev/null" + run_command_on_server "$subvolume" "${host}" "${cmd}" & + done | sort | uniq -c | sort -rnk1 +} + +function get_bricks_from_volume() +{ + local v=$1 + gluster volume info "$v" | grep -E "^Brick[0-9][0-9]*:" | cut -f2- -d':' +} + +function print_entry_gfid() +{ + local host="$1" + local dirpath="$2" + local entry="$3" + local gfid + gfid="$(ssh -n "${host}" "getfattr -d -m. -e hex $dirpath/$entry 2>/dev/null | grep trusted.gfid=|cut -f2 -d'='")" + echo "$entry" - "$gfid" +} + +function print_brick_directory_info() +{ + local h="$1" + local dirpath="$2" + while read -r e + do + print_entry_gfid "${h}" "${dirpath}" "${e}" + done < <(ssh -n "${h}" "ls $dirpath 2>/dev/null") +} + +function print_directory_info() +{ + local entry="$1" + local bricks="$2" + local h + local b + local gfid + for brick in $bricks; + do + h="$(echo "$brick" | cut -f1 -d':')" + b="$(echo "$brick" | cut -f2 -d':')" + dirpath="$b$(get_entry_path_on_brick "$entry")" + print_brick_directory_info "${h}" "${dirpath}" & + done | sort | uniq -c +} + +function print_entries_needing_heal() +{ + local quorum=0 + local entry="$1" + local bricks="$2" + while read -r line + do + quorum=$(echo "$line" | awk '{print $1}') + if [[ "$quorum" -lt 4 ]] + then + echo "$line - Not in Quorum" + else + echo "$line - In Quorum" + fi + done < <(print_directory_info "$entry" "$bricks") +} + +function find_file_paths () +{ + local bpath=$1 + local file_entry="" + local file_host="" + local cmd="" + local brick_root="" + local gfid="" + local actual_path="" + local gfid_path="" + + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + brick_root=$(echo "$file_entry" | cut -d "#" -f 1) + + gfid=$(echo "${file_entry}" | grep ".glusterfs") + + if [[ -n "$gfid" ]] + then + gfid_path=$(echo "$file_entry" | cut -d "#" -f 2) + file_entry=${file_entry//#} + cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep -v '.glusterfs' " + actual_path=$(ssh -n "${file_host}" "${cmd}") + #removing absolute path so that user can refer this from mount point + actual_path=${actual_path#"$brick_root"} + else + actual_path=$(echo "$file_entry" | cut -d "#" -f 2) + file_entry=${file_entry//#} + cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep '.glusterfs' " + gfid_path=$(ssh -n "${file_host}" "${cmd}") + gfid_path=${gfid_path#"$brick_root"} + fi + + echo "${gfid_path}|${actual_path}" +} + +function log_can_not_heal () +{ + local gfid_actual_paths=$1 + local file_paths=$2 + file_paths=${file_paths//#} + + echo "${LINE_SEP}" >> "$CAN_NOT_HEAL" + echo "Can Not Heal : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$CAN_NOT_HEAL" + for bpath in ${file_paths//,/ } + do + echo "${bpath}" >> "$CAN_NOT_HEAL" + done +} + +function main () +{ + local bricks="" + local quorum=0 + local stat_info="" + local file_type="" + local gfid_actual_paths="" + local bpath="" + local file_paths="" + local good=0 + local bad=0 + bricks=$(get_bricks_from_volume "$volume") + rm -f "$CAN_HEAL" + rm -f "$CAN_NOT_HEAL" + mkdir "$LOG_DIR" -p + + heal_log "Started $(basename "$BASH_SOURCE") on $(date) " + while read -r heal_entry + do + heal_log "------------------------------------------------------------------" + heal_log "$heal_entry" + + gfid_actual_paths="" + file_paths="$(get_entry_path_all_bricks "$heal_entry" "$bricks")" + stat_info="$(get_stat_for_entry_from_all_bricks "$heal_entry" "$bricks")" + heal_log "$stat_info" + + quorum=$(echo "$stat_info" | head -1 | awk '{print $1}') + good_stat=$(echo "$stat_info" | head -1 | awk '{print $3}') + file_type="$(echo "$stat_info" | head -1 | cut -f2 -d':')" + if [[ "$file_type" == "directory" ]] + then + print_entries_needing_heal "$heal_entry" "$bricks" + else + if [[ "$quorum" -ge 4 ]] + then + good=$((good + 1)) + heal_log "Verdict: Healable" + + echo "${good_stat}|$file_paths" >> "$CAN_HEAL" + else + bad=$((bad + 1)) + heal_log "Verdict: Not Healable" + for bpath in ${file_paths//,/ } + do + if [[ -z "$gfid_actual_paths" ]] + then + gfid_actual_paths=$(find_file_paths "$bpath") + else + break + fi + done + log_can_not_heal "$gfid_actual_paths" "${file_paths}" + fi + fi + done < <(get_pending_entries "$volume") + heal_log "=========================================" + heal_log "Total number of potential heal : ${good}" + heal_log "Total number of can not heal : ${bad}" + heal_log "=========================================" +} + +_init "$@" && main "$@" diff --git a/extras/failed-tests.py b/extras/failed-tests.py new file mode 100755 index 00000000000..f7f110246b5 --- /dev/null +++ b/extras/failed-tests.py @@ -0,0 +1,180 @@ +#!/usr/bin/python3 + +from __future__ import print_function +import blessings +import requests +from requests.packages.urllib3.exceptions import InsecureRequestWarning +import re +import argparse +from collections import defaultdict +from datetime import timedelta, datetime +from pystache import render + +# This tool goes though the Gluster regression links and checks for failures + +BASE = 'https://build.gluster.org' +TERM = blessings.Terminal() +MAX_BUILDS = 1000 +summary = defaultdict(list) +VERBOSE = None + + +def process_failure(url, node): + text = requests.get(url, verify=False).text + accum = [] + for t in text.split('\n'): + if t.find("Result: FAIL") != -1: + for t2 in accum: + if VERBOSE: + print(t2.encode('utf-8')) + if t2.find("Wstat") != -1: + test_case = re.search('\./tests/.*\.t', t2) + if test_case: + summary[test_case.group()].append((url, node)) + accum = [] + elif t.find("cur_cores=/") != -1: + summary["core"].append([t.split("/")[1]]) + summary["core"].append(url) + else: + accum.append(t) + + +def print_summary(failed_builds, total_builds, html=False): + # All the templates + count = [ + '{{failed}} of {{total}} regressions failed', + '<p><b>{{failed}}</b> of <b>{{total}}</b> regressions failed</p>' + ] + regression_link = [ + '\tRegression Link: {{link}}\n' + '\tNode: {{node}}', + '<p> Regression Link: {{link}}</p>' + '<p> Node: {{node}}</p>' + ] + component = [ + '\tComponent: {{comp}}', + '<p> Component: {{comp}}</p>' + ] + failure_count = [ + ''.join([ + TERM.red, + '{{test}} ; Failed {{count}} times', + TERM.normal + ]), + ( + '<p><font color="red"><b>{{test}};</b> Failed <b>{{count}}' + '</b> times</font></p>' + ) + ] + + template = 0 + if html: + template = 1 + print(render( + count[template], + {'failed': failed_builds, 'total': total_builds} + )) + for k, v in summary.items(): + if k == 'core': + print(''.join([TERM.red, "Found cores:", TERM.normal])) + for comp, link in zip(v[::2], v[1::2]): + print(render(component[template], {'comp': comp})) + print(render( + regression_link[template], + {'link': link[0], 'node': link[1]} + )) + else: + print(render(failure_count[template], {'test': k, 'count': len(v)})) + for link in v: + print(render( + regression_link[template], + {'link': link[0], 'node': link[1]} + )) + + +def get_summary(cut_off_date, reg_link): + ''' + Get links to the failed jobs + ''' + success_count = 0 + failure_count = 0 + for page in range(0, MAX_BUILDS, 100): + build_info = requests.get(''.join([ + BASE, + reg_link, + 'api/json?depth=1&tree=allBuilds' + '[url,result,timestamp,builtOn]', + '{{{0},{1}}}'.format(page, page+100) + ]), verify=False).json() + for build in build_info.get('allBuilds'): + if datetime.fromtimestamp(build['timestamp']/1000) < cut_off_date: + # stop when timestamp older than cut off date + return failure_count, failure_count + success_count + if build['result'] in [None, 'SUCCESS']: + # pass when build is a success or ongoing + success_count += 1 + continue + if VERBOSE: + print(''.join([ + TERM.red, + 'FAILURE on {0}'.format(build['url']), + TERM.normal + ])) + url = ''.join([build['url'], 'consoleText']) + failure_count += 1 + process_failure(url, build['builtOn']) + return failure_count, failure_count + success_count + + +def main(num_days, regression_link, html_report): + cut_off_date = datetime.today() - timedelta(days=num_days) + failure = 0 + total = 0 + for reg in regression_link: + if reg == 'centos': + reg_link = '/job/centos6-regression/' + elif reg == 'netbsd': + reg_link = '/job/netbsd7-regression/' + else: + reg_link = reg + counts = get_summary(cut_off_date, reg_link) + failure += counts[0] + total += counts[1] + print_summary(failure, total, html_report) + + +if __name__ == '__main__': + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + parser = argparse.ArgumentParser() + parser.add_argument("get-summary") + parser.add_argument( + "last_no_of_days", + default=1, + type=int, + help="Regression summary of last number of days" + ) + parser.add_argument( + "regression_link", + default="centos", + nargs='+', + help="\"centos\" | \"netbsd\" | any other regression link" + ) + parser.add_argument( + "--verbose", + default=False, + action="store_true", + help="Print a detailed report of each test case that is failed" + ) + parser.add_argument( + "--html-report", + default=False, + action="store_true", + help="Print a brief report of failed regressions in html format" + ) + args = parser.parse_args() + VERBOSE = args.verbose + main( + num_days=args.last_no_of_days, + regression_link=args.regression_link, + html_report=args.html_report + ) diff --git a/extras/firewalld/Makefile.am b/extras/firewalld/Makefile.am new file mode 100644 index 00000000000..530881fb8eb --- /dev/null +++ b/extras/firewalld/Makefile.am @@ -0,0 +1,8 @@ +EXTRA_DIST = glusterfs.xml + +if USE_FIREWALLD +if WITH_SERVER +staticdir = /usr/lib/firewalld/services/ +static_DATA = glusterfs.xml +endif +endif diff --git a/extras/firewalld/glusterfs.xml b/extras/firewalld/glusterfs.xml new file mode 100644 index 00000000000..7e176442f5b --- /dev/null +++ b/extras/firewalld/glusterfs.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="utf-8"?> +<service> +<short>glusterfs-static</short> +<description>Default ports for gluster-distributed storage</description> +<port protocol="tcp" port="24007"/> <!--For glusterd --> +<port protocol="tcp" port="24008"/> <!--For glusterd RDMA port management --> +<port protocol="tcp" port="24009"/> <!--For glustereventsd --> +<port protocol="tcp" port="38465"/> <!--Gluster NFS service --> +<port protocol="tcp" port="38466"/> <!--Gluster NFS service --> +<port protocol="tcp" port="38467"/> <!--Gluster NFS service --> +<port protocol="tcp" port="38468"/> <!--Gluster NFS service --> +<port protocol="tcp" port="38469"/> <!--Gluster NFS service --> +<port protocol="tcp" port="49152-49664"/> <!--512 ports for bricks --> +</service> diff --git a/extras/ganesha/Makefile.am b/extras/ganesha/Makefile.am new file mode 100644 index 00000000000..9eaa401b6c8 --- /dev/null +++ b/extras/ganesha/Makefile.am @@ -0,0 +1,2 @@ +SUBDIRS = scripts config ocf +CLEANFILES = diff --git a/extras/ganesha/config/Makefile.am b/extras/ganesha/config/Makefile.am new file mode 100644 index 00000000000..c729273096e --- /dev/null +++ b/extras/ganesha/config/Makefile.am @@ -0,0 +1,4 @@ +EXTRA_DIST= ganesha-ha.conf.sample + +confdir = $(sysconfdir)/ganesha +conf_DATA = ganesha-ha.conf.sample diff --git a/extras/ganesha/config/ganesha-ha.conf.sample b/extras/ganesha/config/ganesha-ha.conf.sample new file mode 100644 index 00000000000..c22892bde56 --- /dev/null +++ b/extras/ganesha/config/ganesha-ha.conf.sample @@ -0,0 +1,19 @@ +# Name of the HA cluster created. +# must be unique within the subnet +HA_NAME="ganesha-ha-360" +# +# N.B. you may use short names or long names; you may not use IP addrs. +# Once you select one, stay with it as it will be mildly unpleasant to +# clean up if you switch later on. Ensure that all names - short and/or +# long - are in DNS or /etc/hosts on all machines in the cluster. +# +# The subset of nodes of the Gluster Trusted Pool that form the ganesha +# HA cluster. Hostname is specified. +HA_CLUSTER_NODES="server1,server2,..." +#HA_CLUSTER_NODES="server1.lab.redhat.com,server2.lab.redhat.com,..." +# +# Virtual IPs for each of the nodes specified above. +VIP_server1="10.0.2.1" +VIP_server2="10.0.2.2" +#VIP_server1_lab_redhat_com="10.0.2.1" +#VIP_server2_lab_redhat_com="10.0.2.2" diff --git a/extras/ganesha/ocf/Makefile.am b/extras/ganesha/ocf/Makefile.am new file mode 100644 index 00000000000..990a609f254 --- /dev/null +++ b/extras/ganesha/ocf/Makefile.am @@ -0,0 +1,11 @@ +EXTRA_DIST= ganesha_grace ganesha_mon ganesha_nfsd + +# The root of the OCF resource agent hierarchy +# Per the OCF standard, it's always "lib", +# not "lib64" (even on 64-bit platforms). +ocfdir = $(prefix)/lib/ocf + +# The provider directory +radir = $(ocfdir)/resource.d/heartbeat + +ra_SCRIPTS = ganesha_grace ganesha_mon ganesha_nfsd diff --git a/extras/ganesha/ocf/ganesha_grace b/extras/ganesha/ocf/ganesha_grace new file mode 100644 index 00000000000..825f7164597 --- /dev/null +++ b/extras/ganesha/ocf/ganesha_grace @@ -0,0 +1,221 @@ +#!/bin/bash +# +# Copyright (c) 2014 Anand Subramanian anands@redhat.com +# Copyright (c) 2015 Red Hat Inc. +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# +# + +# Initialization: +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +if [ -n "$OCF_DEBUG_LIBRARY" ]; then + . $OCF_DEBUG_LIBRARY +else + : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} + . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs +fi + +OCF_RESKEY_grace_active_default="grace-active" +: ${OCF_RESKEY_grace_active=${OCF_RESKEY_grace_active_default}} + +ganesha_meta_data() { + cat <<END +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="ganesha_grace"> +<version>1.0</version> + +<longdesc lang="en"> +This Linux-specific resource agent acts as a dummy +resource agent for nfs-ganesha. +</longdesc> + +<shortdesc lang="en">Manages the user-space nfs-ganesha NFS server</shortdesc> + +<parameters> +<parameter name="grace_active"> +<longdesc lang="en">NFS-Ganesha grace active attribute</longdesc> +<shortdesc lang="en">NFS-Ganesha grace active attribute</shortdesc> +<content type="string" default="grace-active" /> +</parameter> +</parameters> + +<actions> +<action name="start" timeout="40s" /> +<action name="stop" timeout="40s" /> +<action name="status" timeout="20s" interval="60s" /> +<action name="monitor" depth="0" timeout="10s" interval="5s" /> +<action name="notify" timeout="10s" /> +<action name="meta-data" timeout="20s" /> +</actions> +</resource-agent> +END + +return ${OCF_SUCCESS} +} + +ganesha_grace_usage() { + echo "ganesha.nfsd USAGE" +} + +# Make sure meta-data and usage always succeed +case $__OCF_ACTION in + meta-data) ganesha_meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) ganesha_usage + exit ${OCF_SUCCESS} + ;; + *) + ;; +esac + +ganesha_grace_start() +{ + local rc=${OCF_ERR_GENERIC} + local host=$(hostname -s) + + ocf_log debug "ganesha_grace_start()" + # give ganesha_mon RA a chance to set the crm_attr first + # I mislike the sleep, but it's not clear that looping + # with a small sleep is necessarily better + # start has a 40sec timeout, so a 5sec sleep here is okay + sleep 5 + attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null) + if [ $? -ne 0 ]; then + host=$(hostname) + attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null ) + if [ $? -ne 0 ]; then + ocf_log info "grace start: crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} failed" + fi + fi + + # Three possibilities: + # 1. There is no attribute at all and attr_updater returns + # a zero length string. This happens when + # ganesha_mon::monitor hasn't run at least once to set + # the attribute. The assumption here is that the system + # is coming up. We pretend, for now, that the node is + # healthy, to allow the system to continue coming up. + # It will cure itself in a few seconds + # 2. There is an attribute, and it has the value "1"; this + # node is healthy. + # 3. There is an attribute, but it has no value or the value + # "0"; this node is not healthy. + + # case 1 + if [[ -z "${attr}" ]]; then + return ${OCF_SUCCESS} + fi + + # case 2 + if [[ "${attr}" = *"value=1" ]]; then + return ${OCF_SUCCESS} + fi + + # case 3 + return ${OCF_NOT_RUNNING} +} + +ganesha_grace_stop() +{ + + ocf_log debug "ganesha_grace_stop()" + return ${OCF_SUCCESS} +} + +ganesha_grace_notify() +{ + # since this is a clone RA we should only ever see pre-start + # or post-stop + mode="${OCF_RESKEY_CRM_meta_notify_type}-${OCF_RESKEY_CRM_meta_notify_operation}" + case "${mode}" in + pre-start | post-stop) + dbus-send --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.grace string:${OCF_RESKEY_CRM_meta_notify_stop_uname} + if [ $? -ne 0 ]; then + ocf_log info "dbus-send --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.grace string:${OCF_RESKEY_CRM_meta_notify_stop_uname} failed" + fi + ;; + esac + + return ${OCF_SUCCESS} +} + +ganesha_grace_monitor() +{ + local host=$(hostname -s) + + ocf_log debug "monitor" + + attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null) + if [ $? -ne 0 ]; then + host=$(hostname) + attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null) + if [ $? -ne 0 ]; then + ocf_log info "crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} failed" + fi + fi + + # if there is no attribute (yet), maybe it's because + # this RA started before ganesha_mon (nfs-mon) has had + # chance to create it. In which case we'll pretend + # everything is okay this time around + if [[ -z "${attr}" ]]; then + return ${OCF_SUCCESS} + fi + + if [[ "${attr}" = *"value=1" ]]; then + return ${OCF_SUCCESS} + fi + + return ${OCF_NOT_RUNNING} +} + +ganesha_grace_validate() +{ + return ${OCF_SUCCESS} +} + +ganesha_grace_validate + +# Translate each action into the appropriate function call +case $__OCF_ACTION in +start) ganesha_grace_start + ;; +stop) ganesha_grace_stop + ;; +status|monitor) ganesha_grace_monitor + ;; +notify) ganesha_grace_notify + ;; +*) ganesha_grace_usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac + +rc=$? + +# The resource agent may optionally log a debug message +ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc" +exit $rc diff --git a/extras/ganesha/ocf/ganesha_mon b/extras/ganesha/ocf/ganesha_mon new file mode 100644 index 00000000000..2b4a9d6da84 --- /dev/null +++ b/extras/ganesha/ocf/ganesha_mon @@ -0,0 +1,234 @@ +#!/bin/bash +# +# Copyright (c) 2014 Anand Subramanian anands@redhat.com +# Copyright (c) 2015 Red Hat Inc. +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# +# + +# Initialization: +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +if [ -n "${OCF_DEBUG_LIBRARY}" ]; then + . ${OCF_DEBUG_LIBRARY} +else + : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} + . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs +fi + +# Defaults +OCF_RESKEY_ganesha_active_default="ganesha-active" +OCF_RESKEY_grace_active_default="grace-active" +OCF_RESKEY_grace_delay_default="5" + +: ${OCF_RESKEY_ganesha_active=${OCF_RESKEY_ganesha_active_default}} +: ${OCF_RESKEY_grace_active=${OCF_RESKEY_grace_active_default}} +: ${OCF_RESKEY_grace_delay=${OCF_RESKEY_grace_delay_default}} + +ganesha_meta_data() { + cat <<END +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="ganesha_mon"> +<version>1.0</version> + +<longdesc lang="en"> +This Linux-specific resource agent acts as a dummy +resource agent for nfs-ganesha. +</longdesc> + +<shortdesc lang="en">Manages the user-space nfs-ganesha NFS server</shortdesc> + +<parameters> +<parameter name="ganesha_active"> +<longdesc lang="en">NFS-Ganesha daemon active attribute</longdesc> +<shortdesc lang="en">NFS-Ganesha daemon active attribute</shortdesc> +<content type="string" default="ganesha-active" /> +</parameter> +<parameter name="grace_active"> +<longdesc lang="en">NFS-Ganesha grace active attribute</longdesc> +<shortdesc lang="en">NFS-Ganesha grace active attribute</shortdesc> +<content type="string" default="grace-active" /> +</parameter> +<parameter name="grace_delay"> +<longdesc lang="en"> +NFS-Ganesha grace delay. +When changing this, adjust the ganesha_grace RA's monitor interval to match. +</longdesc> +<shortdesc lang="en">NFS-Ganesha grace delay</shortdesc> +<content type="string" default="5" /> +</parameter> +</parameters> + +<actions> +<action name="start" timeout="40s" /> +<action name="stop" timeout="40s" /> +<action name="status" timeout="20s" interval="60s" /> +<action name="monitor" depth="0" timeout="10s" interval="10s" /> +<action name="meta-data" timeout="20s" /> +</actions> +</resource-agent> +END + +return ${OCF_SUCCESS} +} + +ganesha_mon_usage() { + echo "ganesha.nfsd USAGE" +} + +# Make sure meta-data and usage always succeed +case ${__OCF_ACTION} in + meta-data) ganesha_meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) ganesha_usage + exit ${OCF_SUCCESS} + ;; + *) + ;; +esac + +ganesha_mon_start() +{ + ocf_log debug "ganesha_mon_start" + ganesha_mon_monitor + return $OCF_SUCCESS +} + +ganesha_mon_stop() +{ + ocf_log debug "ganesha_mon_stop" + return $OCF_SUCCESS +} + +ganesha_mon_monitor() +{ + local host=$(hostname -s) + local pid_file="/var/run/ganesha.pid" + local rhel6_pid_file="/var/run/ganesha.nfsd.pid" + local proc_pid="/proc/" + + # RHEL6 /etc/init.d/nfs-ganesha adds -p /var/run/ganesha.nfsd.pid + # RHEL7 systemd does not. Would be nice if all distros used the + # same pid file. + if [ -e ${rhel6_pid_file} ]; then + pid_file=${rhel6_pid_file} + fi + if [ -e ${pid_file} ]; then + proc_pid="${proc_pid}$(cat ${pid_file})" + fi + + if [ "x${proc_pid}" != "x/proc/" -a -d ${proc_pid} ]; then + + attrd_updater -n ${OCF_RESKEY_ganesha_active} -v 1 + if [ $? -ne 0 ]; then + ocf_log info "warning: attrd_updater -n ${OCF_RESKEY_ganesha_active} -v 1 failed" + fi + + # ganesha_grace (nfs-grace) RA follows grace-active attr + # w/ constraint location + attrd_updater -n ${OCF_RESKEY_grace_active} -v 1 + if [ $? -ne 0 ]; then + ocf_log info "warning: attrd_updater -n ${OCF_RESKEY_grace_active} -v 1 failed" + fi + + # ganesha_mon (nfs-mon) and ganesha_grace (nfs-grace) + # track grace-active crm_attr (attr != crm_attr) + # we can't just use the attr as there's no way to query + # its value in RHEL6 pacemaker + + crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 2> /dev/null + if [ $? -ne 0 ]; then + host=$(hostname) + crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 2> /dev/null + if [ $? -ne 0 ]; then + ocf_log info "mon monitor warning: crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 failed" + fi + fi + + return ${OCF_SUCCESS} + fi + + # VIP fail-over is triggered by clearing the + # ganesha-active node attribute on this node. + # + # Meanwhile the ganesha_grace notify() runs when its + # nfs-grace resource is disabled on a node; which + # is triggered by clearing the grace-active attribute + # on this node. + # + # We need to allow time for it to run and put + # the remaining ganesha.nfsds into grace before + # initiating the VIP fail-over. + + attrd_updater -D -n ${OCF_RESKEY_grace_active} + if [ $? -ne 0 ]; then + ocf_log info "warning: attrd_updater -D -n ${OCF_RESKEY_grace_active} failed" + fi + + host=$(hostname -s) + crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 2> /dev/null + if [ $? -ne 0 ]; then + host=$(hostname) + crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 2> /dev/null + if [ $? -ne 0 ]; then + ocf_log info "mon monitor warning: crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 failed" + fi + fi + + sleep ${OCF_RESKEY_grace_delay} + + attrd_updater -D -n ${OCF_RESKEY_ganesha_active} + if [ $? -ne 0 ]; then + ocf_log info "warning: attrd_updater -D -n ${OCF_RESKEY_ganesha_active} failed" + fi + + return ${OCF_SUCCESS} +} + +ganesha_mon_validate() +{ + return ${OCF_SUCCESS} +} + +ganesha_mon_validate + +# Translate each action into the appropriate function call +case ${__OCF_ACTION} in +start) ganesha_mon_start + ;; +stop) ganesha_mon_stop + ;; +status|monitor) ganesha_mon_monitor + ;; +*) ganesha_mon_usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac + +rc=$? + +# The resource agent may optionally log a debug message +ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc" +exit $rc diff --git a/extras/ganesha/ocf/ganesha_nfsd b/extras/ganesha/ocf/ganesha_nfsd new file mode 100644 index 00000000000..f91e8b6b8f7 --- /dev/null +++ b/extras/ganesha/ocf/ganesha_nfsd @@ -0,0 +1,167 @@ +#!/bin/bash +# +# Copyright (c) 2014 Anand Subramanian anands@redhat.com +# Copyright (c) 2015 Red Hat Inc. +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# +# + +# Initialization: +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +if [ -n "${OCF_DEBUG_LIBRARY}" ]; then + . ${OCF_DEBUG_LIBRARY} +else + : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} + . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs +fi + +OCF_RESKEY_ha_vol_mnt_default="/run/gluster/shared_storage" +: ${OCF_RESKEY_ha_vol_mnt=${OCF_RESKEY_ha_vol_mnt_default}} + +ganesha_meta_data() { + cat <<END +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="ganesha_nfsd"> +<version>1.0</version> + +<longdesc lang="en"> +This Linux-specific resource agent acts as a dummy +resource agent for nfs-ganesha. +</longdesc> + +<shortdesc lang="en">Manages the user-space nfs-ganesha NFS server</shortdesc> + +<parameters> +<parameter name="ha_vol_mnt"> +<longdesc lang="en">HA State Volume Mount Point</longdesc> +<shortdesc lang="en">HA_State Volume Mount Point</shortdesc> +<content type="string" default="" /> +</parameter> +</parameters> + +<actions> +<action name="start" timeout="5s" /> +<action name="stop" timeout="5s" /> +<action name="status" depth="0" timeout="5s" interval="0" /> +<action name="monitor" depth="0" timeout="5s" interval="0" /> +<action name="meta-data" timeout="20s" /> +</actions> +</resource-agent> +END + +return ${OCF_SUCCESS} +} + +ganesha_nfsd_usage() { + echo "ganesha.nfsd USAGE" +} + +# Make sure meta-data and usage always succeed +case $__OCF_ACTION in + meta-data) ganesha_meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) ganesha_usage + exit ${OCF_SUCCESS} + ;; + *) + ;; +esac + +ganesha_nfsd_start() +{ + local long_host=$(hostname) + + if [[ -d /var/lib/nfs ]]; then + mv /var/lib/nfs /var/lib/nfs.backup + if [ $? -ne 0 ]; then + ocf_log notice "mv /var/lib/nfs /var/lib/nfs.backup failed" + fi + ln -s ${OCF_RESKEY_ha_vol_mnt}/nfs-ganesha/${long_host}/nfs /var/lib/nfs + if [ $? -ne 0 ]; then + ocf_log notice "ln -s ${OCF_RESKEY_ha_vol_mnt}/nfs-ganesha/${long_host}/nfs /var/lib/nfs failed" + fi + fi + + return ${OCF_SUCCESS} +} + +ganesha_nfsd_stop() +{ + + if [ -L /var/lib/nfs -a -d /var/lib/nfs.backup ]; then + rm -f /var/lib/nfs + if [ $? -ne 0 ]; then + ocf_log notice "rm -f /var/lib/nfs failed" + fi + mv /var/lib/nfs.backup /var/lib/nfs + if [ $? -ne 0 ]; then + ocf_log notice "mv /var/lib/nfs.backup /var/lib/nfs failed" + fi + fi + + return ${OCF_SUCCESS} +} + +ganesha_nfsd_monitor() +{ + # pacemaker checks to see if RA is already running before starting it. + # if we return success, then it's presumed it's already running and + # doesn't need to be started, i.e. invoke the start action. + # return something other than success to make pacemaker invoke the + # start action + if [[ -L /var/lib/nfs ]]; then + return ${OCF_SUCCESS} + fi + return ${OCF_NOT_RUNNING} +} + +ganesha_nfsd_validate() +{ + return ${OCF_SUCCESS} +} + +ganesha_nfsd_validate + +# ocf_log notice "ganesha_nfsd ${OCF_RESOURCE_INSTANCE} $__OCF_ACTION" + +# Translate each action into the appropriate function call +case $__OCF_ACTION in +start) ganesha_nfsd_start + ;; +stop) ganesha_nfsd_stop + ;; +status|monitor) ganesha_nfsd_monitor + ;; +*) ganesha_nfsd_usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac + +rc=$? + +# The resource agent may optionally log a debug message +ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc" +exit $rc diff --git a/extras/ganesha/scripts/Makefile.am b/extras/ganesha/scripts/Makefile.am new file mode 100644 index 00000000000..7e345fd5f19 --- /dev/null +++ b/extras/ganesha/scripts/Makefile.am @@ -0,0 +1,6 @@ +EXTRA_DIST= create-export-ganesha.sh generate-epoch.py dbus-send.sh \ + ganesha-ha.sh + +scriptsdir = $(libexecdir)/ganesha +scripts_SCRIPTS = create-export-ganesha.sh dbus-send.sh generate-epoch.py \ + ganesha-ha.sh diff --git a/extras/ganesha/scripts/create-export-ganesha.sh b/extras/ganesha/scripts/create-export-ganesha.sh new file mode 100755 index 00000000000..3040e8138b0 --- /dev/null +++ b/extras/ganesha/scripts/create-export-ganesha.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +#This script is called by glusterd when the user +#tries to export a volume via NFS-Ganesha. +#An export file specific to a volume +#is created in GANESHA_DIR/exports. + +# Try loading the config from any of the distro +# specific configuration locations +if [ -f /etc/sysconfig/ganesha ] + then + . /etc/sysconfig/ganesha +fi +if [ -f /etc/conf.d/ganesha ] + then + . /etc/conf.d/ganesha +fi +if [ -f /etc/default/ganesha ] + then + . /etc/default/ganesha +fi + +GANESHA_DIR=${1%/} +OPTION=$2 +VOL=$3 +CONF=$GANESHA_DIR"/ganesha.conf" +declare -i EXPORT_ID + +function check_cmd_status() +{ + if [ "$1" != "0" ] + then + rm -rf $GANESHA_DIR/exports/export.$VOL.conf + sed -i /$VOL.conf/d $CONF + exit 1 + fi +} + + +if [ ! -d "$GANESHA_DIR/exports" ]; + then + mkdir $GANESHA_DIR/exports + check_cmd_status `echo $?` +fi + +function write_conf() +{ +echo -e "# WARNING : Using Gluster CLI will overwrite manual +# changes made to this file. To avoid it, edit the +# file and run ganesha-ha.sh --refresh-config." + +echo "EXPORT{" +echo " Export_Id = 2;" +echo " Path = \"/$VOL\";" +echo " FSAL {" +echo " name = "GLUSTER";" +echo " hostname=\"localhost\";" +echo " volume=\"$VOL\";" +echo " }" +echo " Access_type = RW;" +echo " Disable_ACL = true;" +echo ' Squash="No_root_squash";' +echo " Pseudo=\"/$VOL\";" +echo ' Protocols = "3", "4" ;' +echo ' Transports = "UDP","TCP";' +echo ' SecType = "sys";' +echo ' Security_Label = False;' +echo " }" +} +if [ "$OPTION" = "on" ]; +then + if ! (cat $CONF | grep $VOL.conf\"$ ) + then + write_conf $@ > $GANESHA_DIR/exports/export.$VOL.conf + echo "%include \"$GANESHA_DIR/exports/export.$VOL.conf\"" >> $CONF + count=`ls -l $GANESHA_DIR/exports/*.conf | wc -l` + if [ "$count" = "1" ] ; then + EXPORT_ID=2 + else + EXPORT_ID=`cat $GANESHA_DIR/.export_added` + check_cmd_status `echo $?` + EXPORT_ID=EXPORT_ID+1 + sed -i s/Export_Id.*/"Export_Id= $EXPORT_ID ;"/ \ + $GANESHA_DIR/exports/export.$VOL.conf + check_cmd_status `echo $?` + fi + echo $EXPORT_ID > $GANESHA_DIR/.export_added + fi +else + rm -rf $GANESHA_DIR/exports/export.$VOL.conf + sed -i /$VOL.conf/d $CONF +fi diff --git a/extras/ganesha/scripts/dbus-send.sh b/extras/ganesha/scripts/dbus-send.sh new file mode 100755 index 00000000000..9d613a0e7ad --- /dev/null +++ b/extras/ganesha/scripts/dbus-send.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Try loading the config from any of the distro +# specific configuration locations +if [ -f /etc/sysconfig/ganesha ] + then + . /etc/sysconfig/ganesha +fi +if [ -f /etc/conf.d/ganesha ] + then + . /etc/conf.d/ganesha +fi +if [ -f /etc/default/ganesha ] + then + . /etc/default/ganesha +fi + +GANESHA_DIR=${1%/} +OPTION=$2 +VOL=$3 +CONF=$GANESHA_DIR"/ganesha.conf" + +function check_cmd_status() +{ + if [ "$1" != "0" ] + then + logger "dynamic export failed on node :${hostname -s}" + fi +} + +#This function keeps track of export IDs and increments it with every new entry +function dynamic_export_add() +{ + dbus-send --system \ +--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ +org.ganesha.nfsd.exportmgr.AddExport string:$GANESHA_DIR/exports/export.$VOL.conf \ +string:"EXPORT(Path=/$VOL)" + check_cmd_status `echo $?` +} + +#This function removes an export dynamically(uses the export_id of the export) +function dynamic_export_remove() +{ + # Below bash fetch all the export from ShowExport command and search + # export entry based on path and then get its export entry. + # There are two possiblities for path, either entire volume will be + # exported or subdir. It handles both cases. But it remove only first + # entry from the list based on assumption that entry exported via cli + # has lowest export id value + removed_id=$(dbus-send --type=method_call --print-reply --system \ + --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ + org.ganesha.nfsd.exportmgr.ShowExports | grep -B 1 -we \ + "/"$VOL -e "/"$VOL"/" | grep uint16 | awk '{print $2}' \ + | head -1) + + dbus-send --print-reply --system \ +--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ +org.ganesha.nfsd.exportmgr.RemoveExport uint16:$removed_id + check_cmd_status `echo $?` +} + +if [ "$OPTION" = "on" ]; +then + dynamic_export_add $@ +fi + +if [ "$OPTION" = "off" ]; +then + dynamic_export_remove $@ +fi diff --git a/extras/ganesha/scripts/ganesha-ha.sh b/extras/ganesha/scripts/ganesha-ha.sh new file mode 100644 index 00000000000..9790a719e10 --- /dev/null +++ b/extras/ganesha/scripts/ganesha-ha.sh @@ -0,0 +1,1199 @@ +#!/bin/bash + +# Copyright 2015-2016 Red Hat Inc. All Rights Reserved +# +# Pacemaker+Corosync High Availability for NFS-Ganesha +# +# setup, teardown, add, delete, refresh-config, and status +# +# Each participating node in the cluster is assigned a virtual IP (VIP) +# which fails over to another node when its associated ganesha.nfsd dies +# for any reason. After the VIP is moved to another node all the +# ganesha.nfsds are send a signal using DBUS to put them into NFS GRACE. +# +# There are six resource agent types used: ganesha_mon, ganesha_grace, +# ganesha_nfsd, IPaddr, and Dummy. ganesha_mon is used to monitor the +# ganesha.nfsd. ganesha_grace is used to send the DBUS signal to put +# the remaining ganesha.nfsds into grace. ganesha_nfsd is used to start +# and stop the ganesha.nfsd during setup and teardown. IPaddr manages +# the VIP. A Dummy resource named $hostname-trigger_ip-1 is used to +# ensure that the NFS GRACE DBUS signal is sent after the VIP moves to +# the new host. + +GANESHA_HA_SH=$(realpath $0) +HA_NUM_SERVERS=0 +HA_SERVERS="" +HA_VOL_NAME="gluster_shared_storage" +HA_VOL_MNT="/run/gluster/shared_storage" +HA_CONFDIR=$HA_VOL_MNT"/nfs-ganesha" +SERVICE_MAN="DISTRO_NOT_FOUND" + +# rhel, fedora id, version +ID="" +VERSION_ID="" + +PCS9OR10_PCS_CNAME_OPTION="" +PCS9OR10_PCS_CLONE_OPTION="clone" +SECRET_PEM="/var/lib/glusterd/nfs/secret.pem" + +# UNBLOCK RA uses shared_storage which may become unavailable +# during any of the nodes reboot. Hence increase timeout value. +PORTBLOCK_UNBLOCK_TIMEOUT="60s" + +# Try loading the config from any of the distro +# specific configuration locations +if [ -f /etc/sysconfig/ganesha ] + then + . /etc/sysconfig/ganesha +fi +if [ -f /etc/conf.d/ganesha ] + then + . /etc/conf.d/ganesha +fi +if [ -f /etc/default/ganesha ] + then + . /etc/default/ganesha +fi + +GANESHA_CONF= + +function find_rhel7_conf +{ + while [[ $# > 0 ]] + do + key="$1" + case $key in + -f) + CONFFILE="$2" + break; + ;; + *) + ;; + esac + shift + done +} + +if [ -z ${CONFFILE} ] + then + find_rhel7_conf ${OPTIONS} + +fi + +GANESHA_CONF=${CONFFILE:-/etc/ganesha/ganesha.conf} + +usage() { + + echo "Usage : add|delete|refresh-config|status" + echo "Add-node : ganesha-ha.sh --add <HA_CONF_DIR> \ +<NODE-HOSTNAME> <NODE-VIP>" + echo "Delete-node: ganesha-ha.sh --delete <HA_CONF_DIR> \ +<NODE-HOSTNAME>" + echo "Refresh-config : ganesha-ha.sh --refresh-config <HA_CONFDIR> \ +<volume>" + echo "Status : ganesha-ha.sh --status <HA_CONFDIR>" +} + +determine_service_manager () { + + if [ -e "/bin/systemctl" ]; + then + SERVICE_MAN="/bin/systemctl" + elif [ -e "/sbin/invoke-rc.d" ]; + then + SERVICE_MAN="/sbin/invoke-rc.d" + elif [ -e "/sbin/service" ]; + then + SERVICE_MAN="/sbin/service" + fi + if [[ "${SERVICE_MAN}X" == "DISTRO_NOT_FOUNDX" ]] + then + logger "Service manager not recognized, exiting" + exit 1 + fi +} + +manage_service () +{ + local action=${1} + local new_node=${2} + local option= + + if [[ "${action}" == "start" ]]; then + option="yes" + else + option="no" + fi + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${new_node} "${GANESHA_HA_SH} --setup-ganesha-conf-files $HA_CONFDIR $option" + + if [[ "${SERVICE_MAN}" == "/bin/systemctl" ]] + then + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${new_node} "${SERVICE_MAN} ${action} nfs-ganesha" + else + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${new_node} "${SERVICE_MAN} nfs-ganesha ${action}" + fi +} + + +check_cluster_exists() +{ + local name=${1} + local cluster_name="" + + if [ -e /var/run/corosync.pid ]; then + cluster_name=$(pcs status | grep "Cluster name:" | cut -d ' ' -f 3) + if [[ "${cluster_name}X" == "${name}X" ]]; then + logger "$name already exists, exiting" + exit 0 + fi + fi +} + + +determine_servers() +{ + local cmd=${1} + local num_servers=0 + local tmp_ifs=${IFS} + local ha_servers="" + + if [ "${cmd}X" != "setupX" -a "${cmd}X" != "statusX" ]; then + ha_servers=$(pcs status | grep "Online:" | grep -o '\[.*\]' | sed -e 's/\[//' | sed -e 's/\]//') + IFS=$' ' + for server in ${ha_servers} ; do + num_servers=$(expr ${num_servers} + 1) + done + IFS=${tmp_ifs} + HA_NUM_SERVERS=${num_servers} + HA_SERVERS="${ha_servers}" + else + IFS=$',' + for server in ${HA_CLUSTER_NODES} ; do + num_servers=$(expr ${num_servers} + 1) + done + IFS=${tmp_ifs} + HA_NUM_SERVERS=${num_servers} + HA_SERVERS="${HA_CLUSTER_NODES//,/ }" + fi +} + +stop_ganesha_all() +{ + local serverlist=${1} + for node in ${serverlist} ; do + manage_service "stop" ${node} + done +} + +setup_cluster() +{ + local name=${1} + local num_servers=${2} + local servers=${3} + local unclean="" + local quorum_policy="stop" + + logger "setting up cluster ${name} with the following ${servers}" + + # pcs cluster setup --force ${PCS9OR10_PCS_CNAME_OPTION} ${name} ${servers} + pcs cluster setup --force ${PCS9OR10_PCS_CNAME_OPTION} ${name} --enable ${servers} + if [ $? -ne 0 ]; then + logger "pcs cluster setup ${PCS9OR10_PCS_CNAME_OPTION} ${name} --enable ${servers} failed, shutting down ganesha and bailing out" + #set up failed stop all ganesha process and clean up symlinks in cluster + stop_ganesha_all "${servers}" + exit 1; + fi + + # pcs cluster auth ${servers} + pcs cluster auth + if [ $? -ne 0 ]; then + logger "pcs cluster auth failed" + fi + + pcs cluster start --all + if [ $? -ne 0 ]; then + logger "pcs cluster start failed" + exit 1; + fi + + sleep 1 + # wait for the cluster to elect a DC before querying or writing + # to the CIB. BZ 1334092 + crmadmin --dc_lookup --timeout=5000 > /dev/null 2>&1 + while [ $? -ne 0 ]; do + crmadmin --dc_lookup --timeout=5000 > /dev/null 2>&1 + done + + unclean=$(pcs status | grep -u "UNCLEAN") + while [[ "${unclean}X" == "UNCLEANX" ]]; do + sleep 1 + unclean=$(pcs status | grep -u "UNCLEAN") + done + sleep 1 + + if [ ${num_servers} -lt 3 ]; then + quorum_policy="ignore" + fi + pcs property set no-quorum-policy=${quorum_policy} + if [ $? -ne 0 ]; then + logger "warning: pcs property set no-quorum-policy=${quorum_policy} failed" + fi + + pcs property set stonith-enabled=false + if [ $? -ne 0 ]; then + logger "warning: pcs property set stonith-enabled=false failed" + fi +} + + +setup_finalize_ha() +{ + local cibfile=${1} + local stopped="" + + stopped=$(pcs status | grep -u "Stopped") + while [[ "${stopped}X" == "StoppedX" ]]; do + sleep 1 + stopped=$(pcs status | grep -u "Stopped") + done +} + + +refresh_config () +{ + local short_host=$(hostname -s) + local VOL=${1} + local HA_CONFDIR=${2} + local short_host=$(hostname -s) + + local export_id=$(grep ^[[:space:]]*Export_Id $HA_CONFDIR/exports/export.$VOL.conf |\ + awk -F"[=,;]" '{print $2}' | tr -d '[[:space:]]') + + + if [ -e ${SECRET_PEM} ]; then + while [[ ${3} ]]; do + current_host=`echo ${3} | cut -d "." -f 1` + if [[ ${short_host} != ${current_host} ]]; then + output=$(ssh -oPasswordAuthentication=no \ +-oStrictHostKeyChecking=no -i ${SECRET_PEM} root@${current_host} \ +"dbus-send --print-reply --system --dest=org.ganesha.nfsd \ +/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.UpdateExport \ +string:$HA_CONFDIR/exports/export.$VOL.conf \ +string:\"EXPORT(Export_Id=$export_id)\" 2>&1") + ret=$? + logger <<< "${output}" + if [ ${ret} -ne 0 ]; then + echo "Refresh-config failed on ${current_host}. Please check logs on ${current_host}" + else + echo "Refresh-config completed on ${current_host}." + fi + + fi + shift + done + else + echo "Error: refresh-config failed. Passwordless ssh is not enabled." + exit 1 + fi + + # Run the same command on the localhost, + output=$(dbus-send --print-reply --system --dest=org.ganesha.nfsd \ +/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.UpdateExport \ +string:$HA_CONFDIR/exports/export.$VOL.conf \ +string:"EXPORT(Export_Id=$export_id)" 2>&1) + ret=$? + logger <<< "${output}" + if [ ${ret} -ne 0 ] ; then + echo "Refresh-config failed on localhost." + else + echo "Success: refresh-config completed." + fi +} + + +teardown_cluster() +{ + local name=${1} + + for server in ${HA_SERVERS} ; do + if [[ ${HA_CLUSTER_NODES} != *${server}* ]]; then + logger "info: ${server} is not in config, removing" + + pcs cluster stop ${server} --force + if [ $? -ne 0 ]; then + logger "warning: pcs cluster stop ${server} failed" + fi + + pcs cluster node remove ${server} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster node remove ${server} failed" + fi + fi + done + + # BZ 1193433 - pcs doesn't reload cluster.conf after modification + # after teardown completes, a subsequent setup will appear to have + # 'remembered' the deleted node. You can work around this by + # issuing another `pcs cluster node remove $node`, + # `crm_node -f -R $server`, or + # `cibadmin --delete --xml-text '<node id="$server" + # uname="$server"/>' + + pcs cluster stop --all + if [ $? -ne 0 ]; then + logger "warning pcs cluster stop --all failed" + fi + + pcs cluster destroy + if [ $? -ne 0 ]; then + logger "error pcs cluster destroy failed" + exit 1 + fi +} + + +cleanup_ganesha_config () +{ + rm -f /etc/corosync/corosync.conf + rm -rf /etc/cluster/cluster.conf* + rm -rf /var/lib/pacemaker/cib/* + sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $HA_CONFDIR/ganesha.conf +} + +do_create_virt_ip_constraints() +{ + local cibfile=${1}; shift + local primary=${1}; shift + local weight="1000" + + # first a constraint location rule that says the VIP must be where + # there's a ganesha.nfsd running + pcs -f ${cibfile} constraint location ${primary}-group rule score=-INFINITY ganesha-active ne 1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint location ${primary}-group rule score=-INFINITY ganesha-active ne 1 failed" + fi + + # then a set of constraint location prefers to set the prefered order + # for where a VIP should move + while [[ ${1} ]]; do + pcs -f ${cibfile} constraint location ${primary}-group prefers ${1}=${weight} + if [ $? -ne 0 ]; then + logger "warning: pcs constraint location ${primary}-group prefers ${1}=${weight} failed" + fi + weight=$(expr ${weight} + 1000) + shift + done + # and finally set the highest preference for the VIP to its home node + # default weight when created is/was 100. + # on Fedora setting appears to be additive, so to get the desired + # value we adjust the weight + # weight=$(expr ${weight} - 100) + pcs -f ${cibfile} constraint location ${primary}-group prefers ${primary}=${weight} + if [ $? -ne 0 ]; then + logger "warning: pcs constraint location ${primary}-group prefers ${primary}=${weight} failed" + fi +} + + +wrap_create_virt_ip_constraints() +{ + local cibfile=${1}; shift + local primary=${1}; shift + local head="" + local tail="" + + # build a list of peers, e.g. for a four node cluster, for node1, + # the result is "node2 node3 node4"; for node2, "node3 node4 node1" + # and so on. + while [[ ${1} ]]; do + if [[ ${1} == ${primary} ]]; then + shift + while [[ ${1} ]]; do + tail=${tail}" "${1} + shift + done + else + head=${head}" "${1} + fi + shift + done + do_create_virt_ip_constraints ${cibfile} ${primary} ${tail} ${head} +} + + +create_virt_ip_constraints() +{ + local cibfile=${1}; shift + + while [[ ${1} ]]; do + wrap_create_virt_ip_constraints ${cibfile} ${1} ${HA_SERVERS} + shift + done +} + + +setup_create_resources() +{ + local cibfile=$(mktemp -u) + + # fixup /var/lib/nfs + logger "pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION}" + pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION} + if [ $? -ne 0 ]; then + logger "warning: pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION} failed" + fi + + pcs resource create nfs-mon ocf:heartbeat:ganesha_mon ${PCS9OR10_PCS_CLONE_OPTION} + if [ $? -ne 0 ]; then + logger "warning: pcs resource create nfs-mon ocf:heartbeat:ganesha_mon ${PCS9OR10_PCS_CLONE_OPTION} failed" + fi + + # see comment in (/usr/lib/ocf/resource.d/heartbeat/ganesha_grace + # start method. Allow time for ganesha_mon to start and set the + # ganesha-active crm_attribute + sleep 5 + + pcs resource create nfs-grace ocf:heartbeat:ganesha_grace ${PCS9OR10_PCS_CLONE_OPTION} notify=true + if [ $? -ne 0 ]; then + logger "warning: pcs resource create nfs-grace ocf:heartbeat:ganesha_grace ${PCS9OR10_PCS_CLONE_OPTION} failed" + fi + + pcs constraint location nfs-grace-clone rule score=-INFINITY grace-active ne 1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint location nfs-grace-clone rule score=-INFINITY grace-active ne 1" + fi + + pcs cluster cib ${cibfile} + + while [[ ${1} ]]; do + + # this is variable indirection + # from a nvs like 'VIP_host1=10.7.6.5' or 'VIP_host1="10.7.6.5"' + # (or VIP_host-1=..., or VIP_host-1.my.domain.name=...) + # a variable 'clean_name' is created (e.g. w/ value 'VIP_host_1') + # and a clean nvs (e.g. w/ value 'VIP_host_1="10_7_6_5"') + # after the `eval ${clean_nvs}` there is a variable VIP_host_1 + # with the value '10_7_6_5', and the following \$$ magic to + # reference it, i.e. `eval tmp_ipaddr=\$${clean_name}` gives us + # ${tmp_ipaddr} with 10_7_6_5 and then convert the _s back to .s + # to give us ipaddr="10.7.6.5". whew! + name="VIP_${1}" + clean_name=${name//[-.]/_} + nvs=$(grep "^${name}=" ${HA_CONFDIR}/ganesha-ha.conf) + clean_nvs=${nvs//[-.]/_} + eval ${clean_nvs} + eval tmp_ipaddr=\$${clean_name} + ipaddr=${tmp_ipaddr//_/.} + + pcs -f ${cibfile} resource create ${1}-nfs_block ocf:heartbeat:portblock protocol=tcp \ + portno=2049 action=block ip=${ipaddr} --group ${1}-group + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-nfs_block failed" + fi + pcs -f ${cibfile} resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \ + cidr_netmask=32 op monitor interval=15s --group ${1}-group --after ${1}-nfs_block + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \ + cidr_netmask=32 op monitor interval=15s failed" + fi + + pcs -f ${cibfile} constraint order nfs-grace-clone then ${1}-cluster_ip-1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint order nfs-grace-clone then ${1}-cluster_ip-1 failed" + fi + + pcs -f ${cibfile} resource create ${1}-nfs_unblock ocf:heartbeat:portblock protocol=tcp \ + portno=2049 action=unblock ip=${ipaddr} reset_local_on_unblock_stop=true \ + tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${1}-group --after ${1}-cluster_ip-1 \ + op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} \ + op monitor interval=10s timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-nfs_unblock failed" + fi + + + shift + done + + create_virt_ip_constraints ${cibfile} ${HA_SERVERS} + + pcs cluster cib-push ${cibfile} + if [ $? -ne 0 ]; then + logger "warning pcs cluster cib-push ${cibfile} failed" + fi + rm -f ${cibfile} +} + + +teardown_resources() +{ + # local mntpt=$(grep ha-vol-mnt ${HA_CONFIG_FILE} | cut -d = -f 2) + + # restore /var/lib/nfs + logger "notice: pcs resource delete nfs_setup-clone" + pcs resource delete nfs_setup-clone + if [ $? -ne 0 ]; then + logger "warning: pcs resource delete nfs_setup-clone failed" + fi + + # delete -clone resource agents + # in particular delete the ganesha monitor so we don't try to + # trigger anything when we shut down ganesha next. + pcs resource delete nfs-mon-clone + if [ $? -ne 0 ]; then + logger "warning: pcs resource delete nfs-mon-clone failed" + fi + + pcs resource delete nfs-grace-clone + if [ $? -ne 0 ]; then + logger "warning: pcs resource delete nfs-grace-clone failed" + fi + + while [[ ${1} ]]; do + pcs resource delete ${1}-group + if [ $? -ne 0 ]; then + logger "warning: pcs resource delete ${1}-group failed" + fi + shift + done + +} + + +recreate_resources() +{ + local cibfile=${1}; shift + + while [[ ${1} ]]; do + # this is variable indirection + # see the comment on the same a few lines up + name="VIP_${1}" + clean_name=${name//[-.]/_} + nvs=$(grep "^${name}=" ${HA_CONFDIR}/ganesha-ha.conf) + clean_nvs=${nvs//[-.]/_} + eval ${clean_nvs} + eval tmp_ipaddr=\$${clean_name} + ipaddr=${tmp_ipaddr//_/.} + + pcs -f ${cibfile} resource create ${1}-nfs_block ocf:heartbeat:portblock protocol=tcp \ + portno=2049 action=block ip=${ipaddr} --group ${1}-group + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-nfs_block failed" + fi + pcs -f ${cibfile} resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \ + cidr_netmask=32 op monitor interval=15s --group ${1}-group --after ${1}-nfs_block + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \ + cidr_netmask=32 op monitor interval=15s failed" + fi + + pcs -f ${cibfile} constraint order nfs-grace-clone then ${1}-cluster_ip-1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint order nfs-grace-clone then ${1}-cluster_ip-1 failed" + fi + + pcs -f ${cibfile} resource create ${1}-nfs_unblock ocf:heartbeat:portblock protocol=tcp \ + portno=2049 action=unblock ip=${ipaddr} reset_local_on_unblock_stop=true \ + tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${1}-group --after ${1}-cluster_ip-1 \ + op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} \ + op monitor interval=10s timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-nfs_unblock failed" + fi + + shift + done +} + + +addnode_recreate_resources() +{ + local cibfile=${1}; shift + local add_node=${1}; shift + local add_vip=${1}; shift + + recreate_resources ${cibfile} ${HA_SERVERS} + + pcs -f ${cibfile} resource create ${add_node}-nfs_block ocf:heartbeat:portblock \ + protocol=tcp portno=2049 action=block ip=${add_vip} --group ${add_node}-group + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${add_node}-nfs_block failed" + fi + pcs -f ${cibfile} resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr \ + ip=${add_vip} cidr_netmask=32 op monitor interval=15s --group ${add_node}-group \ + --after ${add_node}-nfs_block + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr \ + ip=${add_vip} cidr_netmask=32 op monitor interval=15s failed" + fi + + pcs -f ${cibfile} constraint order nfs-grace-clone then ${add_node}-cluster_ip-1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint order nfs-grace-clone then ${add_node}-cluster_ip-1 failed" + fi + pcs -f ${cibfile} resource create ${add_node}-nfs_unblock ocf:heartbeat:portblock \ + protocol=tcp portno=2049 action=unblock ip=${add_vip} reset_local_on_unblock_stop=true \ + tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${add_node}-group --after \ + ${add_node}-cluster_ip-1 op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start \ + timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op monitor interval=10s \ + timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${add_node}-nfs_unblock failed" + fi +} + + +clear_resources() +{ + local cibfile=${1}; shift + + while [[ ${1} ]]; do + pcs -f ${cibfile} resource delete ${1}-group + if [ $? -ne 0 ]; then + logger "warning: pcs -f ${cibfile} resource delete ${1}-group" + fi + + shift + done +} + + +addnode_create_resources() +{ + local add_node=${1}; shift + local add_vip=${1}; shift + local cibfile=$(mktemp -u) + + # start HA on the new node + pcs cluster start ${add_node} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster start ${add_node} failed" + fi + + pcs cluster cib ${cibfile} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster cib ${cibfile} failed" + fi + + # delete all the -cluster_ip-1 resources, clearing + # their constraints, then create them again so we can + # recompute their constraints + clear_resources ${cibfile} ${HA_SERVERS} + addnode_recreate_resources ${cibfile} ${add_node} ${add_vip} + + HA_SERVERS="${HA_SERVERS} ${add_node}" + create_virt_ip_constraints ${cibfile} ${HA_SERVERS} + + pcs cluster cib-push ${cibfile} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster cib-push ${cibfile} failed" + fi + rm -f ${cibfile} +} + + +deletenode_delete_resources() +{ + local node=${1}; shift + local ha_servers=$(echo "${HA_SERVERS}" | sed s/${node}//) + local cibfile=$(mktemp -u) + + pcs cluster cib ${cibfile} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster cib ${cibfile} failed" + fi + + # delete all the -cluster_ip-1 and -trigger_ip-1 resources, + # clearing their constraints, then create them again so we can + # recompute their constraints + clear_resources ${cibfile} ${HA_SERVERS} + recreate_resources ${cibfile} ${ha_servers} + HA_SERVERS=$(echo "${ha_servers}" | sed -e "s/ / /") + + create_virt_ip_constraints ${cibfile} ${HA_SERVERS} + + pcs cluster cib-push ${cibfile} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster cib-push ${cibfile} failed" + fi + rm -f ${cibfile} + +} + + +deletenode_update_haconfig() +{ + local name="VIP_${1}" + local clean_name=${name//[-.]/_} + + ha_servers=$(echo ${HA_SERVERS} | sed -e "s/ /,/") + sed -i -e "s/^HA_CLUSTER_NODES=.*$/HA_CLUSTER_NODES=\"${ha_servers// /,}\"/" -e "s/^${name}=.*$//" -e "/^$/d" ${HA_CONFDIR}/ganesha-ha.conf +} + + +setup_state_volume() +{ + local mnt=${HA_VOL_MNT} + local longname="" + local shortname="" + local dname="" + local dirname="" + + longname=$(hostname) + dname=${longname#$(hostname -s)} + + while [[ ${1} ]]; do + + if [[ ${1} == *${dname} ]]; then + dirname=${1} + else + dirname=${1}${dname} + fi + + if [ ! -d ${mnt}/nfs-ganesha/tickle_dir ]; then + mkdir ${mnt}/nfs-ganesha/tickle_dir + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname} ]; then + mkdir ${mnt}/nfs-ganesha/${dirname} + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd + fi + if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/state ]; then + touch ${mnt}/nfs-ganesha/${dirname}/nfs/state + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/state + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak + fi + if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state ]; then + touch ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state + fi + for server in ${HA_SERVERS} ; do + if [[ ${server} != ${dirname} ]]; then + ln -s ${mnt}/nfs-ganesha/${server}/nfs/ganesha ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/${server} + ln -s ${mnt}/nfs-ganesha/${server}/nfs/statd ${mnt}/nfs-ganesha/${dirname}/nfs/statd/${server} + fi + done + shift + done + +} + + +enable_pacemaker() +{ + while [[ ${1} ]]; do + if [[ "${SERVICE_MAN}" == "/bin/systemctl" ]]; then + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${1} "${SERVICE_MAN} enable pacemaker" + else + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${1} "${SERVICE_MAN} pacemaker enable" + fi + shift + done +} + + +addnode_state_volume() +{ + local newnode=${1}; shift + local mnt=${HA_VOL_MNT} + local longname="" + local dname="" + local dirname="" + + longname=$(hostname) + dname=${longname#$(hostname -s)} + + if [[ ${newnode} == *${dname} ]]; then + dirname=${newnode} + else + dirname=${newnode}${dname} + fi + + if [ ! -d ${mnt}/nfs-ganesha/${dirname} ]; then + mkdir ${mnt}/nfs-ganesha/${dirname} + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd + fi + if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/state ]; then + touch ${mnt}/nfs-ganesha/${dirname}/nfs/state + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/state + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak + fi + if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state ]; then + touch ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state + fi + + for server in ${HA_SERVERS} ; do + + if [[ ${server} != ${dirname} ]]; then + ln -s ${mnt}/nfs-ganesha/${server}/nfs/ganesha ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/${server} + ln -s ${mnt}/nfs-ganesha/${server}/nfs/statd ${mnt}/nfs-ganesha/${dirname}/nfs/statd/${server} + + ln -s ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ${mnt}/nfs-ganesha/${server}/nfs/ganesha/${dirname} + ln -s ${mnt}/nfs-ganesha/${dirname}/nfs/statd ${mnt}/nfs-ganesha/${server}/nfs/statd/${dirname} + fi + done + +} + + +delnode_state_volume() +{ + local delnode=${1}; shift + local mnt=${HA_VOL_MNT} + local longname="" + local dname="" + local dirname="" + + longname=$(hostname) + dname=${longname#$(hostname -s)} + + if [[ ${delnode} == *${dname} ]]; then + dirname=${delnode} + else + dirname=${delnode}${dname} + fi + + rm -rf ${mnt}/nfs-ganesha/${dirname} + + for server in ${HA_SERVERS} ; do + if [[ ${server} != ${dirname} ]]; then + rm -f ${mnt}/nfs-ganesha/${server}/nfs/ganesha/${dirname} + rm -f ${mnt}/nfs-ganesha/${server}/nfs/statd/${dirname} + fi + done +} + + +status() +{ + local scratch=$(mktemp) + local regex_str="^${1}-cluster_ip-1" + local healthy=0 + local index=1 + local nodes + + # change tabs to spaces, strip leading spaces, including any + # new '*' at the beginning of a line introduced in pcs-0.10.x + pcs status | sed -e "s/\t/ /g" -e "s/^[ ]*\*//" -e "s/^[ ]*//" > ${scratch} + + nodes[0]=${1}; shift + + # make a regex of the configured nodes + # and initalize the nodes array for later + while [[ ${1} ]]; do + + regex_str="${regex_str}|^${1}-cluster_ip-1" + nodes[${index}]=${1} + ((index++)) + shift + done + + # print the nodes that are expected to be online + grep -E "Online:" ${scratch} + + echo + + # print the VIPs and which node they are on + grep -E "${regex_str}" < ${scratch} | cut -d ' ' -f 1,4 + + echo + + # check if the VIP and port block/unblock RAs are on the expected nodes + for n in ${nodes[*]}; do + + grep -E -x "${n}-nfs_block \(ocf::heartbeat:portblock\): Started ${n}" > /dev/null 2>&1 ${scratch} + result=$? + ((healthy+=${result})) + grep -E -x "${n}-cluster_ip-1 \(ocf::heartbeat:IPaddr\): Started ${n}" > /dev/null 2>&1 ${scratch} + result=$? + ((healthy+=${result})) + grep -E -x "${n}-nfs_unblock \(ocf::heartbeat:portblock\): Started ${n}" > /dev/null 2>&1 ${scratch} + result=$? + ((healthy+=${result})) + done + + grep -E "\):\ Stopped|FAILED" > /dev/null 2>&1 ${scratch} + result=$? + + if [ ${result} -eq 0 ]; then + echo "Cluster HA Status: BAD" + elif [ ${healthy} -eq 0 ]; then + echo "Cluster HA Status: HEALTHY" + else + echo "Cluster HA Status: FAILOVER" + fi + + rm -f ${scratch} +} + +create_ganesha_conf_file() +{ + if [[ "$1" == "yes" ]]; + then + if [ -e $GANESHA_CONF ]; + then + rm -rf $GANESHA_CONF + fi + # The symlink /etc/ganesha/ganesha.conf need to be + # created using ganesha conf file mentioned in the + # shared storage. Every node will only have this + # link and actual file will stored in shared storage, + # so that ganesha conf editing of ganesha conf will + # be easy as well as it become more consistent. + + ln -s $HA_CONFDIR/ganesha.conf $GANESHA_CONF + else + # Restoring previous file + rm -rf $GANESHA_CONF + cp $HA_CONFDIR/ganesha.conf $GANESHA_CONF + sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $GANESHA_CONF + fi +} + +set_quorum_policy() +{ + local quorum_policy="stop" + local num_servers=${1} + + if [ ${num_servers} -lt 3 ]; then + quorum_policy="ignore" + fi + pcs property set no-quorum-policy=${quorum_policy} + if [ $? -ne 0 ]; then + logger "warning: pcs property set no-quorum-policy=${quorum_policy} failed" + fi +} + +main() +{ + + local cmd=${1}; shift + if [[ ${cmd} == *help ]]; then + usage + exit 0 + fi + + if (selinuxenabled) ;then + semanage boolean -m gluster_use_execmem --on + fi + + local osid="" + + osid=$(grep ^ID= /etc/os-release) + eval $(echo ${osid} | grep -F ID=) + osid=$(grep ^VERSION_ID= /etc/os-release) + eval $(echo ${osid} | grep -F VERSION_ID=) + + HA_CONFDIR=${1%/}; shift + local ha_conf=${HA_CONFDIR}/ganesha-ha.conf + local node="" + local vip="" + + # ignore any comment lines + cfgline=$(grep ^HA_NAME= ${ha_conf}) + eval $(echo ${cfgline} | grep -F HA_NAME=) + cfgline=$(grep ^HA_CLUSTER_NODES= ${ha_conf}) + eval $(echo ${cfgline} | grep -F HA_CLUSTER_NODES=) + + case "${cmd}" in + + setup | --setup) + logger "setting up ${HA_NAME}" + + check_cluster_exists ${HA_NAME} + + determine_servers "setup" + + # Fedora 29+ and rhel/centos 8 has PCS-0.10.x + # default is pcs-0.10.x options but check for + # rhel/centos 7 (pcs-0.9.x) and adjust accordingly + if [[ ! ${ID} =~ {rhel,centos} ]]; then + if [[ ${VERSION_ID} == 7.* ]]; then + PCS9OR10_PCS_CNAME_OPTION="--name" + PCS9OR10_PCS_CLONE_OPTION="--clone" + fi + fi + + if [[ "${HA_NUM_SERVERS}X" != "1X" ]]; then + + determine_service_manager + + setup_cluster ${HA_NAME} ${HA_NUM_SERVERS} "${HA_SERVERS}" + + setup_create_resources ${HA_SERVERS} + + setup_finalize_ha + + setup_state_volume ${HA_SERVERS} + + enable_pacemaker ${HA_SERVERS} + + else + + logger "insufficient servers for HA, aborting" + fi + ;; + + teardown | --teardown) + logger "tearing down ${HA_NAME}" + + determine_servers "teardown" + + teardown_resources ${HA_SERVERS} + + teardown_cluster ${HA_NAME} + + cleanup_ganesha_config ${HA_CONFDIR} + ;; + + cleanup | --cleanup) + cleanup_ganesha_config ${HA_CONFDIR} + ;; + + add | --add) + node=${1}; shift + vip=${1}; shift + + logger "adding ${node} with ${vip} to ${HA_NAME}" + + determine_service_manager + + manage_service "start" ${node} + + determine_servers "add" + + pcs cluster node add ${node} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster node add ${node} failed" + fi + + addnode_create_resources ${node} ${vip} + # Subsequent add-node recreates resources for all the nodes + # that already exist in the cluster. The nodes are picked up + # from the entries in the ganesha-ha.conf file. Adding the + # newly added node to the file so that the resources specfic + # to this node is correctly recreated in the future. + clean_node=${node//[-.]/_} + echo "VIP_${node}=\"${vip}\"" >> ${HA_CONFDIR}/ganesha-ha.conf + + NEW_NODES="$HA_CLUSTER_NODES,${node}" + + sed -i s/HA_CLUSTER_NODES.*/"HA_CLUSTER_NODES=\"$NEW_NODES\""/ \ +$HA_CONFDIR/ganesha-ha.conf + + addnode_state_volume ${node} + + # addnode_create_resources() already appended ${node} to + # HA_SERVERS, so only need to increment HA_NUM_SERVERS + # and set quorum policy + HA_NUM_SERVERS=$(expr ${HA_NUM_SERVERS} + 1) + set_quorum_policy ${HA_NUM_SERVERS} + ;; + + delete | --delete) + node=${1}; shift + + logger "deleting ${node} from ${HA_NAME}" + + determine_servers "delete" + + deletenode_delete_resources ${node} + + pcs cluster node remove ${node} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster node remove ${node} failed" + fi + + deletenode_update_haconfig ${node} + + delnode_state_volume ${node} + + determine_service_manager + + manage_service "stop" ${node} + + HA_NUM_SERVERS=$(expr ${HA_NUM_SERVERS} - 1) + set_quorum_policy ${HA_NUM_SERVERS} + ;; + + status | --status) + determine_servers "status" + + status ${HA_SERVERS} + ;; + + refresh-config | --refresh-config) + VOL=${1} + + determine_servers "refresh-config" + + refresh_config ${VOL} ${HA_CONFDIR} ${HA_SERVERS} + ;; + + setup-ganesha-conf-files | --setup-ganesha-conf-files) + + create_ganesha_conf_file ${1} + ;; + + *) + # setup and teardown are not intended to be used by a + # casual user + usage + logger "Usage: ganesha-ha.sh add|delete|status" + ;; + + esac + + if (selinuxenabled) ;then + semanage boolean -m gluster_use_execmem --off + fi +} + +main $* diff --git a/extras/ganesha/scripts/generate-epoch.py b/extras/ganesha/scripts/generate-epoch.py new file mode 100755 index 00000000000..77af014bab9 --- /dev/null +++ b/extras/ganesha/scripts/generate-epoch.py @@ -0,0 +1,48 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. +# +# Generates unique epoch value on each gluster node to be used by +# nfs-ganesha service on that node. +# +# Configure 'EPOCH_EXEC' option to this script path in +# '/etc/sysconfig/ganesha' file used by nfs-ganesha service. +# +# Construct epoch as follows - +# first 32-bit contains the now() time +# rest 32-bit value contains the local glusterd node uuid + +import time +import binascii + +# Calculate the now() time into a 64-bit integer value +def epoch_now(): + epoch_time = int(time.mktime(time.localtime())) << 32 + return epoch_time + +# Read glusterd UUID and extract first 32-bit of it +def epoch_uuid(): + file_name = '/var/lib/glusterd/glusterd.info' + + for line in open(file_name): + if "UUID" in line: + glusterd_uuid = line.split('=')[1].strip() + + uuid_bin = binascii.unhexlify(glusterd_uuid.replace("-","")) + + epoch_uuid = int(binascii.hexlify(uuid_bin), 32) & 0xFFFF0000 + return epoch_uuid + +# Construct epoch as follows - +# first 32-bit contains the now() time +# rest 32-bit value contains the local glusterd node uuid +epoch = (epoch_now() | epoch_uuid()) +print((str(epoch))) + +exit(0) diff --git a/extras/geo-rep/Makefile.am b/extras/geo-rep/Makefile.am index 6d3d10b88a0..09eff308ac4 100644 --- a/extras/geo-rep/Makefile.am +++ b/extras/geo-rep/Makefile.am @@ -1,12 +1,16 @@ -scriptsdir = $(datadir)/glusterfs/scripts -scripts_DATA = gsync-upgrade.sh generate-gfid-file.sh get-gfid.sh \ - slave-upgrade.sh +scriptsdir = $(libexecdir)/glusterfs/scripts +scripts_SCRIPTS = gsync-upgrade.sh generate-gfid-file.sh get-gfid.sh \ + slave-upgrade.sh schedule_georep.py scripts_PROGRAMS = gsync-sync-gfid gsync_sync_gfid_CFLAGS = $(GF_CFLAGS) -Wall -I$(top_srcdir)/libglusterfs/src gsync_sync_gfid_LDFLAGS = $(GF_LDFLAGS) -gsync_sync_gfid_LDADD = $(GF_LIBS) $(top_builddir)/libglusterfs/src/libglusterfs.la +gsync_sync_gfid_LDADD = $(GF_LDADD) $(top_builddir)/libglusterfs/src/libglusterfs.la gsync_sync_gfid_SOURCES = gsync-sync-gfid.c +gsync_sync_gfid_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ + -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src EXTRA_DIST = gsync-sync-gfid.c gsync-upgrade.sh generate-gfid-file.sh \ - get-gfid.sh slave-upgrade.sh + get-gfid.sh slave-upgrade.sh schedule_georep.py.in + +CLEANFILES = schedule_georep.py diff --git a/extras/geo-rep/generate-gfid-file.sh b/extras/geo-rep/generate-gfid-file.sh index c6739fbf140..14f104b986d 100644 --- a/extras/geo-rep/generate-gfid-file.sh +++ b/extras/geo-rep/generate-gfid-file.sh @@ -1,11 +1,12 @@ #!/bin/bash -#Usage: generate-gfid-file.sh <master-volfile-server:master-volume> <path-to-get-gfid.sh> <output-file> +#Usage: generate-gfid-file.sh <master-volfile-server:master-volume> <path-to-get-gfid.sh> <output-file> [dirs-list-file] function get_gfids() { GET_GFID_CMD=$1 OUTPUT_FILE=$2 - find . -exec $GET_GFID_CMD {} \; > $OUTPUT_FILE + DIR_PATH=$3 + find "$DIR_PATH" -exec $GET_GFID_CMD {} \; >> $OUTPUT_FILE } function mount_client() @@ -18,7 +19,7 @@ function mount_client() GFID_CMD=$3; OUTPUT=$4; - T=$(mktemp -d); + T=$(mktemp -d -t ${0##*/}.XXXXXX); glusterfs -s $VOLFILE_SERVER --volfile-id $VOLUME $T; @@ -27,8 +28,18 @@ function mount_client() [ "x$i" = "x1" ] || fatal "could not mount volume $MASTER on $T"; cd $T; - - get_gfids $GFID_CMD $OUTPUT + rm -f $OUTPUT; + touch $OUTPUT; + + if [ "$DIRS_FILE" = "." ] + then + get_gfids $GFID_CMD $OUTPUT "." + else + while read line + do + get_gfids $GFID_CMD $OUTPUT "$line" + done < $DIRS_FILE + fi; cd -; @@ -47,7 +58,13 @@ function main() VOLFILE_SERVER=`echo $SLAVE | sed -e 's/\(.*\):.*/\1/'` VOLUME_NAME=`echo $SLAVE | sed -e 's/.*:\(.*\)/\1/'` - mount_client $VOLFILE_SERVER $VOLUME_NAME $GET_GFID_CMD $OUTPUT + if [ "$#" -lt 4 ] + then + DIRS_FILE="." + else + DIRS_FILE=$4 + fi + mount_client $VOLFILE_SERVER $VOLUME_NAME $GET_GFID_CMD $OUTPUT $DIRS_FILE } main "$@"; diff --git a/extras/geo-rep/gsync-sync-gfid.c b/extras/geo-rep/gsync-sync-gfid.c index e9b9e633402..47dca0413e9 100644 --- a/extras/geo-rep/gsync-sync-gfid.c +++ b/extras/geo-rep/gsync-sync-gfid.c @@ -7,103 +7,103 @@ #include <libgen.h> #include <ctype.h> #include <stdlib.h> -#include "glusterfs.h" -#include "syscall.h" +#include <glusterfs/glusterfs.h> +#include <glusterfs/syscall.h> #ifndef UUID_CANONICAL_FORM_LEN #define UUID_CANONICAL_FORM_LEN 36 #endif #ifndef GF_FUSE_AUX_GFID_HEAL -#define GF_FUSE_AUX_GFID_HEAL "glusterfs.gfid.heal" +#define GF_FUSE_AUX_GFID_HEAL "glusterfs.gfid.heal" #endif -#define GLFS_LINE_MAX (PATH_MAX + (2 * UUID_CANONICAL_FORM_LEN)) +#define GLFS_LINE_MAX (PATH_MAX + (2 * UUID_CANONICAL_FORM_LEN)) int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - char *file = NULL; - char *tmp = NULL; - char *tmp1 = NULL; - char *parent_dir = NULL; - char *gfid = NULL; - char *bname = NULL; - int ret = -1; - int len = 0; - FILE *fp = NULL; - char line[GLFS_LINE_MAX] = {0,}; - char *path = NULL; - void *blob = NULL; - void *tmp_blob = NULL; - - if (argc != 2) { - /* each line in the file has the following format - * uuid-in-canonical-form path-relative-to-gluster-mount. - * Both uuid and relative path are from master mount. - */ - fprintf (stderr, "usage: %s <file-of-paths-to-be-synced>\n", - argv[0]); - goto out; + char *file = NULL; + char *tmp = NULL; + char *tmp1 = NULL; + char *parent_dir = NULL; + char *gfid = NULL; + char *bname = NULL; + int ret = -1; + int len = 0; + FILE *fp = NULL; + char line[GLFS_LINE_MAX] = { + 0, + }; + char *path = NULL; + void *blob = NULL; + void *tmp_blob = NULL; + + if (argc != 2) { + /* each line in the file has the following format + * uuid-in-canonical-form path-relative-to-gluster-mount. + * Both uuid and relative path are from master mount. + */ + fprintf(stderr, "usage: %s <file-of-paths-to-be-synced>\n", argv[0]); + goto out; + } + + file = argv[1]; + + fp = fopen(file, "r"); + if (fp == NULL) { + fprintf(stderr, "cannot open %s for reading (%s)\n", file, + strerror(errno)); + goto out; + } + + while (fgets(line, GLFS_LINE_MAX, fp) != NULL) { + tmp = line; + path = gfid = line; + + path += UUID_CANONICAL_FORM_LEN + 1; + + while (isspace(*path)) + path++; + + len = strlen(line); + if ((len < GLFS_LINE_MAX) && (line[len - 1] == '\n')) + line[len - 1] = '\0'; + + line[UUID_CANONICAL_FORM_LEN] = '\0'; + + tmp = strdup(path); + tmp1 = strdup(path); + parent_dir = dirname(tmp); + bname = basename(tmp1); + + /* gfid + '\0' + bname + '\0' */ + len = UUID_CANONICAL_FORM_LEN + 1 + strlen(bname) + 1; + + blob = malloc(len); + + memcpy(blob, gfid, UUID_CANONICAL_FORM_LEN); + + tmp_blob = blob + UUID_CANONICAL_FORM_LEN + 1; + + memcpy(tmp_blob, bname, strlen(bname)); + + ret = sys_lsetxattr(parent_dir, GF_FUSE_AUX_GFID_HEAL, blob, len, 0); + if (ret < 0) { + fprintf(stderr, "setxattr on %s/%s failed (%s)\n", parent_dir, + bname, strerror(errno)); } + memset(line, 0, GLFS_LINE_MAX); - file = argv[1]; + free(blob); + free(tmp); + free(tmp1); + blob = NULL; + } - fp = fopen (file, "r"); - if (fp == NULL) { - fprintf (stderr, "cannot open %s for reading (%s)\n", - file, strerror (errno)); - goto out; - } - - while (fgets (line, GLFS_LINE_MAX, fp) != NULL) { - tmp = line; - path = gfid = line; - - path += UUID_CANONICAL_FORM_LEN + 1; - - while(isspace (*path)) - path++; - - if ((strlen (line) < GLFS_LINE_MAX) && - (line[strlen (line) - 1] == '\n')) - line[strlen (line) - 1] = '\0'; - - line[UUID_CANONICAL_FORM_LEN] = '\0'; - - tmp = strdup (path); - tmp1 = strdup (path); - parent_dir = dirname (tmp); - bname = basename (tmp1); - - /* gfid + '\0' + bname + '\0' */ - len = UUID_CANONICAL_FORM_LEN + 1 + strlen (bname) + 1; - - blob = calloc (1, len); - - memcpy (blob, gfid, UUID_CANONICAL_FORM_LEN); - - tmp_blob = blob + UUID_CANONICAL_FORM_LEN + 1; - - memcpy (tmp_blob, bname, strlen (bname)); - - ret = sys_lsetxattr (parent_dir, GF_FUSE_AUX_GFID_HEAL, - blob, len, 0); - if (ret < 0) { - fprintf (stderr, "setxattr on %s/%s failed (%s)\n", - parent_dir, bname, strerror (errno)); - } - memset (line, 0, GLFS_LINE_MAX); - - free (blob); - free (tmp); free (tmp1); - blob = NULL; - } - - ret = 0; + ret = 0; out: - if (fp) - fclose(fp); - return ret; + if (fp) + fclose(fp); + return ret; } - diff --git a/extras/geo-rep/gsync-upgrade.sh b/extras/geo-rep/gsync-upgrade.sh index b179487365a..0f73a33884b 100644 --- a/extras/geo-rep/gsync-upgrade.sh +++ b/extras/geo-rep/gsync-upgrade.sh @@ -75,7 +75,7 @@ function mount_client() GFID_FILE=$3 SYNC_CMD=$4 - T=$(mktemp -d); + T=$(mktemp -d -t ${0##*/}.XXXXXX); glusterfs --aux-gfid-mount -s $1 --volfile-id $2 $T; diff --git a/extras/geo-rep/schedule_georep.py.in b/extras/geo-rep/schedule_georep.py.in new file mode 100644 index 00000000000..48b2b507060 --- /dev/null +++ b/extras/geo-rep/schedule_georep.py.in @@ -0,0 +1,492 @@ +#!/usr/bin/python3 +""" +Schedule Geo-replication +------------------------ +A tool to run Geo-replication when required. This can be used to +schedule the Geo-replication to run once in a day using + + # Run daily at 08:30pm + 30 20 * * * root python /usr/share/glusterfs/scripts/schedule_georep.py \\ + --no-color gv1 fvm1 gv2 >> /var/log/glusterfs/schedule_georep.log 2>&1 + +This tool does the following, + +1. Stop Geo-replication if Started +2. Start Geo-replication +3. Set Checkpoint +4. Check the Status and see Checkpoint is Complete.(LOOP) +5. If checkpoint complete, Stop Geo-replication + +Usage: + + python /usr/share/glusterfs/scripts/schedule_georep.py <MASTERVOL> \\ + <SLAVEHOST> <SLAVEVOL> + +For example, + + python /usr/share/glusterfs/scripts/schedule_georep.py gv1 fvm1 gv2 + +""" +import subprocess +import time +import xml.etree.cElementTree as etree +import sys +from contextlib import contextmanager +import tempfile +import os +from argparse import ArgumentParser, RawDescriptionHelpFormatter + +ParseError = etree.ParseError if hasattr(etree, 'ParseError') else SyntaxError +cache_data = {} + +SESSION_MOUNT_LOG_FILE = ("/var/log/glusterfs/geo-replication" + "/schedule_georep.mount.log") + +USE_CLI_COLOR = True +mnt_list = [] + +class GlusterBadXmlFormat(Exception): + """ + Exception class for XML Parse Errors + """ + pass + + +def output_notok(msg, err="", exitcode=1): + if USE_CLI_COLOR: + out = "\033[31m[NOT OK]\033[0m {0}\n{1}\n" + else: + out = "[NOT OK] {0}\n{1}\n" + sys.stderr.write(out.format(msg, err)) + sys.exit(exitcode) + + +def output_warning(msg): + if USE_CLI_COLOR: + out = "\033[33m[ WARN]\033[0m {0}\n" + else: + out = "[ WARN] {0}\n" + sys.stderr.write(out.format(msg)) + + +def output_ok(msg): + if USE_CLI_COLOR: + out = "\033[32m[ OK]\033[0m {0}\n" + else: + out = "[ OK] {0}\n" + sys.stderr.write(out.format(msg)) + + +def execute(cmd, success_msg="", failure_msg="", exitcode=-1): + """ + Generic wrapper to execute the CLI commands. Returns Output if success. + On success it can print message in stdout if specified. + On failure, exits after writing to stderr. + """ + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + out, err = p.communicate() + if p.returncode == 0: + if success_msg: + output_ok(success_msg) + return out + else: + if exitcode == 0: + return + err_msg = err if err else out + output_notok(failure_msg, err=err_msg, exitcode=exitcode) + + +def cache_output_with_args(func): + """ + Decorator function to remember the output of any function + """ + def wrapper(*args, **kwargs): + global cache_data + key = "_".join([func.func_name] + list(args)) + if cache_data.get(key, None) is None: + cache_data[key] = func(*args, **kwargs) + + return cache_data[key] + return wrapper + + +def cleanup(hostname, volname, mnt): + """ + Unmount the Volume and Remove the temporary directory + """ + execute(["umount", "-l", mnt], + failure_msg="Unable to Unmount Gluster Volume " + "{0}:{1}(Mounted at {2})".format(hostname, volname, mnt)) + execute(["rmdir", mnt], + failure_msg="Unable to Remove temp directory " + "{0}".format(mnt), exitcode=0) + + +@contextmanager +def glustermount(hostname, volname): + """ + Context manager for Mounting Gluster Volume + Use as + with glustermount(HOSTNAME, VOLNAME) as MNT: + # Do your stuff + Automatically unmounts it in case of Exceptions/out of context + """ + mnt = tempfile.mkdtemp(prefix="georepsetup_") + mnt_list.append(mnt) + execute(["@SBIN_DIR@/glusterfs", + "--volfile-server", hostname, + "--volfile-id", volname, + "-l", SESSION_MOUNT_LOG_FILE, + mnt], + failure_msg="Unable to Mount Gluster Volume " + "{0}:{1}".format(hostname, volname)) + if os.path.ismount(mnt): + yield mnt + else: + output_notok("Unable to Mount Gluster Volume " + "{0}:{1}".format(hostname, volname)) + cleanup(hostname, volname, mnt) + + +@cache_output_with_args +def get_bricks(volname): + """ + Returns Bricks list, caches the Bricks list for a volume once + parsed. + """ + value = [] + cmd = ["@SBIN_DIR@/gluster", "volume", "info", volname, "--xml"] + info = execute(cmd) + try: + tree = etree.fromstring(info) + volume_el = tree.find('volInfo/volumes/volume') + for b in volume_el.findall('bricks/brick'): + value.append({"name": b.find("name").text, + "hostUuid": b.find("hostUuid").text}) + except ParseError: + raise GlusterBadXmlFormat("Bad XML Format: %s" % " ".join(cmd)) + + return value + + +def get_georep_status(mastervol, slave): + session_keys = set() + out = {} + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication"] + if mastervol is not None: + cmd += [mastervol] + if slave: + cmd += [slave] + + cmd += ["status", "--xml"] + info = execute(cmd) + + try: + tree = etree.fromstring(info) + # Get All Sessions + for volume_el in tree.findall("geoRep/volume"): + sessions_el = volume_el.find("sessions") + # Master Volume name if multiple Volumes + mvol = volume_el.find("name").text + + # For each session, collect the details + for session in sessions_el.findall("session"): + session_slave = "{0}:{1}".format(mvol, session.find( + "session_slave").text) + session_keys.add(session_slave) + out[session_slave] = {} + + for pair in session.findall('pair'): + master_brick = "{0}:{1}".format( + pair.find("master_node").text, + pair.find("master_brick").text + ) + + out[session_slave][master_brick] = { + "mastervol": mvol, + "slavevol": pair.find("slave").text.split("::")[-1], + "master_node": pair.find("master_node").text, + "master_brick": pair.find("master_brick").text, + "slave_user": pair.find("slave_user").text, + "slave": pair.find("slave").text, + "slave_node": pair.find("slave_node").text, + "status": pair.find("status").text, + "crawl_status": pair.find("crawl_status").text, + "entry": pair.find("entry").text, + "data": pair.find("data").text, + "meta": pair.find("meta").text, + "failures": pair.find("failures").text, + "checkpoint_completed": pair.find( + "checkpoint_completed").text, + "master_node_uuid": pair.find("master_node_uuid").text, + "last_synced": pair.find("last_synced").text, + "checkpoint_time": pair.find("checkpoint_time").text, + "checkpoint_completion_time": + pair.find("checkpoint_completion_time").text + } + except ParseError: + raise GlusterBadXmlFormat("Bad XML Format: %s" % " ".join(cmd)) + + return session_keys, out + + +def get_offline_status(volname, brick, node_uuid, slave): + node, brick = brick.split(":") + if "@" not in slave: + slave_user = "root" + else: + slave_user, _ = slave.split("@") + + return { + "mastervol": volname, + "slavevol": slave.split("::")[-1], + "master_node": node, + "master_brick": brick, + "slave_user": slave_user, + "slave": slave, + "slave_node": "N/A", + "status": "Offline", + "crawl_status": "N/A", + "entry": "N/A", + "data": "N/A", + "meta": "N/A", + "failures": "N/A", + "checkpoint_completed": "N/A", + "master_node_uuid": node_uuid, + "last_synced": "N/A", + "checkpoint_time": "N/A", + "checkpoint_completion_time": "N/A" + } + + +def get(mastervol=None, slave=None): + """ + This function gets list of Bricks of Master Volume and collects + respective Geo-rep status. Output will be always ordered as the + bricks list in Master Volume. If Geo-rep status is not available + for any brick then it updates OFFLINE status. + """ + out = [] + session_keys, gstatus = get_georep_status(mastervol, slave) + + for session in session_keys: + mvol, _, slave = session.split(":", 2) + slave = slave.replace("ssh://", "") + master_bricks = get_bricks(mvol) + out.append([]) + for brick in master_bricks: + bname = brick["name"] + if gstatus.get(session) and gstatus[session].get(bname, None): + out[-1].append(gstatus[session][bname]) + else: + out[-1].append( + get_offline_status(mvol, bname, brick["hostUuid"], slave)) + + return out + + +def get_summary(mastervol, slave_url): + """ + Wrapper function around Geo-rep Status and Gluster Volume Info + This combines the output from Bricks list and Geo-rep Status. + If a Master Brick node is down or Status is faulty then increments + the faulty counter. It also collects the checkpoint status from all + workers and compares with Number of Bricks. + """ + down_rows = [] + faulty_rows = [] + out = [] + + status_data = get(mastervol, slave_url) + + for session in status_data: + session_name = "" + summary = { + "active": 0, + "passive": 0, + "faulty": 0, + "initializing": 0, + "stopped": 0, + "created": 0, + "offline": 0, + "paused": 0, + "workers": 0, + "completed_checkpoints": 0, + "checkpoint": False, + "checkpoints_ok": False, + "ok": False + } + + for row in session: + summary[row["status"].replace("...", "").lower()] += 1 + summary["workers"] += 1 + if row["checkpoint_completed"] == "Yes": + summary["completed_checkpoints"] += 1 + + session_name = "{0}=>{1}".format( + row["mastervol"], + row["slave"].replace("ssh://", "") + ) + + if row["status"] == "Faulty": + faulty_rows.append("{0}:{1}".format(row["master_node"], + row["master_brick"])) + + if row["status"] == "Offline": + down_rows.append("{0}:{1}".format(row["master_node"], + row["master_brick"])) + + if summary["active"] == summary["completed_checkpoints"] and \ + summary["faulty"] == 0 and summary["offline"] == 0: + summary["checkpoints_ok"] = True + + if summary["faulty"] == 0 and summary["offline"] == 0: + summary["ok"] = True + + if session_name != "": + out.append([session_name, summary, faulty_rows, down_rows]) + + return out + + +def touch_mount_root(mastervol): + # Create a Mount and Touch the Mount point root, + # Hack to make sure some event available after + # setting Checkpoint. Without this there is a chance of + # Checkpoint never completes. + with glustermount("localhost", mastervol) as mnt: + execute(["touch", mnt]) + + +def main(args): + turns = 1 + + # Stop Force + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", args.mastervol, + "%s::%s" % (args.slave, args.slavevol), "stop", "force"] + execute(cmd) + output_ok("Stopped Geo-replication") + + # Set Checkpoint to NOW + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", args.mastervol, + "%s::%s" % (args.slave, args.slavevol), "config", "checkpoint", + "now"] + execute(cmd) + output_ok("Set Checkpoint") + + # Start the Geo-replication + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", args.mastervol, + "%s::%s" % (args.slave, args.slavevol), "start"] + execute(cmd) + output_ok("Started Geo-replication and watching Status for " + "Checkpoint completion") + + start_time = int(time.time()) + duration = 0 + + # Sleep till Geo-rep initializes + time.sleep(60) + + touch_mount_root(args.mastervol) + + slave_url = "{0}::{1}".format(args.slave, args.slavevol) + + # Loop to Check the Geo-replication Status and Checkpoint + # If All Status OK and all Checkpoints complete, + # Stop the Geo-replication and Log the Completeness + while True: + session_summary = get_summary(args.mastervol, + slave_url) + if len(session_summary) == 0: + # If Status command fails with another transaction error + # or any other error. Gluster cmd still produces XML output + # with different message + output_warning("Unable to get Geo-replication Status") + else: + session_name, summary, faulty_rows, down_rows = session_summary[0] + chkpt_status = "COMPLETE" if summary["checkpoints_ok"] else \ + "NOT COMPLETE" + ok_status = "OK" if summary["ok"] else "NOT OK" + + if summary["ok"]: + output_ok("All Checkpoints {1}, " + "All status {2} (Turns {0:>3})".format( + turns, chkpt_status, ok_status)) + else: + output_warning("All Checkpoints {1}, " + "All status {2} (Turns {0:>3})".format( + turns, chkpt_status, ok_status)) + + output_warning("Geo-rep workers Faulty/Offline, " + "Faulty: {0} Offline: {1}".format( + repr(faulty_rows), + repr(down_rows))) + + if summary["checkpoints_ok"]: + output_ok("Stopping Geo-replication session now") + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", + args.mastervol, + "%s::%s" % (args.slave, args.slavevol), "stop"] + execute(cmd) + break + else: + # If Checkpoint is not complete after a iteration means brick + # was down and came online now. SETATTR on mount is not + # recorded, So again issue touch on mount root So that + # Stime will increase and Checkpoint will complete. + touch_mount_root(args.mastervol) + + # Increment the turns and Sleep for 10 sec + turns += 1 + duration = int(time.time()) - start_time + if args.timeout > 0 and duration > (args.timeout * 60): + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", + args.mastervol, + "%s::%s" % (args.slave, args.slavevol), "stop", "force"] + execute(cmd) + output_notok("Timed out, Stopping Geo-replication(" + "Duration: {0}sec)".format(duration)) + + time.sleep(args.interval) + + for mnt in mnt_list: + execute(["rmdir", mnt], + failure_msg="Unable to Remove temp directory " + "{0}".format(mnt), exitcode=0) + +if __name__ == "__main__": + parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter, + description=__doc__) + parser.add_argument("mastervol", help="Master Volume Name") + parser.add_argument("slave", + help="Slave hostname " + "(<username>@SLAVEHOST or SLAVEHOST)", + metavar="SLAVE") + parser.add_argument("slavevol", help="Slave Volume Name") + parser.add_argument("--interval", help="Interval in Seconds. " + "Wait time before each status check", + type=int, default=10) + parser.add_argument("--timeout", help="Timeout in minutes. Script will " + "stop Geo-replication if Checkpoint is not complete " + "in the specified timeout time", type=int, + default=0) + parser.add_argument("--no-color", help="Don't use Color in CLI output", + action="store_true") + args = parser.parse_args() + if args.no_color: + USE_CLI_COLOR = False + try: + # Check for session existence + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", + args.mastervol, "%s::%s" % (args.slave, args.slavevol), "status"] + execute(cmd) + main(args) + except KeyboardInterrupt: + for mnt in mnt_list: + execute(["umount", "-l", mnt], + failure_msg="Unable to Unmount Gluster Volume " + "Mounted at {0}".format(mnt), exitcode=0) + execute(["rmdir", mnt], + failure_msg="Unable to Remove temp directory " + "{0}".format(mnt), exitcode=0) + output_notok("Exiting...") diff --git a/extras/geo-rep/slave-upgrade.sh b/extras/geo-rep/slave-upgrade.sh index 6198f408af7..3a37f8e3579 100644 --- a/extras/geo-rep/slave-upgrade.sh +++ b/extras/geo-rep/slave-upgrade.sh @@ -55,7 +55,7 @@ function mount_client() GFID_FILE=$3 SYNC_CMD=$4 - T=$(mktemp -d); + T=$(mktemp -d -t ${0##*/}.XXXXXX); glusterfs --aux-gfid-mount -s $1 --volfile-id $VOLUME_NAME $T; diff --git a/extras/gfid-to-dirname.sh b/extras/gfid-to-dirname.sh new file mode 100755 index 00000000000..fd359fab58a --- /dev/null +++ b/extras/gfid-to-dirname.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +function read_symlink() +{ + DOT_GLUSTERFS_PATH=$BRICK_PATH/.glusterfs + gfid_string=$1 + symlink_path="$DOT_GLUSTERFS_PATH/${gfid_string:0:2}/${gfid_string:2:2}/$gfid_string" + #remove trailing '/' + symlink_path=${symlink_path%/} + linkname=$(readlink $symlink_path) + if [ $? -ne 0 ]; then + echo "readlink of $symlink_path returned an error." >&2 + exit -1 + fi + echo $linkname +} + +main() +{ + if [ $# -lt 2 ] ;then + echo "Usage: $0 <brick-path> <gfid-string-of-directory>" + echo "Example: $0 /bricks/brick1 1b835012-1ae5-4f0d-9db4-64de574d891c" + exit -1 + fi + + BRICK_PATH=$1 + name=$(read_symlink $2) + if [ $? -ne 0 ]; then + exit -1 + fi + + while [ ${name:12:36} != "00000000-0000-0000-0000-000000000001" ] + do + LOCATION=`basename $name`/$LOCATION + GFID_STRING=${name:12:36} + name=$(read_symlink $GFID_STRING) + if [ $? -ne 0 ]; then + exit -1 + fi + done + + LOCATION=`basename $name`/$LOCATION + echo "Location of the directory corresponding to gfid:$2 is $BRICK_PATH/$LOCATION" +} + +main "$@" diff --git a/extras/git-branch-diff.py b/extras/git-branch-diff.py new file mode 100755 index 00000000000..382513e069e --- /dev/null +++ b/extras/git-branch-diff.py @@ -0,0 +1,285 @@ +#!/bin/python2 + +""" + Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +""" + +""" + ABOUT: + This script helps in visualizing backported and missed commits between two + different branches, tags or commit ranges. In the list of missed commits, + it will help you identify patches which are posted for reviews on gerrit server. + + USAGE: + $ ./extras/git-branch-diff.py --help + usage: git-branch-diff.py [-h] [-s SOURCE] -t TARGET [-a AUTHOR] [-p PATH] + [-o OPTIONS] + + git wrapper to diff local or remote branches/tags/commit-ranges + + optional arguments: + -h, --help show this help message and exit + -s SOURCE, --source SOURCE + source pattern, it could be a branch, tag or a commit + range + -t TARGET, --target TARGET + target pattern, it could be a branch, tag or a commit + range + -a AUTHOR, --author AUTHOR + default: git config name/email, to provide multiple + specify comma separated values + -p PATH, --path PATH show source and target diff w.r.t given path, to + provide multiple specify space in between them + -o OPTIONS, --options OPTIONS + add other git options such as --after=<>, --before=<> + etc. experts use; + + SAMPLE EXECUTIONS: + $ ./extras/git-branch-diff.py -t origin/release-3.8 + + $ ./extras/git-branch-diff.py -s local_branch -t origin/release-3.7 + + $ ./extras/git-branch-diff.py -s 4517bf8..e66add8 -t origin/release-3.7 + $ ./extras/git-branch-diff.py -s HEAD..c4efd39 -t origin/release-3.7 + + $ ./extras/git-branch-diff.py -t v3.7.11 --author="author@redhat.com" + $ ./extras/git-branch-diff.py -t v3.7.11 --author="authorX, authorY, authorZ" + + $ ./extras/git-branch-diff.py -t origin/release-3.8 --path="xlators/" + $ ./extras/git-branch-diff.py -t origin/release-3.8 --path="./xlators ./rpc" + + $ ./extras/git-branch-diff.py -t origin/release-3.6 --author="*" + $ ./extras/git-branch-diff.py -t origin/release-3.6 --author="All" + $ ./extras/git-branch-diff.py -t origin/release-3.6 --author="Null" + + $ ./extras/git-branch-diff.py -t v3.7.11 --options="--after=2015-03-01 \ + --before=2016-01-30" + + DECLARATION: + While backporting commit to another branch only subject of the patch may + remain unchanged, all others such as commit message, commit Id, change Id, + bug Id, may be changed. This script works by taking commit subject as the + key value for comparing two git branches, which can be local or remote. + + Note: This script may ignore commits which have altered their commit subjects + while backporting patches. Also this script doesn't have any intelligence to + detect squashed commits. + + AUTHOR: + Prasanna Kumar Kalever <prasanna.kalever@redhat.com> +""" + +from __future__ import print_function +import os +import sys +import argparse +import commands +import subprocess +import requests + +class GitBranchDiff: + def __init__ (self): + " color symbols" + self.tick = u'\033[1;32m[ \u2714 ]\033[0m' + self.cross = u'\033[1;31m[ \u2716 ]\033[0m' + self.green_set = u'\033[1;34m' + self.yello_set = u'\033[4;33m' + self.color_unset = '\033[0m' + + self.parse_cmd_args() + + " replace default values with actual values from command args" + self.g_author = self.argsdict['author'] + self.s_pattern = self.argsdict['source'] + self.t_pattern = self.argsdict['target'] + self.r_path = self.argsdict['path'] + self.options = ' '.join(self.argsdict['options']) + + self.gerrit_server = "http://review.gluster.org" + + def check_dir_exist (self, os_path): + " checks whether given path exist" + path_list = os_path.split() + for path in path_list: + if not os.path.exists(path): + raise argparse.ArgumentTypeError("'%s' path %s is not valid" + %(os_path, path)) + return os_path + + def check_pattern_exist (self): + " defend to check given branch[s] exit" + status_sbr, op = commands.getstatusoutput('git log ' + + self.s_pattern) + status_tbr, op = commands.getstatusoutput('git log ' + + self.t_pattern) + if status_sbr != 0: + print("Error: --source=" + self.s_pattern + " doesn't exit\n") + self.parser.print_help() + exit(status_sbr) + elif status_tbr != 0: + print("Error: --target=" + self.t_pattern + " doesn't exit\n") + self.parser.print_help() + exit(status_tbr) + + def check_author_exist (self): + " defend to check given author exist, format in case of multiple" + contrib_list = ['', '*', 'all', 'All', 'ALL', 'null', 'Null', 'NULL'] + if self.g_author in contrib_list: + self.g_author = "" + else: + ide_list = self.g_author.split(',') + for ide in ide_list: + cmd4 = 'git log ' + self.s_pattern + ' --author=' + ide + c_list = subprocess.check_output(cmd4, shell = True) + if len(c_list) is 0: + print("Error: --author=%s doesn't exit" %self.g_author) + print("see '%s --help'" %__file__) + exit(1) + if len(ide_list) > 1: + self.g_author = "\|".join(ide_list) + + def connected_to_gerrit (self): + "check if gerrit server is reachable" + try: + r = requests.get(self.gerrit_server, timeout=3) + return True + except requests.Timeout as err: + " request timed out" + print("Warning: failed to get list of open review commits on " \ + "gerrit.\n" \ + "hint: Request timed out! gerrit server could possibly " \ + "slow ...\n") + return False + except requests.RequestException as err: + " handle other errors" + print("Warning: failed to get list of open review commits on " \ + "gerrit\n" \ + "hint: check with internet connection ...\n") + return False + + def parse_cmd_args (self): + " command line parser" + author = subprocess.check_output('git config user.email', + shell = True).rstrip('\n') + source = "remotes/origin/master" + options = [' --pretty=format:"%h %s" '] + path = subprocess.check_output('git rev-parse --show-toplevel', + shell = True).rstrip('\n') + self.parser = argparse.ArgumentParser(description = 'git wrapper to ' + 'diff local or remote branches/' + 'tags/commit-ranges') + self.parser.add_argument('-s', + '--source', + help = 'source pattern, it could be a branch,' + ' tag or a commit range', + default = source, + dest = 'source') + self.parser.add_argument('-t', + '--target', + help = 'target pattern, it could be a branch,' + ' tag or a commit range', + required = True, + dest = 'target') + self.parser.add_argument('-a', + '--author', + help = 'default: git config name/email, ' + 'to provide multiple specify comma' + ' separated values', + default = author, + dest = 'author') + self.parser.add_argument('-p', + '--path', + type = self.check_dir_exist, + help = 'show source and target diff w.r.t ' + 'given path, to provide multiple ' + 'specify space in between them', + default = path, + dest = 'path') + self.parser.add_argument('-o', + '--options', + help = 'add other git options such as ' + '--after=<>, --before=<> etc. ' + 'experts use;', + default = options, + dest = 'options', + action='append') + self.argsdict = vars(self.parser.parse_args()) + + def print_output (self): + " display the result list" + print("\n------------------------------------------------------------\n") + print(self.tick + " Successfully Backported changes:") + print(' {' + 'from: ' + self.s_pattern + \ + ' to: '+ self.t_pattern + '}\n') + for key, value in self.s_dict.items(): + if value in self.t_dict.itervalues(): + print("[%s%s%s] %s" %(self.yello_set, + key, + self.color_unset, + value)) + print("\n------------------------------------------------------------\n") + print(self.cross + " Missing patches in " + self.t_pattern + ':\n') + if self.connected_to_gerrit(): + cmd3 = "git review -r origin -l" + review_list = subprocess.check_output(cmd3, shell = True).split('\n') + else: + review_list = [] + + for key, value in self.s_dict.items(): + if value not in self.t_dict.itervalues(): + if any(value in s for s in review_list): + print("[%s%s%s] %s %s(under review)%s" %(self.yello_set, + key, + self.color_unset, + value, + self.green_set, + self.color_unset)) + else: + print("[%s%s%s] %s" %(self.yello_set, + key, + self.color_unset, + value)) + print("\n------------------------------------------------------------\n") + + def main (self): + self.check_pattern_exist() + self.check_author_exist() + + " actual git commands" + cmd1 = 'git log' + self.options + ' ' + self.s_pattern + \ + ' --author=\'' + self.g_author + '\' ' + self.r_path + + " could be backported by anybody so --author doesn't apply here" + cmd2 = 'git log' + self.options + ' ' + self.t_pattern + \ + ' ' + self.r_path + + s_list = subprocess.check_output(cmd1, shell = True).split('\n') + t_list = subprocess.check_output(cmd2, shell = True) + + if len(t_list) is 0: + print("No commits in the target: %s" %self.t_pattern) + print("see '%s --help'" %__file__) + exit() + else: + t_list = t_list.split('\n') + + self.s_dict = dict() + self.t_dict = dict() + + for item in s_list: + self.s_dict.update(dict([item.split(' ', 1)])) + for item in t_list: + self.t_dict.update(dict([item.split(' ', 1)])) + + self.print_output() + + +if __name__ == '__main__': + run = GitBranchDiff() + run.main() diff --git a/extras/glusterd.vol.in b/extras/glusterd.vol.in index 690dbe71823..5d7bad0e4c8 100644 --- a/extras/glusterd.vol.in +++ b/extras/glusterd.vol.in @@ -1,10 +1,15 @@ volume management type mgmt/glusterd option working-directory @GLUSTERD_WORKDIR@ - option transport-type socket,rdma + option transport-type socket option transport.socket.keepalive-time 10 option transport.socket.keepalive-interval 2 option transport.socket.read-fail-log off - option ping-timeout 30 + option transport.socket.listen-port 24007 + option ping-timeout 0 + option event-threads 1 +# option lock-timer 180 +# option transport.address-family inet6 # option base-port 49152 + option max-port 60999 end-volume diff --git a/extras/glusterfs-georep-logrotate b/extras/glusterfs-georep-logrotate index 6fdb8c65aaf..3e7ecf373a1 100644 --- a/extras/glusterfs-georep-logrotate +++ b/extras/glusterfs-georep-logrotate @@ -1,6 +1,12 @@ /var/log/glusterfs/geo-replication/*/*.log { sharedscripts - rotate 52 + weekly + maxsize 10M + minsize 100k + + # 6 months of logs are good enough + rotate 26 + missingok compress delaycompress @@ -15,7 +21,13 @@ /var/log/glusterfs/geo-replication-slaves/*.log { sharedscripts - rotate 52 + weekly + maxsize 10M + minsize 100k + + # 6 months of logs are good enough + rotate 26 + missingok compress delaycompress @@ -30,7 +42,13 @@ /var/log/glusterfs/geo-replication-slaves/*/*.log { sharedscripts - rotate 52 + weekly + maxsize 10M + minsize 100k + + # 6 months of logs are good enough + rotate 26 + missingok compress delaycompress diff --git a/extras/glusterfs-georep-upgrade.py b/extras/glusterfs-georep-upgrade.py new file mode 100755 index 00000000000..634576058d6 --- /dev/null +++ b/extras/glusterfs-georep-upgrade.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 +""" + +Copyright (c) 2020 Red Hat, Inc. <http://www.redhat.com> +This file is part of GlusterFS. + +This file is licensed to you under your choice of the GNU Lesser +General Public License, version 3 or any later version (LGPLv3 or +later), or the GNU General Public License, version 2 (GPLv2), in all +cases as published by the Free Software Foundation. + +""" + +import argparse +import errno +import os, sys +import shutil +from datetime import datetime + +def find_htime_path(brick_path): + dirs = [] + htime_dir = os.path.join(brick_path, '.glusterfs/changelogs/htime') + for file in os.listdir(htime_dir): + if os.path.isfile(os.path.join(htime_dir,file)) and file.startswith("HTIME"): + dirs.append(os.path.join(htime_dir, file)) + else: + raise FileNotFoundError("%s unavailable" % (os.path.join(htime_dir, file))) + return dirs + +def modify_htime_file(brick_path): + htime_file_path_list = find_htime_path(brick_path) + + for htime_file_path in htime_file_path_list: + changelog_path = os.path.join(brick_path, '.glusterfs/changelogs') + temp_htime_path = os.path.join(changelog_path, 'htime/temp_htime_file') + with open(htime_file_path, 'r') as htime_file, open(temp_htime_path, 'w') as temp_htime_file: + #extract epoch times from htime file + paths = htime_file.read().split("\x00") + + for pth in paths: + epoch_no = pth.split(".")[-1] + changelog = os.path.basename(pth) + #convert epoch time to year, month and day + if epoch_no != '': + date=(datetime.fromtimestamp(float(int(epoch_no))).strftime("%Y/%m/%d")) + #update paths in temp htime file + temp_htime_file.write("%s/%s/%s\x00" % (changelog_path, date, changelog)) + #create directory in the format year/month/days + path = os.path.join(changelog_path, date) + + if changelog.startswith("CHANGELOG."): + try: + os.makedirs(path, mode = 0o600); + except OSError as exc: + if exc.errno == errno.EEXIST: + pass + else: + raise + + #copy existing changelogs to new directory structure, delete old changelog files + shutil.copyfile(pth, os.path.join(path, changelog)) + os.remove(pth) + + #rename temp_htime_file with htime file + os.rename(htime_file_path, os.path.join('%s.bak'%htime_file_path)) + os.rename(temp_htime_path, htime_file_path) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('brick_path', help="This upgrade script, which is to be run on\ + server side, takes brick path as the argument, \ + updates paths inside htime file and alters the directory structure \ + above the changelog files inorder to support new optimised format \ + of the directory structure as per \ + https://review.gluster.org/#/c/glusterfs/+/23733/") + args = parser.parse_args() + modify_htime_file(args.brick_path) diff --git a/extras/glusterfs-logrotate b/extras/glusterfs-logrotate index e3319afaa96..6ba6ef18e9f 100644 --- a/extras/glusterfs-logrotate +++ b/extras/glusterfs-logrotate @@ -2,7 +2,12 @@ /var/log/glusterfs/*.log { sharedscripts weekly - rotate 52 + maxsize 10M + minsize 100k + +# 6 months of logs are good enough + rotate 26 + missingok compress delaycompress @@ -17,7 +22,12 @@ /var/log/glusterfs/bricks/*.log { sharedscripts weekly - rotate 52 + maxsize 10M + minsize 100k + +# 6 months of logs are good enough + rotate 26 + missingok compress delaycompress @@ -26,3 +36,33 @@ /usr/bin/killall -HUP glusterfsd > /dev/null 2>&1 || true endscript } + +/var/log/glusterfs/samples/*.samp { + daily + rotate 3 + sharedscripts + missingok + compress + delaycompress +} + +# Rotate snapd log +/var/log/glusterfs/snaps/*/*.log { + sharedscripts + weekly + maxsize 10M + minsize 100k + + # 6 months of logs are good enough + rotate 26 + + missingok + compress + delaycompress + notifempty + postrotate + for pid in `ps -aef | grep glusterfs | egrep "snapd" | awk '{print $2}'`; do + /usr/bin/kill -HUP $pid > /dev/null 2>&1 || true + done + endscript +} diff --git a/extras/glusterfs-mode.el b/extras/glusterfs-mode.el index d4f6dc568b6..a9ed2335ab3 100644 --- a/extras/glusterfs-mode.el +++ b/extras/glusterfs-mode.el @@ -1,112 +1,113 @@ -;;; Copyright (C) 2007-2011 Gluster Inc. <http://www.gluster.com>
-;;;
-;;; This program is free software; you can redistribute it and/or modify
-;;; it under the terms of the GNU General Public License as published by
-;;; the Free Software Foundation; either version 2 of the License, or
-;;; (at your option) any later version.
-;;;
-;;; This program is distributed in the hope that it will be useful,
-;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
-;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-;;; GNU General Public License for more details.
-;;;
-;;; You should have received a copy of the GNU General Public License
-;;; along with this program; if not, write to the Free Software
-;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-;;;
-
-(defvar glusterfs-mode-hook nil)
-
-;; (defvar glusterfs-mode-map
-;; (let ((glusterfs-mode-map (make-keymap)))
-;; (define-key glusterfs-mode-map "\C-j" 'newline-and-indent)
-;; glusterfs-mode-map)
-;; "Keymap for WPDL major mode")
-
-(add-to-list 'auto-mode-alist '("\\.vol\\'" . glusterfs-mode))
-
-(defconst glusterfs-font-lock-keywords-1
- (list
- ; "cluster/{unify,afr,stripe}"
- ; "performance/{io-cache,io-threads,write-behind,read-ahead,stat-prefetch}"
- ; "protocol/{client/server}"
- ; "features/{trash,posix-locks,fixed-id,filter}"
- ; "stroage/posix"
- ; "encryption/rot-13"
- ; "debug/trace"
- '("\\<\\(cluster/\\(unify\\|afr\\|replicate\\|stripe\\|ha\\|dht\\|distribute\\)\\|\\performance/\\(io-\\(cache\\|threads\\)\\|write-behind\\|read-ahead\\|symlink-cache\\)\\|protocol/\\(server\\|client\\)\\|features/\\(trash\\|posix-locks\\|locks\\|path-converter\\|filter\\)\\|storage/\\(posix\\|bdb\\)\\|encryption/rot-13\\|debug/trace\\)\\>" . font-lock-keyword-face))
-"Additional Keywords to highlight in GlusterFS mode.")
-
-(defconst glusterfs-font-lock-keywords-2
- (append glusterfs-font-lock-keywords-1
- (list
- ; "replicate" "namespace" "scheduler" "remote-subvolume" "remote-host"
- ; "auth.addr" "block-size" "remote-port" "listen-port" "transport-type"
- ; "limits.min-free-disk" "directory"
- ; TODO: add all the keys here.
- '("\\<\\(inode-lru-limit\\|replicate\\|namespace\\|scheduler\\|username\\|password\\|allow\\|reject\\|block-size\\|listen-port\\|transport-type\\|transport-timeout\\|directory\\|page-size\\|page-count\\|aggregate-size\\|non-blocking-io\\|client-volume-filename\\|bind-address\\|self-heal\\|read-only-subvolumes\\|read-subvolume\\|thread-count\\|cache-size\\|window-size\\|force-revalidate-timeout\\|priority\\|include\\|exclude\\|remote-\\(host\\|subvolume\\|port\\)\\|auth.\\(addr\\|login\\)\\|limits.\\(min-disk-free\\|transaction-size\\|ib-verbs-\\(work-request-\\(send-\\|recv-\\(count\\|size\\)\\)\\|port\\|mtu\\|device-name\\)\\)\\)\ \\>" . font-lock-constant-face)))
- "option keys in GlusterFS mode.")
-
-(defconst glusterfs-font-lock-keywords-3
- (append glusterfs-font-lock-keywords-2
- (list
- ; "option" "volume" "end-volume" "subvolumes" "type"
- '("\\<\\(option\ \\|volume\ \\|subvolumes\ \\|type\ \\|end-volume\\)\\>" . font-lock-builtin-face)))
- ;'((regexp-opt (" option " "^volume " "^end-volume" "subvolumes " " type ") t) . font-lock-builtin-face))
- "Minimal highlighting expressions for GlusterFS mode.")
-
-
-(defvar glusterfs-font-lock-keywords glusterfs-font-lock-keywords-3
- "Default highlighting expressions for GlusterFS mode.")
-
-(defvar glusterfs-mode-syntax-table
- (let ((glusterfs-mode-syntax-table (make-syntax-table)))
- (modify-syntax-entry ?\# "<" glusterfs-mode-syntax-table)
- (modify-syntax-entry ?* ". 23" glusterfs-mode-syntax-table)
- (modify-syntax-entry ?\n ">#" glusterfs-mode-syntax-table)
- glusterfs-mode-syntax-table)
- "Syntax table for glusterfs-mode")
-
-;; TODO: add an indentation table
-
-(defun glusterfs-indent-line ()
- "Indent current line as GlusterFS code"
- (interactive)
- (beginning-of-line)
- (if (bobp)
- (indent-line-to 0) ; First line is always non-indented
- (let ((not-indented t) cur-indent)
- (if (looking-at "^[ \t]*volume\ ")
- (progn
- (save-excursion
- (forward-line -1)
- (setq not-indented nil)
- (setq cur-indent 0))))
- (if (looking-at "^[ \t]*end-volume")
- (progn
- (save-excursion
- (forward-line -1)
- (setq cur-indent 0))
- (if (< cur-indent 0) ; We can't indent past the left margin
- (setq cur-indent 0)))
- (save-excursion
- (while not-indented ; Iterate backwards until we find an indentation hint
- (progn
- (setq cur-indent 2) ; Do the actual indenting
- (setq not-indented nil)))))
- (if cur-indent
- (indent-line-to cur-indent)
- (indent-line-to 0)))))
-
-(defun glusterfs-mode ()
- (interactive)
- (kill-all-local-variables)
- ;; (use-local-map glusterfs-mode-map)
- (set-syntax-table glusterfs-mode-syntax-table)
- (set (make-local-variable 'indent-line-function) 'glusterfs-indent-line)
- (set (make-local-variable 'font-lock-defaults) '(glusterfs-font-lock-keywords))
- (setq major-mode 'glusterfs-mode)
- (setq mode-name "GlusterFS")
- (run-hooks 'glusterfs-mode-hook))
-
-(provide 'glusterfs-mode)
+;;; Copyright (C) 2007-2017 Red Hat, Inc. <http://www.redhat.com> +;;; Copyright (C) 2007-2011 Gluster Inc. <http://www.gluster.com> +;;; +;;; This program is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation; either version 2 +;;; of the License, or (at your option) any later version. +;;; +;;; This program is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with this program; if not, write to the Free Software +;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +;;; + +(defvar glusterfs-mode-hook nil) + +;; (defvar glusterfs-mode-map +;; (let ((glusterfs-mode-map (make-keymap))) +;; (define-key glusterfs-mode-map "\C-j" 'newline-and-indent) +;; glusterfs-mode-map) +;; "Keymap for WPDL major mode") + +(add-to-list 'auto-mode-alist '("\\.vol\\'" . glusterfs-mode)) + +(defconst glusterfs-font-lock-keywords-1 + (list + ; "cluster/{unify,afr,stripe}" + ; "performance/{io-cache,io-threads,write-behind,read-ahead,stat-prefetch}" + ; "protocol/{client/server}" + ; "features/{trash,posix-locks,fixed-id,filter}" + ; "storage/posix" + ; "encryption/rot-13" + ; "debug/trace" + '("\\<\\(cluster/\\(unify\\|afr\\|replicate\\|stripe\\|ha\\|dht\\|distribute\\)\\|\\performance/\\(io-\\(cache\\|threads\\)\\|write-behind\\|read-ahead\\|symlink-cache\\)\\|protocol/\\(server\\|client\\)\\|features/\\(trash\\|posix-locks\\|locks\\|path-converter\\|filter\\)\\|storage/\\(posix\\|bdb\\)\\|encryption/rot-13\\|debug/trace\\)\\>" . font-lock-keyword-face)) +"Additional Keywords to highlight in GlusterFS mode.") + +(defconst glusterfs-font-lock-keywords-2 + (append glusterfs-font-lock-keywords-1 + (list + ; "replicate" "namespace" "scheduler" "remote-subvolume" "remote-host" + ; "auth.addr" "block-size" "remote-port" "listen-port" "transport-type" + ; "limits.min-free-disk" "directory" + ; TODO: add all the keys here. + '("\\<\\(inode-lru-limit\\|replicate\\|namespace\\|scheduler\\|username\\|password\\|allow\\|reject\\|block-size\\|listen-port\\|transport-type\\|transport-timeout\\|directory\\|page-size\\|page-count\\|aggregate-size\\|non-blocking-io\\|client-volume-filename\\|bind-address\\|self-heal\\|read-only-subvolumes\\|read-subvolume\\|thread-count\\|cache-size\\|window-size\\|force-revalidate-timeout\\|priority\\|include\\|exclude\\|remote-\\(host\\|subvolume\\|port\\)\\|auth.\\(addr\\|login\\)\\|limits.\\(min-disk-free\\|transaction-size\\|ib-verbs-\\(work-request-\\(send-\\|recv-\\(count\\|size\\)\\)\\|port\\|mtu\\|device-name\\)\\)\\)\ \\>" . font-lock-constant-face))) + "option keys in GlusterFS mode.") + +(defconst glusterfs-font-lock-keywords-3 + (append glusterfs-font-lock-keywords-2 + (list + ; "option" "volume" "end-volume" "subvolumes" "type" + '("\\<\\(option\ \\|volume\ \\|subvolumes\ \\|type\ \\|end-volume\\)\\>" . font-lock-builtin-face))) + ;'((regexp-opt (" option " "^volume " "^end-volume" "subvolumes " " type ") t) . font-lock-builtin-face)) + "Minimal highlighting expressions for GlusterFS mode.") + + +(defvar glusterfs-font-lock-keywords glusterfs-font-lock-keywords-3 + "Default highlighting expressions for GlusterFS mode.") + +(defvar glusterfs-mode-syntax-table + (let ((glusterfs-mode-syntax-table (make-syntax-table))) + (modify-syntax-entry ?\# "<" glusterfs-mode-syntax-table) + (modify-syntax-entry ?* ". 23" glusterfs-mode-syntax-table) + (modify-syntax-entry ?\n ">#" glusterfs-mode-syntax-table) + glusterfs-mode-syntax-table) + "Syntax table for glusterfs-mode") + +;; TODO: add an indentation table + +(defun glusterfs-indent-line () + "Indent current line as GlusterFS code" + (interactive) + (beginning-of-line) + (if (bobp) + (indent-line-to 0) ; First line is always non-indented + (let ((not-indented t) cur-indent) + (if (looking-at "^[ \t]*volume\ ") + (progn + (save-excursion + (forward-line -1) + (setq not-indented nil) + (setq cur-indent 0)))) + (if (looking-at "^[ \t]*end-volume") + (progn + (save-excursion + (forward-line -1) + (setq cur-indent 0)) + (if (< cur-indent 0) ; We can't indent past the left margin + (setq cur-indent 0))) + (save-excursion + (while not-indented ; Iterate backwards until we find an indentation hint + (progn + (setq cur-indent 2) ; Do the actual indenting + (setq not-indented nil))))) + (if cur-indent + (indent-line-to cur-indent) + (indent-line-to 0))))) + +(defun glusterfs-mode () + (interactive) + (kill-all-local-variables) + ;; (use-local-map glusterfs-mode-map) + (set-syntax-table glusterfs-mode-syntax-table) + (set (make-local-variable 'indent-line-function) 'glusterfs-indent-line) + (set (make-local-variable 'font-lock-defaults) '(glusterfs-font-lock-keywords)) + (setq major-mode 'glusterfs-mode) + (setq mode-name "GlusterFS") + (run-hooks 'glusterfs-mode-hook)) + +(provide 'glusterfs-mode) diff --git a/extras/gnfs-loganalyse.py b/extras/gnfs-loganalyse.py index 71e79b6be4e..6341d007188 100644..100755 --- a/extras/gnfs-loganalyse.py +++ b/extras/gnfs-loganalyse.py @@ -10,6 +10,7 @@ """ +from __future__ import print_function import os import string import sys @@ -72,7 +73,7 @@ class NFSRequest: self.replygfid = tokens [gfididx + 1].strip(",") def dump (self): - print "ReqLine: " + str(self.calllinecount) + " TimeStamp: " + self.timestamp + ", XID: " + self.xid + " " + self.op + " ARGS: " + self.opdata + " RepLine: " + str(self.replylinecount) + " " + self.replydata + print("ReqLine: " + str(self.calllinecount) + " TimeStamp: " + self.timestamp + ", XID: " + self.xid + " " + self.op + " ARGS: " + self.opdata + " RepLine: " + str(self.replylinecount) + " " + self.replydata) class NFSLogAnalyzer: @@ -149,7 +150,7 @@ class NFSLogAnalyzer: return rcount = len (self.xid_request_map.keys ()) orphancount = len (self.orphan_replies.keys ()) - print "Requests: " + str(rcount) + ", Orphans: " + str(orphancount) + print("Requests: " + str(rcount) + ", Orphans: " + str(orphancount)) def dump (self): self.getStats () diff --git a/extras/group-db-workload b/extras/group-db-workload new file mode 100644 index 00000000000..9334d6fb942 --- /dev/null +++ b/extras/group-db-workload @@ -0,0 +1,12 @@ +performance.open-behind=on +performance.write-behind=off +performance.stat-prefetch=off +performance.quick-read=off +performance.strict-o-direct=on +performance.read-ahead=off +performance.io-cache=off +performance.readdir-ahead=off +performance.client-io-threads=on +server.event-threads=4 +client.event-threads=4 +performance.read-after-open=yes diff --git a/extras/group-distributed-virt b/extras/group-distributed-virt new file mode 100644 index 00000000000..a960b76c694 --- /dev/null +++ b/extras/group-distributed-virt @@ -0,0 +1,10 @@ +performance.quick-read=off +performance.read-ahead=off +performance.io-cache=off +performance.low-prio-threads=32 +network.remote-dio=enable +features.shard=on +user.cifs=off +client.event-threads=4 +server.event-threads=4 +performance.client-io-threads=on diff --git a/extras/group-gluster-block b/extras/group-gluster-block new file mode 100644 index 00000000000..1e398019e6b --- /dev/null +++ b/extras/group-gluster-block @@ -0,0 +1,27 @@ +performance.quick-read=off +performance.read-ahead=off +performance.io-cache=off +performance.stat-prefetch=off +performance.open-behind=off +performance.readdir-ahead=off +performance.strict-o-direct=on +performance.client-io-threads=on +performance.io-thread-count=32 +performance.high-prio-threads=32 +performance.normal-prio-threads=32 +performance.low-prio-threads=32 +performance.least-prio-threads=4 +client.event-threads=8 +server.event-threads=8 +network.remote-dio=disable +cluster.eager-lock=enable +cluster.quorum-type=auto +cluster.data-self-heal-algorithm=full +cluster.locking-scheme=granular +cluster.shd-max-threads=8 +cluster.shd-wait-qlength=10000 +features.shard=on +features.shard-block-size=64MB +user.cifs=off +server.allow-insecure=on +cluster.choose-local=off diff --git a/extras/group-metadata-cache b/extras/group-metadata-cache new file mode 100644 index 00000000000..b890b288fc7 --- /dev/null +++ b/extras/group-metadata-cache @@ -0,0 +1,6 @@ +features.cache-invalidation=on +features.cache-invalidation-timeout=600 +performance.stat-prefetch=on +performance.cache-invalidation=on +performance.md-cache-timeout=600 +network.inode-lru-limit=200000 diff --git a/extras/group-nl-cache b/extras/group-nl-cache new file mode 100644 index 00000000000..897807e8933 --- /dev/null +++ b/extras/group-nl-cache @@ -0,0 +1,5 @@ +features.cache-invalidation=on +features.cache-invalidation-timeout=600 +performance.nl-cache=on +performance.nl-cache-timeout=600 +network.inode-lru-limit=200000 diff --git a/extras/group-samba b/extras/group-samba new file mode 100644 index 00000000000..eeee6e06031 --- /dev/null +++ b/extras/group-samba @@ -0,0 +1,11 @@ +features.cache-invalidation=on +features.cache-invalidation-timeout=600 +performance.cache-samba-metadata=on +performance.stat-prefetch=on +performance.cache-invalidation=on +performance.md-cache-timeout=600 +network.inode-lru-limit=200000 +performance.nl-cache=on +performance.nl-cache-timeout=600 +performance.readdir-ahead=on +performance.parallel-readdir=on diff --git a/extras/group-virt.example b/extras/group-virt.example index 0abe9f400a6..cc37c98a25c 100644 --- a/extras/group-virt.example +++ b/extras/group-virt.example @@ -1,8 +1,24 @@ -quick-read=off -read-ahead=off -io-cache=off -stat-prefetch=off -eager-lock=enable -remote-dio=enable -quorum-type=auto -server-quorum-type=server +performance.quick-read=off +performance.read-ahead=off +performance.io-cache=off +performance.low-prio-threads=32 +network.remote-dio=disable +performance.strict-o-direct=on +cluster.eager-lock=enable +cluster.quorum-type=auto +cluster.server-quorum-type=server +cluster.data-self-heal-algorithm=full +cluster.locking-scheme=granular +cluster.shd-max-threads=8 +cluster.shd-wait-qlength=10000 +features.shard=on +user.cifs=off +cluster.choose-local=off +client.event-threads=4 +server.event-threads=4 +performance.client-io-threads=on +network.ping-timeout=20 +server.tcp-user-timeout=20 +server.keepalive-time=10 +server.keepalive-interval=2 +server.keepalive-count=5 diff --git a/extras/hook-scripts/Makefile.am b/extras/hook-scripts/Makefile.am index 4a1adb3d5db..26059d7dbb9 100644 --- a/extras/hook-scripts/Makefile.am +++ b/extras/hook-scripts/Makefile.am @@ -1,2 +1,7 @@ EXTRA_DIST = S40ufo-stop.py S56glusterd-geo-rep-create-post.sh -SUBDIRS = add-brick set start stop reset +SUBDIRS = add-brick create delete set start stop reset + +scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/gsync-create/post/ +if USE_GEOREP +scripts_SCRIPTS = S56glusterd-geo-rep-create-post.sh +endif diff --git a/extras/hook-scripts/S40ufo-stop.py b/extras/hook-scripts/S40ufo-stop.py index 107f1968355..2c79eb1d54a 100755 --- a/extras/hook-scripts/S40ufo-stop.py +++ b/extras/hook-scripts/S40ufo-stop.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 import os from optparse import OptionParser diff --git a/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh index 835daf819c9..7d6052315bb 100755 --- a/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh +++ b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh @@ -1,9 +1,19 @@ #!/bin/bash +#key_val_pair is the arguments passed to the script in the form of +#key value pair + key_val_pair1=`echo $2 | cut -d ',' -f 1` key_val_pair2=`echo $2 | cut -d ',' -f 2` key_val_pair3=`echo $2 | cut -d ',' -f 3` key_val_pair4=`echo $2 | cut -d ',' -f 4` +key_val_pair5=`echo $2 | cut -d ',' -f 5` +key_val_pair6=`echo $2 | cut -d ',' -f 6` + +mastervol=`echo $1 | cut -d '=' -f 2` +if [ "$mastervol" == "" ]; then + exit; +fi key=`echo $key_val_pair1 | cut -d '=' -f 1` val=`echo $key_val_pair1 | cut -d '=' -f 2` @@ -22,7 +32,10 @@ fi if [ "$val" == "" ]; then exit; fi + pub_file=`echo $val` +pub_file_bname="$(basename $pub_file)" +pub_file_dname="$(dirname $pub_file)" pub_file_tmp=`echo $val`_tmp key=`echo $key_val_pair3 | cut -d '=' -f 1` @@ -45,17 +58,47 @@ if [ "$val" == "" ]; then fi slave_ip=`echo $val` +key=`echo $key_val_pair5 | cut -d '=' -f 1` +val=`echo $key_val_pair5 | cut -d '=' -f 2` +if [ "$key" != "slave_vol" ]; then + exit; +fi +if [ "$val" == "" ]; then + exit; +fi +slavevol=`echo $val` + +key=`echo $key_val_pair6 | cut -d '=' -f 1` +val=`echo $key_val_pair6 | cut -d '=' -f 2` +if [ "$key" != "ssh_port" ]; then + exit; +fi +if [ "$val" == "" ]; then + exit; +fi +SSH_PORT=`echo $val` +SSH_OPT="-oPasswordAuthentication=no -oStrictHostKeyChecking=no" + if [ -f $pub_file ]; then # For a non-root user copy the pub file to the user's home directory # For a root user copy the pub files to priv_dir->geo-rep. if [ "$slave_user" != "root" ]; then - slave_user_home_dir=`ssh $slave_user@$slave_ip "getent passwd $slave_user | cut -d ':' -f 6"` - scp $pub_file $slave_user@$slave_ip:$slave_user_home_dir/common_secret.pem.pub_tmp - ssh $slave_user@$slave_ip "mv $slave_user_home_dir/common_secret.pem.pub_tmp $slave_user_home_dir/common_secret.pem.pub" + slave_user_home_dir=`ssh -p ${SSH_PORT} ${SSH_OPT} $slave_user@$slave_ip "getent passwd $slave_user | cut -d ':' -f 6"` + scp -P ${SSH_PORT} ${SSH_OPT} $pub_file $slave_user@$slave_ip:$slave_user_home_dir/common_secret.pem.pub_tmp + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_user@$slave_ip "mv $slave_user_home_dir/common_secret.pem.pub_tmp $slave_user_home_dir/${mastervol}_${slavevol}_common_secret.pem.pub" else - scp $pub_file $slave_ip:$pub_file_tmp - ssh $slave_ip "mv $pub_file_tmp $pub_file" - ssh $slave_ip "gluster system:: copy file /geo-replication/common_secret.pem.pub > /dev/null" - ssh $slave_ip "gluster system:: execute add_secret_pub > /dev/null" + if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then + scp -P ${SSH_PORT} ${SSH_OPT} $pub_file $slave_ip:$pub_file_tmp + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}" + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster vol set ${slavevol} features.read-only on" + else + scp -P ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $pub_file $slave_ip:$pub_file_tmp + ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}" + ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster vol set ${slavevol} features.read-only on" + fi fi fi diff --git a/extras/hook-scripts/add-brick/post/Makefile.am b/extras/hook-scripts/add-brick/post/Makefile.am index 12f510291a9..9b236df096d 100644 --- a/extras/hook-scripts/add-brick/post/Makefile.am +++ b/extras/hook-scripts/add-brick/post/Makefile.am @@ -1 +1,6 @@ -EXTRA_DIST = disabled-quota-root-xattr-heal.sh +EXTRA_DIST = disabled-quota-root-xattr-heal.sh S10selinux-label-brick.sh S13create-subdir-mounts.sh + +hookdir = $(GLUSTERD_WORKDIR)/hooks/1/add-brick/post/ +if WITH_SERVER +hook_SCRIPTS = disabled-quota-root-xattr-heal.sh S10selinux-label-brick.sh S13create-subdir-mounts.sh +endif diff --git a/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh b/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh new file mode 100755 index 00000000000..4a17c993a77 --- /dev/null +++ b/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# +# Install to hooks/<HOOKS_VER>/add-brick/post +# +# Add an SELinux file context for each brick using the glusterd_brick_t type. +# This ensures that the brick is relabeled correctly on an SELinux restart or +# restore. Subsequently, run a restore on the brick path to set the selinux +# labels. +# +### + +PROGNAME="Sselinux" +OPTSPEC="volname:,version:,gd-workdir:,volume-op:" +VOL= + +parse_args () { + ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@") + eval set -- "${ARGS}" + + while true; do + case ${1} in + --volname) + shift + VOL=${1} + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --version) + shift + ;; + --volume-op) + shift + ;; + *) + shift + break + ;; + esac + shift + done +} + +set_brick_labels() +{ + local volname="${1}" + local fctx + local list=() + + fctx="$(semanage fcontext --list -C)" + + # wait for new brick path to be updated under + # ${GLUSTERD_WORKDIR}/vols/${volname}/bricks/ + sleep 5 + + # grab the path for each local brick + brickpath="${GLUSTERD_WORKDIR}/vols/${volname}/bricks/" + brickdirs=$( + find "${brickpath}" -type f -exec grep '^path=' {} \; | \ + cut -d= -f 2 | \ + sort -u + ) + + # create a list of bricks for which custom SELinux + # label doesn't exist + for b in ${brickdirs}; do + pattern="${b}(/.*)?" + echo "${fctx}" | grep "^${pattern}\s" >/dev/null + if [[ $? -ne 0 ]]; then + list+=("${pattern}") + fi + done + + # Add a file context for each brick path in the list and associate with the + # glusterd_brick_t SELinux type. + for p in ${list[@]} + do + semanage fcontext --add -t glusterd_brick_t -r s0 "${p}" + done + + # Set the labels for which SELinux label was added above + for b in ${brickdirs} + do + echo "${list[@]}" | grep "${b}" >/dev/null + if [[ $? -eq 0 ]]; then + restorecon -R "${b}" + fi + done +} + +SELINUX_STATE=$(which getenforce && getenforce) +[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0 + +parse_args "$@" +[ -z "${VOL}" ] && exit 1 + +set_brick_labels "${VOL}" + +exit 0 diff --git a/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh b/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh new file mode 100755 index 00000000000..1a6923ee7aa --- /dev/null +++ b/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +##--------------------------------------------------------------------------- +## This script runs the self-heal of the directories which are expected to +## be present as they are mounted as subdirectory mounts. +##--------------------------------------------------------------------------- + +MOUNT_DIR=`mktemp -d -t ${0##*/}.XXXXXX`; +OPTSPEC="volname:,version:,gd-workdir:,volume-op:" +PROGNAME="add-brick-create-subdir" +VOL_NAME=test +GLUSTERD_WORKDIR="/var/lib/glusterd" + +cleanup_mountpoint () +{ + umount -f $MOUNT_DIR; + if [ 0 -ne $? ] + then + return $? + fi + + rmdir $MOUNT_DIR; + if [ 0 -ne $? ] + then + return $? + fi +} + +##------------------------------------------ +## Parse the arguments +##------------------------------------------ +ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) +eval set -- "$ARGS" + +while true; +do + case $1 in + --volname) + shift + VOL_NAME=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --version) + shift + ;; + --volume-op) + shift + ;; + *) + shift + break + ;; + esac + shift +done + +## See if we have any subdirs to be healed before going further +subdirs=$(grep 'auth.allow' ${GLUSTERD_WORKDIR}/vols/${VOL_NAME}/info | cut -f2 -d'=' | tr ',' '\n' | cut -f1 -d'('); + +if [ -z ${subdirs} ]; then + rmdir $MOUNT_DIR; + exit 0; +fi + +##---------------------------------------- +## Mount the volume in temp directory. +## ----------------------------------- +glusterfs -s localhost --volfile-id=$VOL_NAME --client-pid=-50 $MOUNT_DIR; +if [ 0 -ne $? ] +then + exit $?; +fi + +## ----------------------------------- +# Do the 'stat' on all the directory for now. Ideal fix is to look at subdir +# list from 'auth.allow' option and only stat them. +for subdir in ${subdirs} +do + stat ${MOUNT_DIR}/${subdir} > /dev/null; +done + +## Clean up and exit +cleanup_mountpoint; diff --git a/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh b/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh index d18367fc9df..ca17a903549 100755 --- a/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh +++ b/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh @@ -11,131 +11,135 @@ ## 4. Disable itself ##--------------------------------------------------------------------------- -QUOTA_CONFIG_XATTR="trusted.glusterfs.quota.limit-set"; -MOUNT_DIR=`mktemp --directory --tmpdir`; +QUOTA_LIMIT_XATTR="trusted.glusterfs.quota.limit-set" +QUOTA_OBJECT_LIMIT_XATTR="trusted.glusterfs.quota.limit-objects" +MOUNT_DIR=$(mktemp -d -t "${0##*/}.XXXXXX"); OPTSPEC="volname:,version:,gd-workdir:,volume-op:" PROGNAME="Quota-xattr-heal-add-brick" VOL_NAME= VERSION= VOLUME_OP= GLUSTERD_WORKDIR= -ENABLED_NAME="S28Quota-root-xattr-heal.sh" +ENABLED_NAME_PREFIX="S28" +ENABLED_NAME="Quota-root-xattr-heal.sh" +THIS_SCRIPT=$(echo "${0}" | awk -F'/' '{print $NF}') cleanup_mountpoint () { - umount -f $MOUNT_DIR; - if [ 0 -ne $? ] - then - return $? - fi - - rmdir $MOUNT_DIR; - if [ 0 -ne $? ] - then - return $? - fi + + if umount -f "${MOUNT_DIR}"; then + return $? + fi + + if rmdir "${MOUNT_DIR}"; then + return $? + fi +} + +disable_and_exit () +{ + if [ -e "${ENABLED_STATE}" ] + then + unlink "${ENABLED_STATE}"; + exit $? + fi + + exit 0 +} + +get_and_set_xattr () +{ + XATTR=$1 + + VALUE=$(getfattr -n "${XATTR}" -e hex --absolute-names "${MOUNT_DIR}" 2>&1) + RET=$? + if [ 0 -eq ${RET} ]; then + VALUE=$(echo "${VALUE}" | grep "${XATTR}" | awk -F'=' '{print $NF}') + setfattr -n "${XATTR}" -v "${VALUE}" "${MOUNT_DIR}"; + RET=$? + else + if echo "${VALUE}" | grep -iq "No such attribute" ; then + RET=0 + fi + fi + + return ${RET}; } ##------------------------------------------ ## Parse the arguments ##------------------------------------------ -ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) +ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@") eval set -- "$ARGS" while true; do - case $1 in - --volname) - shift - VOL_NAME=$1 - ;; - --version) - shift - VERSION=$1 - ;; - --gd-workdir) - shift - GLUSTERD_WORKDIR=$1 - ;; - --volume-op) - shift - VOLUME_OP=$1 - ;; - *) - shift - break - ;; - esac - shift + case $1 in + --volname) + shift + VOL_NAME=$1 + ;; + --version) + shift + VERSION=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --volume-op) + shift + VOLUME_OP=$1 + ;; + *) + shift + break + ;; + esac + shift done ##---------------------------------------- -ENABLED_STATE="$GLUSTERD_WORKDIR/hooks/$VERSION/$VOLUME_OP/post/$ENABLED_NAME" +# Avoid long lines +ENABLED_STATE_1="${GLUSTERD_WORKDIR}/hooks/${VERSION}/${VOLUME_OP}/" +ENABLED_STATE_2="post/${ENABLED_NAME_PREFIX}${VOL_NAME}-${ENABLED_NAME}" +ENABLED_STATE="${ENABLED_STATE_1}${ENABLED_STATE_2}" +if [ "${THIS_SCRIPT}" != *"${VOL_NAME}"* ]; then + exit 0 +fi -FLAG=`gluster volume quota $VOL_NAME list / 2>&1 | grep \ - '\(No quota configured on volume\)\|\(Limit not set\)'`; -if ! [ -z $FLAG ] +## Is quota enabled? +FLAG=$(grep "^features.quota=" "${GLUSTERD_WORKDIR}/vols/${VOL_NAME}/info" \ +| awk -F'=' '{print $NF}'); +if [ "${FLAG}" != "on" ] then - ls $ENABLED_STATE; - RET=$? - if [ 0 -eq $RET ] - then - unlink $ENABLED_STATE; - exit $? - fi - - exit $RET; + disable_and_exit fi ## ----------------------------------- ## Mount the volume in temp directory. ## ----------------------------------- -glusterfs -s localhost --volfile-id=$VOL_NAME --client-pid=-42 $MOUNT_DIR; -if [ 0 -ne $? ] -then - exit $?; -fi -## ----------------------------------- +# Avoid long lines +CMD_1="glusterfs -s localhost" +CMD_2="--volfile-id=${VOL_NAME} client-pid=-42 ${MOUNT_DIR}" +CMD="${CMD_1}${CMD_2}" -## ------------------ -## Getfattr the value -## ------------------ -VALUE=`getfattr -n "$QUOTA_CONFIG_XATTR" -e hex --absolute-names $MOUNT_DIR \ - 2>&1 | grep $QUOTA_CONFIG_XATTR | awk -F'=' '{print $2}'` -RET=$? -if [ 0 -ne $RET ] +if ${CMD} then - ## Clean up and exit - cleanup_mountpoint; - - exit $RET; + exit $?; fi -## ------------------ - -## --------- -## Set xattr -## --------- -setfattr -n "$QUOTA_CONFIG_XATTR" -v $VALUE $MOUNT_DIR; -RET=$? -if [ 0 -ne $RET ] -then - ## Clean up and exit - cleanup_mountpoint; +## ----------------------------------- - exit $RET; -fi -## --------- +RET1=$(get_and_set_xattr "${QUOTA_LIMIT_XATTR}") +RET2=$(get_and_set_xattr "${QUOTA_OBJECT_LIMIT_XATTR}") +## Clean up and exit cleanup_mountpoint; -## Disable -ls $ENABLED_STATE; -RET=$? -if [ 0 -eq $RET ] -then - unlink $ENABLED_STATE; - exit $? +if [ "${RET1}" -ne 0 ] || [ "${RET2}" -ne 0 ]; then + exit 1 fi -exit $? + +disable_and_exit; diff --git a/extras/hook-scripts/add-brick/pre/Makefile.am b/extras/hook-scripts/add-brick/pre/Makefile.am index 4d22d3c489b..3288581aa57 100644 --- a/extras/hook-scripts/add-brick/pre/Makefile.am +++ b/extras/hook-scripts/add-brick/pre/Makefile.am @@ -1 +1,6 @@ EXTRA_DIST = S28Quota-enable-root-xattr-heal.sh + +hookdir = $(GLUSTERD_WORKDIR)/hooks/1/add-brick/pre/ +if WITH_SERVER +hook_SCRIPTS = S28Quota-enable-root-xattr-heal.sh +endif diff --git a/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh b/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh index 348f34ec3db..27e85231f45 100755 --- a/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh +++ b/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh @@ -26,10 +26,11 @@ VOL_NAME= GLUSTERD_WORKDIR= VOLUME_OP= VERSION= -ENABLED_NAME="S28Quota-root-xattr-heal.sh" +ENABLED_NAME_PREFIX="S28" +ENABLED_NAME="Quota-root-xattr-heal.sh" DISABLED_NAME="disabled-quota-root-xattr-heal.sh" -enable () +activate () { ln -sf $DISABLED_STATE $1; } @@ -37,7 +38,7 @@ enable () ##------------------------------------------ ## Parse the arguments ##------------------------------------------ -ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) +ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") eval set -- "$ARGS" while true; @@ -69,8 +70,8 @@ done ##---------------------------------------- DISABLED_STATE="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/$DISABLED_NAME" -ENABLED_STATE_START="$GLUSTERD_WORKDIR/hooks/$VERSION/start/post/$ENABLED_NAME" -ENABLED_STATE_ADD_BRICK="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/$ENABLED_NAME"; +ENABLED_STATE_START="$GLUSTERD_WORKDIR/hooks/$VERSION/start/post/""$ENABLED_NAME_PREFIX$VOL_NAME""-""$ENABLED_NAME" +ENABLED_STATE_ADD_BRICK="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/""$ENABLED_NAME_PREFIX""$VOL_NAME""-""$ENABLED_NAME"; ## Why to proceed if the required script itself is not present? ls $DISABLED_STATE; @@ -92,9 +93,9 @@ FLAG=`cat $GLUSTERD_WORKDIR/vols/$VOL_NAME/info | grep "^status=" \ | awk -F'=' '{print $NF}'`; if [ "$FLAG" != "1" ] then - enable $ENABLED_STATE_START; + activate $ENABLED_STATE_START; exit $? fi -enable $ENABLED_STATE_ADD_BRICK; +activate $ENABLED_STATE_ADD_BRICK; exit $? diff --git a/extras/hook-scripts/create/Makefile.am b/extras/hook-scripts/create/Makefile.am new file mode 100644 index 00000000000..b083a9145d6 --- /dev/null +++ b/extras/hook-scripts/create/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = post diff --git a/extras/hook-scripts/create/post/Makefile.am b/extras/hook-scripts/create/post/Makefile.am new file mode 100644 index 00000000000..fd1892e9589 --- /dev/null +++ b/extras/hook-scripts/create/post/Makefile.am @@ -0,0 +1,8 @@ +EXTRA_DIST = S10selinux-label-brick.sh + +scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/create/post/ +if WITH_SERVER +if USE_SELINUX +scripts_SCRIPTS = S10selinux-label-brick.sh +endif +endif diff --git a/extras/hook-scripts/create/post/S10selinux-label-brick.sh b/extras/hook-scripts/create/post/S10selinux-label-brick.sh new file mode 100755 index 00000000000..f9b4b1a57e3 --- /dev/null +++ b/extras/hook-scripts/create/post/S10selinux-label-brick.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Install to hooks/<HOOKS_VER>/create/post +# +# Add an SELinux file context for each brick using the glusterd_brick_t type. +# This ensures that the brick is relabeled correctly on an SELinux restart or +# restore. Subsequently, run a restore on the brick path to set the selinux +# labels. +# +### + +PROGNAME="Sselinux" +OPTSPEC="volname:" +VOL= + +parse_args () { + ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@") + eval set -- "${ARGS}" + + while true; do + case ${1} in + --volname) + shift + VOL=${1} + ;; + *) + shift + break + ;; + esac + shift + done +} + +set_brick_labels() +{ + volname="${1}" + + # grab the path for each local brick + brickpath="/var/lib/glusterd/vols/${volname}/bricks/" + brickdirs=$( + find "${brickpath}" -type f -exec grep '^path=' {} \; | \ + cut -d= -f 2 | \ + sort -u + ) + + for b in ${brickdirs}; do + # Add a file context for each brick path and associate with the + # glusterd_brick_t SELinux type. + pattern="${b}(/.*)?" + semanage fcontext --add -t glusterd_brick_t -r s0 "${pattern}" + # Set the labels on the new brick path. + restorecon -R "${b}" + done +} + +SELINUX_STATE=$(which getenforce && getenforce) +[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0 + +parse_args "$@" +[ -z "${VOL}" ] && exit 1 + +set_brick_labels "${VOL}" + +exit 0 diff --git a/extras/hook-scripts/delete/Makefile.am b/extras/hook-scripts/delete/Makefile.am new file mode 100644 index 00000000000..c98a05d9205 --- /dev/null +++ b/extras/hook-scripts/delete/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = pre diff --git a/extras/hook-scripts/delete/pre/Makefile.am b/extras/hook-scripts/delete/pre/Makefile.am new file mode 100644 index 00000000000..4fbfbe7311f --- /dev/null +++ b/extras/hook-scripts/delete/pre/Makefile.am @@ -0,0 +1,8 @@ +EXTRA_DIST = S10selinux-del-fcontext.sh + +scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/delete/pre/ +if WITH_SERVER +if USE_SELINUX +scripts_SCRIPTS = S10selinux-del-fcontext.sh +endif +endif diff --git a/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh b/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh new file mode 100755 index 00000000000..056b52afe76 --- /dev/null +++ b/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# +# Install to hooks/<HOOKS_VER>/delete/pre +# +# Delete the file context associated with the brick path on volume deletion. The +# associated file context was added during volume creation. +# +# We do not explicitly relabel the brick, as this could be time consuming and +# unnecessary. +# +### + +PROGNAME="Sselinux" +OPTSPEC="volname:" +VOL= + +function parse_args () { + ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@") + eval set -- "${ARGS}" + + while true; do + case ${1} in + --volname) + shift + VOL=${1} + ;; + *) + shift + break + ;; + esac + shift + done +} + +function delete_brick_fcontext() +{ + local volname=$1 + local fctx + local list=() + + fctx="$(semanage fcontext --list -C)" + # grab the path for each local brick + brickpath="/var/lib/glusterd/vols/${volname}/bricks/" + brickdirs=$(find "${brickpath}" -type f -exec grep '^path=' {} \; | \ + cut -d= -f 2 | sort -u) + for b in ${brickdirs} + do + pattern="${b}(/.*)?" + echo "${fctx}" | grep "^${pattern}\s" >/dev/null + if [[ $? -eq 0 ]]; then + list+=("${pattern}") + fi + done + if [[ ${#list[@]} -gt 0 ]]; then + printf 'fcontext --delete %s\n' "${list[@]}" | semanage -i - + fi + for b in ${brickdirs} + do + restorecon -R "${b}" + done +} + +SELINUX_STATE=$(which getenforce && getenforce) +[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0 + +parse_args "$@" +[ -z "${VOL}" ] && exit 1 + +delete_brick_fcontext "${VOL}" + +# failure to delete the fcontext is not fatal +exit 0 diff --git a/extras/hook-scripts/reset/post/Makefile.am b/extras/hook-scripts/reset/post/Makefile.am index fcdd8ab55ba..1b336ac1a85 100644 --- a/extras/hook-scripts/reset/post/Makefile.am +++ b/extras/hook-scripts/reset/post/Makefile.am @@ -1 +1 @@ -EXTRA_DIST = S31ganesha-reset.sh +EXTRA_DIST = diff --git a/extras/hook-scripts/reset/post/S31ganesha-reset.sh b/extras/hook-scripts/reset/post/S31ganesha-reset.sh deleted file mode 100755 index 8191d960592..00000000000 --- a/extras/hook-scripts/reset/post/S31ganesha-reset.sh +++ /dev/null @@ -1,47 +0,0 @@ -#/bin/bash -PROGNAME="Sganesha-reset" -OPTSPEC="volname:,gd-workdir:" -VOL= -GLUSTERD_WORKDIR= - -function parse_args () { - ARGS=$(getopt -l $OPTSPEC -o "o" -name $PROGNAME $@) - eval set -- "$ARGS" - while true; do - case $1 in - --volname) - shift - VOL=$1 - ;; - --gd-workdir) - shift - GLUSTERD_WORKDIR=$1 - ;; - *) - shift - break - ;; - esac - shift - done -} - -function is_volume_started () { - volname=$1 - echo "$(grep status $GLUSTERD_WORKDIR/vols/"$volname"/info |\ - cut -d"=" -f2)" -} - -parse_args $@ -if ps aux | grep -q "[g]anesha.nfsd" - then - kill -s TERM `cat /var/run/ganesha.pid` - sleep 10 - rm -rf /var/lib/glusterfs-ganesha/exports - rm -rf /var/lib/glusterfs-ganesha/.export_added - sed -i /conf/d /var/lib/ganesha/nfs-ganesha.conf - if [ "1" = $(is_volume_started "$VOL") ]; - then - gluster volume start $VOL force - fi -fi diff --git a/extras/hook-scripts/set/post/Makefile.am b/extras/hook-scripts/set/post/Makefile.am index 3ec25d94134..506a25a8666 100644 --- a/extras/hook-scripts/set/post/Makefile.am +++ b/extras/hook-scripts/set/post/Makefile.am @@ -1 +1,6 @@ -EXTRA_DIST = S30samba-set.sh S31ganesha-set.sh +EXTRA_DIST = S30samba-set.sh S32gluster_enable_shared_storage.sh + +hookdir = $(GLUSTERD_WORKDIR)/hooks/1/set/post/ +if WITH_SERVER +hook_SCRIPTS = $(EXTRA_DIST) +endif diff --git a/extras/hook-scripts/set/post/S30samba-set.sh b/extras/hook-scripts/set/post/S30samba-set.sh index 9153dcd0bb2..854f131f6c8 100755 --- a/extras/hook-scripts/set/post/S30samba-set.sh +++ b/extras/hook-scripts/set/post/S30samba-set.sh @@ -24,11 +24,11 @@ CONFIGFILE= LOGFILEBASE= PIDDIR= GLUSTERD_WORKDIR= - -enable_smb="" +USERSMB_SET="" +USERCIFS_SET="" function parse_args () { - ARGS=$(getopt -l $OPTSPEC -o "o" -name $PROGNAME $@) + ARGS=$(getopt -o 'o:' -l $OPTSPEC -n $PROGNAME -- "$@") eval set -- "$ARGS" while true; do @@ -41,21 +41,25 @@ function parse_args () { shift GLUSTERD_WORKDIR=$1 ;; - *) + --) + shift + break + ;; + -o) shift - for pair in $@; do - read key value < <(echo "$pair" | tr "=" " ") + read key value < <(echo "$1" | tr "=" " ") case "$key" in "user.cifs") - enable_smb=$value + USERCIFS_SET="YES" ;; "user.smb") - enable_smb=$value + USERSMB_SET="YES" ;; *) ;; esac - done + ;; + *) shift break ;; @@ -85,7 +89,7 @@ function add_samba_share () { STRING+="glusterfs:loglevel = 7\n" STRING+="path = /\n" STRING+="read only = no\n" - STRING+="guest ok = yes\n" + STRING+="kernel share modes = no\n" printf "$STRING" >> ${CONFIGFILE} } @@ -95,13 +99,13 @@ function sighup_samba () { then kill -HUP "$pid"; else - /etc/init.d/smb condrestart + service smb condrestart fi } -function del_samba_share () { +function deactivate_samba_share () { volname=$1 - sed -i "/\[gluster-$volname\]/,/^$/d" /etc/samba/smb.conf + sed -i -e '/^\[gluster-'"$volname"'\]/{ :a' -e 'n; /available = no/H; /^$/!{$!ba;}; x; /./!{ s/^/available = no/; $!{G;x}; $H; }; s/.*//; x; };' ${CONFIGFILE} } function is_volume_started () { @@ -110,21 +114,48 @@ function is_volume_started () { cut -d"=" -f2)" } -parse_args $@ -if [ "0" = $(is_volume_started "$VOL") ]; then +function get_smb () { + volname=$1 + uservalue= + + usercifsvalue=$(grep user.cifs $GLUSTERD_WORKDIR/vols/"$volname"/info |\ + cut -d"=" -f2) + usersmbvalue=$(grep user.smb $GLUSTERD_WORKDIR/vols/"$volname"/info |\ + cut -d"=" -f2) + + if [ -n "$usercifsvalue" ]; then + if [ "$usercifsvalue" = "disable" ] || [ "$usercifsvalue" = "off" ]; then + uservalue="disable" + fi + fi + + if [ -n "$usersmbvalue" ]; then + if [ "$usersmbvalue" = "disable" ] || [ "$usersmbvalue" = "off" ]; then + uservalue="disable" + fi + fi + + echo "$uservalue" +} + +parse_args "$@" +if [ "0" = "$(is_volume_started "$VOL")" ]; then exit 0 fi -#Find smb.conf, smbd pid directory and smbd logfile path -find_config_info -if [ "$enable_smb" = "enable" ]; then - if ! grep --quiet "\[gluster-$VOL\]" /etc/samba/smb.conf ; then +if [ "$USERCIFS_SET" = "YES" ] || [ "$USERSMB_SET" = "YES" ]; then + #Find smb.conf, smbd pid directory and smbd logfile path + find_config_info + + if [ "$(get_smb "$VOL")" = "disable" ]; then + deactivate_samba_share $VOL + else + if ! grep --quiet "\[gluster-$VOL\]" ${CONFIGFILE} ; then add_samba_share $VOL - sighup_samba + else + sed -i '/\[gluster-'"$VOL"'\]/,/^$/!b;/available = no/d' ${CONFIGFILE} + fi fi - -elif [ "$enable_smb" = "disable" ]; then - del_samba_share $VOL sighup_samba fi diff --git a/extras/hook-scripts/set/post/S31ganesha-set.sh b/extras/hook-scripts/set/post/S31ganesha-set.sh deleted file mode 100755 index 6ee4d62debc..00000000000 --- a/extras/hook-scripts/set/post/S31ganesha-set.sh +++ /dev/null @@ -1,284 +0,0 @@ -#!/bin/bash -PROGNAME="Sganesha-set" -OPTSPEC="volname:,gd-workdir:" -VOL= -declare -i EXPORT_ID -GANESHA_DIR="/var/lib/glusterfs-ganesha" -CONF1="$GANESHA_DIR/nfs-ganesha.conf" -GANESHA_LOG_DIR="/var/log/nfs-ganesha/" -LOG="$GANESHA_LOG_DIR/ganesha.nfsd.log" -gnfs="enabled" -enable_ganesha="" -host_name="none" -LOC="" -GLUSTERD_WORKDIR= - - -function parse_args () -{ - ARGS=$(getopt -l $OPTSPEC -o "o" -name $PROGNAME $@) - eval set -- "$ARGS" - - while true; do - case $1 in - --volname) - shift - VOL=$1 - ;; - --gd-workdir) - shift - GLUSTERD_WORKDIR=$1 - ;; - *) - shift - for pair in $@; do - read key value < <(echo "$pair" | tr "=" " ") - case "$key" in - "nfs-ganesha.enable") - enable_ganesha=$value - ;; - "nfs-ganesha.host") - host_name=$value - ;; - *) - ;; - esac - done - shift - break - ;; - esac - shift - done -} - - -function check_if_host_set() -{ - if ! cat $GLUSTERD_WORKDIR/vols/$VOL/info | grep -q "nfs-ganesha.host" - then - exit 1 - fi -} - -function check_nfsd_loc() -{ - if ls /usr/bin | grep "[g]anesha.nfsd" - then - LOC="/usr" - else - LOC="/usr/local" - fi -} - - -function check_gluster_nfs() -{ - if cat $GLUSTERD_WORKDIR/vols/$VOL/info | grep -q "nfs.disable=ON" - then - gnfs="disabled" - fi -} - -function check_cmd_status() -{ - if [ "$1" != "0" ] - then - rm -rf $GANESHA_DIR/exports/export.$VOL.conf - exit 1 - fi -} - - - -#This function generates a new export entry as export.volume_name.conf -function write_conf() -{ - echo "EXPORT{ - " - echo "Export_Id = ;" - echo "Path=\"/$1\";" - echo "FSAL { - " - echo "name = "GLUSTER";" - echo "hostname=\"$2\";" - echo "volume=\"$1\";" - echo "}" - echo "Access_type = RW;" - echo "Squash = No_root_squash;" - echo "Disable_ACL = TRUE;" - echo "Pseudo=\"/$1\";" - echo "Protocols = \"3,4\" ;" - echo "Transports = \"UDP,TCP\" ;" - echo "SecType = \"sys\";" - echo "Tag = \"$1\";" - echo "}" -} - -#This function keeps track of export IDs and increments it with every new entry -function export_add() -{ - count=`ls -l $GANESHA_DIR/exports/*.conf | wc -l` - if [ "$count" = "1" ] ; - then - EXPORT_ID=1 - else - #if [ -s /var/lib/ganesha/export_removed ]; - # then - # EXPORT_ID=`head -1 /var/lib/ganesha/export_removed` - # sed -i -e "1d" /var/lib/ganesha/export_removed - # else - - EXPORT_ID=`cat $GANESHA_DIR/.export_added` - check_cmd_status `echo $?` - EXPORT_ID=EXPORT_ID+1 - #fi - fi - echo $EXPORT_ID > $GANESHA_DIR/.export_added - check_cmd_status `echo $?` - sed -i s/Export_Id.*/"Export_Id = $EXPORT_ID ;"/ \ -$GANESHA_DIR/exports/export.$VOL.conf - echo "%include \"$GANESHA_DIR/exports/export.$VOL.conf\"" >> $CONF1 - check_cmd_status `echo $?` -} - -#This function removes an export dynamically(uses the export_id of the export) -function dynamic_export_remove() -{ - removed_id=`cat $GANESHA_DIR/exports/export.$VOL.conf |\ -grep Export_Id | cut -d " " -f3` - check_cmd_status `echo $?` - dbus-send --print-reply --system \ ---dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ -org.ganesha.nfsd.exportmgr.RemoveExport int32:$removed_id - check_cmd_status `echo $?` - -} - -#This function adds a new export dynamically by sending dbus signals -function dynamic_export_add() -{ - dbus-send --print-reply --system --dest=org.ganesha.nfsd \ -/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.AddExport \ -string:$GANESHA_DIR/exports/export.$VOL.conf string:"EXPORT(Tag=$VOL)" - check_cmd_status `echo $?` - -} - -function start_ganesha() -{ - check_gluster_nfs - #Remove export entry from nfs-ganesha.conf - sed -i /$VOL.conf/d $CONF1 - #Create a new export entry - export_add - if ! ps aux | grep -q "[g]anesha.nfsd" - then - if ls /usr/bin/ganesha.nfsd - then - /usr/bin/ganesha.nfsd -f $CONF1 -L $LOG -N NIV_EVENT -d - sleep 2 - else - /usr/local/bin/ganesha.nfsd -f $CONF1 -L $LOG -N NIV_EVENT -d - sleep 2 - fi - else - dynamic_export_add $VOL - fi - - - if !( ps aux | grep -q "[g]anesha.nfsd") - then - rm -rf $GANESHA_DIR/exports/* - rm -rf $GANESHA_DIR/.export_added - exit 1 - fi - -} - -#This function generates a new config file when ganesha.host is set -#If the volume is already exported, only hostname is changed -function set_hostname() -{ - if ! ls $GANESHA_DIR/exports/ | grep -q $VOL.conf - then - write_conf $VOL $host_name >\ -$GANESHA_DIR/exports/export.$VOL.conf - else - sed -i s/hostname.*/"hostname=\ -\"$host_name\";"/ $GANESHA_DIR/exports/export.$VOL.conf - fi -} - - -function check_ganesha_dir() -{ - #Check if the configuration file is placed in /etc/glusterfs-ganesha - if ! ls /etc/glusterfs-ganesha | grep "nfs-ganesha.conf" - then - exit 1 - else - if [ ! -d "$GANESHA_DIR" ]; - then - mkdir $GANESHA_DIR - check_cmd_status `echo $?` - fi - cp /etc/glusterfs-ganesha/nfs-ganesha.conf $GANESHA_DIR/ - check_cmd_status `echo $?` - fi - if [ ! -d "$GANESHA_DIR/exports" ]; - then - mkdir $GANESHA_DIR/exports - check_cmd_status `echo $?` - fi - if [ ! -d "$GANESHA_LOG_DIR" ] ; - then - mkdir $GANESHA_LOG_DIR - check_cmd_status `echo $?` - fi - - - -} - -function stop_ganesha() -{ - dynamic_export_remove $VOL - #Remove the specfic export configuration file - rm -rf $GANESHA_DIR/exports/export.$VOL.conf - #Remove that entry from nfs-ganesha.conf - sed -i /$VOL.conf/d $CONF1 - #If there are no other volumes exported, stop nfs-ganesha - if [ ! "$(ls -A $GANESHA_DIR/exports)" ]; - then - pkill ganesha.nfsd - rm -rf $GANESHA_DIR/.export_added - fi -} - - parse_args $@ - check_ganesha_dir $VOL - if echo $enable_ganesha | grep -q -i "ON" - then - check_if_host_set $VOL - if ! showmount -e localhost | cut -d "" -f1 | grep -q "$VOL[[:space:]]" - then - start_ganesha - fi - elif echo $enable_ganesha | grep -q -i "OFF" - then - check_if_host_set $VOL - stop_ganesha - fi - if [ "$host_name" != "none" ]; - then - if showmount -e localhost | cut -d "" -f1 | grep -q "$VOL[[:space:]]" - then - dynamic_export_remove $VOL - set_hostname - start_ganesha - else - set_hostname - fi - - fi diff --git a/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh b/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh new file mode 100755 index 00000000000..1f2564b44ff --- /dev/null +++ b/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +key=`echo $3 | cut -d '=' -f 1` +val=`echo $3 | cut -d '=' -f 2` +if [ "$key" != "cluster.enable-shared-storage" ] && [ "$key" != "enable-shared-storage" ]; then + exit; +fi +if [ "$val" != 'enable' ]; then + if [ "$val" != 'disable' ]; then + exit; + fi +fi + +option=$val + +key_val_pair1=`echo $4 | cut -d ',' -f 1` +key_val_pair2=`echo $4 | cut -d ',' -f 2` + +key=`echo $key_val_pair1 | cut -d '=' -f 1` +val=`echo $key_val_pair1 | cut -d '=' -f 2` +if [ "$key" != "is_originator" ]; then + exit; +fi +is_originator=$val; + +key=`echo $key_val_pair2 | cut -d '=' -f 1` +val=`echo $key_val_pair2 | cut -d '=' -f 2` +if [ "$key" != "local_node_hostname" ]; then + exit; +fi +local_node_hostname=$val; + +# Read gluster peer status to find the peers +# which are in 'Peer in Cluster' mode and +# are connected. + +number_of_connected_peers=0 +while read -r line +do + # Already got two connected peers. Including the current node + # we have 3 peers which is enough to create a shared storage + # with replica 3 + if [ "$number_of_connected_peers" == "2" ]; then + break; + fi + + key=`echo $line | cut -d ':' -f 1` + if [ "$key" == "Hostname" ]; then + hostname=`echo $line | cut -d ':' -f 2 | xargs` + fi + + if [ "$key" == "State" ]; then + peer_state=`echo $line | cut -d ':' -f 2 | cut -d '(' -f 1 | xargs` + conn_state=`echo $line | cut -d '(' -f 2 | cut -d ')' -f 1 | xargs` + + if [ "$peer_state" == "Peer in Cluster" ]; then + if [ "$conn_state" == "Connected" ]; then + ((number_of_connected_peers++)) + connected_peer[$number_of_connected_peers]=$hostname + fi + fi + fi + +done < <(gluster peer status) + +# Include current node in connected peer list +((number_of_connected_peers++)) +connected_peer[$number_of_connected_peers]=$local_node_hostname + +# forming the create vol command +create_cmd="gluster --mode=script --wignore volume create \ + gluster_shared_storage replica $number_of_connected_peers" + +# Adding the brick names in the command +for i in "${connected_peer[@]}" +do + create_cmd=$create_cmd" "$i:"$GLUSTERD_WORKDIR"/ss_brick +done + +if [ "$option" == "disable" ]; then + # Unmount the volume on all the nodes + umount /run/gluster/shared_storage + cat /etc/fstab | grep -v "gluster_shared_storage /run/gluster/shared_storage/" > /run/gluster/fstab.tmp + mv /run/gluster/fstab.tmp /etc/fstab +fi + +if [ "$is_originator" == 1 ]; then + if [ "$option" == "enable" ]; then + # Create and start the volume + $create_cmd + gluster --mode=script --wignore volume start gluster_shared_storage + fi + + if [ "$option" == "disable" ]; then + # Stop and delete the volume + gluster --mode=script --wignore volume stop gluster_shared_storage + gluster --mode=script --wignore volume delete gluster_shared_storage + fi +fi + +function check_volume_status() +{ + status=`gluster volume info gluster_shared_storage | grep Status | cut -d ':' -f 2 | xargs` + echo $status +} + +key=`echo $5 | cut -d '=' -f 1` +val=`echo $5 | cut -d '=' -f 2` +if [ "$key" == "transport.address-family" ]; then + mount_cmd="mount -t glusterfs -o xlator-option=transport.address-family=inet6 \ + $local_node_hostname:/gluster_shared_storage /run/gluster/shared_storage" +else + mount_cmd="mount -t glusterfs $local_node_hostname:/gluster_shared_storage \ + /run/gluster/shared_storage" +fi + +if [ "$option" == "enable" ]; then + retry=0; + # Wait for volume to start before mounting + status=$(check_volume_status) + while [ "$status" != "Started" ]; do + sleep 5; + ((retry++)) + if [ "$retry" == 3 ]; then + break; + fi + status=$(check_volume_status) + done + # Mount the volume on all the nodes + umount /run/gluster/shared_storage + mkdir -p /run/gluster/shared_storage + $mount_cmd + cp /etc/fstab /run/gluster/fstab.tmp + echo "$local_node_hostname:/gluster_shared_storage /run/gluster/shared_storage/ glusterfs defaults 0 0" >> /run/gluster/fstab.tmp + mv /run/gluster/fstab.tmp /etc/fstab +fi diff --git a/extras/hook-scripts/start/post/Makefile.am b/extras/hook-scripts/start/post/Makefile.am index d9cba93ed52..792019d3c9f 100644 --- a/extras/hook-scripts/start/post/Makefile.am +++ b/extras/hook-scripts/start/post/Makefile.am @@ -1 +1,6 @@ -EXTRA_DIST = S29CTDBsetup.sh S30samba-start.sh +EXTRA_DIST = S29CTDBsetup.sh S30samba-start.sh S31ganesha-start.sh + +hookdir = $(GLUSTERD_WORKDIR)/hooks/1/start/post/ +if WITH_SERVER +hook_SCRIPTS = $(EXTRA_DIST) +endif diff --git a/extras/hook-scripts/start/post/S29CTDBsetup.sh b/extras/hook-scripts/start/post/S29CTDBsetup.sh index cbb76767eb9..69a0d89a3eb 100755 --- a/extras/hook-scripts/start/post/S29CTDBsetup.sh +++ b/extras/hook-scripts/start/post/S29CTDBsetup.sh @@ -1,21 +1,22 @@ #! /bin/bash -# RHS-2.0 only +# # - The script mounts the 'meta-vol' on start 'event' on a known # directory (eg. /gluster/lock) -# - Adds the necessary configuration changes for ctdb in smb.conf and -# restarts smb service. # - P.S: There are other 'tasks' that need to be done outside this script # to get CTDB based failover up and running. -SMB_CONF=/etc/samba/smb.conf CTDB_MNT=/gluster/lock +# Make sure ping-timeout is not default for CTDB volume PING_TIMEOUT_SECS=10 PROGNAME="ctdb" -OPTSPEC="volname:" +OPTSPEC="volname:,gd-workdir:,version:,volume-op:,first:" HOSTNAME=`hostname` -MNTOPTS="_netdev,defaults" -MNTOPTS_GLUSTERFS="transport=tcp,xlator-option=*client*.ping-timeout=${PING_TIMEOUT_SECS}" +MNTOPTS="_netdev,transport=tcp,xlator-option=*client*.ping-timeout=${PING_TIMEOUT_SECS}" VOL= +GLUSTERD_WORKDIR= +VERSION= +VOLUME_OP= +FIRST= # $META is the volume that will be used by CTDB as a shared filesystem. # It is not desirable to use this volume for storing 'data' as well. # META is set to 'all' (viz. a keyword and hence not a legal volume name) @@ -24,7 +25,7 @@ VOL= META="all" function parse_args () { - ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) + ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") eval set -- "$ARGS" while true; do @@ -32,33 +33,37 @@ function parse_args () { --volname) shift VOL=$1 - ;; - + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --version) + shift + VERSION=$1 + ;; + --volume-op) + shift + VOLUME_OP=$1 + ;; + --first) + shift + FIRST=$1 + ;; *) - shift - break - ;; - + shift + break + ;; esac shift done } -function add_glusterfs_ctdb_options () { - PAT="Share Definitions" - GLUSTER_CTDB_CONFIG="# ctdb config for glusterfs\n\tclustering = yes\n\tidmap backend = tdb2\n" - exists=`grep "clustering = yes" "$SMB_CONF"` - if [ "$exists" == "" ] - then - sed -i /"$PAT"/i\ "$GLUSTER_CTDB_CONFIG" "$SMB_CONF" - fi -} - function add_fstab_entry () { volname=$1 mntpt=$2 - mntopts="${MNTOPTS},${MNTOPTS_GLUSTERFS}" + mntopts="${MNTOPTS}" mntent="${HOSTNAME}:/${volname} ${mntpt} glusterfs ${mntopts} 0 0" exists=`grep "${mntpt}" /etc/fstab` @@ -68,16 +73,12 @@ function add_fstab_entry () { fi } -parse_args $@ +parse_args "$@" if [ "$META" = "$VOL" ] then - # expects ctdb service to manage smb - service smb stop - add_glusterfs_ctdb_options mkdir -p $CTDB_MNT sleep 5 - # Make sure ping-timeout is not default for CTDB volume - mount -t glusterfs -oxlator-option=*client*.ping-timeout=${PING_TIMEOUT_SECS} `hostname`:$VOL "$CTDB_MNT" && \ + mount -t glusterfs -o${MNTOPTS} ${HOSTNAME}:/$VOL "$CTDB_MNT" && \ add_fstab_entry $VOL $CTDB_MNT chkconfig ctdb on fi diff --git a/extras/hook-scripts/start/post/S30samba-start.sh b/extras/hook-scripts/start/post/S30samba-start.sh index 43dc8e108da..cac0cbf1464 100755 --- a/extras/hook-scripts/start/post/S30samba-start.sh +++ b/extras/hook-scripts/start/post/S30samba-start.sh @@ -21,15 +21,18 @@ #volume. PROGNAME="Ssamba-start" -OPTSPEC="volname:,gd-workdir:" +OPTSPEC="volname:,gd-workdir:,version:,volume-op:,first:" VOL= CONFIGFILE= LOGFILEBASE= PIDDIR= GLUSTERD_WORKDIR= +VERSION= +VOLUME_OP= +FIRST= function parse_args () { - ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) + ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") eval set -- "$ARGS" while true; do @@ -42,24 +45,37 @@ function parse_args () { shift GLUSTERD_WORKDIR=$1 ;; + --version) + shift + VERSION=$1 + ;; + --volume-op) + shift + VOLUME_OP=$1 + ;; + --first) + shift + FIRST=$1 + ;; *) shift break ;; esac + shift done } function find_config_info () { - cmdout=`smbd -b | grep smb.conf` - if [ $? -ne 0 ];then + cmdout=$(smbd -b 2> /dev/null) + CONFIGFILE=$(echo "$cmdout" | grep CONFIGFILE | awk '{print $2}') + if [ -z "$CONFIGFILE" ]; then echo "Samba is not installed" exit 1 fi - CONFIGFILE=`echo $cmdout | awk {'print $2'}` - PIDDIR=`smbd -b | grep PIDDIR | awk {'print $2'}` - LOGFILEBASE=`smbd -b | grep 'LOGFILEBASE' | awk '{print $2}'` + PIDDIR=$(echo "$cmdout" | grep PIDDIR | awk '{print $2}') + LOGFILEBASE=$(echo "$cmdout" | grep 'LOGFILEBASE' | awk '{print $2}') } function add_samba_share () { @@ -72,17 +88,17 @@ function add_samba_share () { STRING+="glusterfs:loglevel = 7\n" STRING+="path = /\n" STRING+="read only = no\n" - STRING+="guest ok = yes\n" - printf "$STRING" >> ${CONFIGFILE} + STRING+="kernel share modes = no\n" + printf "$STRING" >> "${CONFIGFILE}" } function sighup_samba () { - pid=`cat ${PIDDIR}/smbd.pid` + pid=$(cat "${PIDDIR}/smbd.pid" 2> /dev/null) if [ "x$pid" != "x" ] then kill -HUP "$pid"; else - /etc/init.d/smb condrestart + service smb condrestart fi } @@ -90,26 +106,40 @@ function get_smb () { volname=$1 uservalue= - usercifsvalue=$(grep user.cifs $GLUSTERD_WORKDIR/vols/"$volname"/info |\ + usercifsvalue=$(grep user.cifs "$GLUSTERD_WORKDIR"/vols/"$volname"/info |\ cut -d"=" -f2) - usersmbvalue=$(grep user.smb $GLUSTERD_WORKDIR/vols/"$volname"/info |\ + usersmbvalue=$(grep user.smb "$GLUSTERD_WORKDIR"/vols/"$volname"/info |\ cut -d"=" -f2) - if [[ $usercifsvalue = "disable" || $usersmbvalue = "disable" ]]; then - uservalue="disable" + if [ -n "$usercifsvalue" ]; then + if [ "$usercifsvalue" = "enable" ] || [ "$usercifsvalue" = "on" ]; then + uservalue="enable" + fi + fi + + if [ -n "$usersmbvalue" ]; then + if [ "$usersmbvalue" = "enable" ] || [ "$usersmbvalue" = "on" ]; then + uservalue="enable" + fi fi + echo "$uservalue" } -parse_args $@ -if [ $(get_smb "$VOL") = "disable" ]; then +parse_args "$@" + +value=$(get_smb "$VOL") + +if [ -z "$value" ] || [ "$value" != "enable" ]; then exit 0 fi #Find smb.conf, smbd pid directory and smbd logfile path find_config_info -if ! grep --quiet "\[gluster-$VOL\]" ${CONFIGFILE} ; then - add_samba_share $VOL - sighup_samba +if ! grep --quiet "\[gluster-$VOL\]" "${CONFIGFILE}" ; then + add_samba_share "$VOL" +else + sed -i '/\[gluster-'"$VOL"'\]/,/^$/!b;/available = no/d' "${CONFIGFILE}" fi +sighup_samba diff --git a/extras/hook-scripts/start/post/S31ganesha-start.sh b/extras/hook-scripts/start/post/S31ganesha-start.sh new file mode 100755 index 00000000000..7ad6f23ad06 --- /dev/null +++ b/extras/hook-scripts/start/post/S31ganesha-start.sh @@ -0,0 +1,122 @@ +#!/bin/bash +PROGNAME="Sganesha-start" +OPTSPEC="volname:,gd-workdir:" +VOL= +declare -i EXPORT_ID +ganesha_key="ganesha.enable" +GANESHA_DIR="/run/gluster/shared_storage/nfs-ganesha" +CONF1="$GANESHA_DIR/ganesha.conf" +GLUSTERD_WORKDIR= + +function parse_args () +{ + ARGS=$(getopt -l $OPTSPEC -o "o" -name $PROGNAME $@) + eval set -- "$ARGS" + + while true; do + case $1 in + --volname) + shift + VOL=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + *) + shift + break + ;; + esac + shift + done +} + + + +#This function generates a new export entry as export.volume_name.conf +function write_conf() +{ +echo -e "# WARNING : Using Gluster CLI will overwrite manual +# changes made to this file. To avoid it, edit the +# file, copy it over to all the NFS-Ganesha nodes +# and run ganesha-ha.sh --refresh-config." + +echo "EXPORT{" +echo " Export_Id = 2;" +echo " Path = \"/$VOL\";" +echo " FSAL {" +echo " name = \"GLUSTER\";" +echo " hostname=\"localhost\";" +echo " volume=\"$VOL\";" +echo " }" +echo " Access_type = RW;" +echo " Disable_ACL = true;" +echo " Squash=\"No_root_squash\";" +echo " Pseudo=\"/$VOL\";" +echo " Protocols = \"3\", \"4\" ;" +echo " Transports = \"UDP\",\"TCP\";" +echo " SecType = \"sys\";" +echo "}" +} + +#It adds the export dynamically by sending dbus signals +function export_add() +{ + dbus-send --print-reply --system --dest=org.ganesha.nfsd \ +/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.AddExport \ +string:$GANESHA_DIR/exports/export.$VOL.conf string:"EXPORT(Export_Id=$EXPORT_ID)" + +} + +# based on src/scripts/ganeshactl/Ganesha/export_mgr.py +function is_exported() +{ + local volume="${1}" + + dbus-send --type=method_call --print-reply --system \ + --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ + org.ganesha.nfsd.exportmgr.ShowExports \ + | grep -w -q "/${volume}" + + return $? +} + +# Check the info file (contains the volume options) to see if Ganesha is +# enabled for this volume. +function ganesha_enabled() +{ + local volume="${1}" + local info_file="${GLUSTERD_WORKDIR}/vols/${VOL}/info" + local enabled="off" + + enabled=$(grep -w ${ganesha_key} ${info_file} | cut -d"=" -f2) + + [ "${enabled}" == "on" ] + + return $? +} + +parse_args $@ + +if ganesha_enabled ${VOL} && ! is_exported ${VOL} +then + if [ ! -e ${GANESHA_DIR}/exports/export.${VOL}.conf ] + then + #Remove export entry from nfs-ganesha.conf + sed -i /$VOL.conf/d $CONF1 + write_conf ${VOL} > ${GANESHA_DIR}/exports/export.${VOL}.conf + EXPORT_ID=`cat $GANESHA_DIR/.export_added` + EXPORT_ID=EXPORT_ID+1 + echo $EXPORT_ID > $GANESHA_DIR/.export_added + sed -i s/Export_Id.*/"Export_Id=$EXPORT_ID;"/ \ + $GANESHA_DIR/exports/export.$VOL.conf + echo "%include \"$GANESHA_DIR/exports/export.$VOL.conf\"" >> $CONF1 + else + EXPORT_ID=$(grep ^[[:space:]]*Export_Id $GANESHA_DIR/exports/export.$VOL.conf |\ + awk -F"[=,;]" '{print $2}' | tr -d '[[:space:]]') + fi + export_add $VOL +fi + +exit 0 diff --git a/extras/hook-scripts/stop/pre/Makefile.am b/extras/hook-scripts/stop/pre/Makefile.am index 85243adbec9..9e8d1565e93 100644 --- a/extras/hook-scripts/stop/pre/Makefile.am +++ b/extras/hook-scripts/stop/pre/Makefile.am @@ -1 +1,6 @@ EXTRA_DIST = S29CTDB-teardown.sh S30samba-stop.sh + +hookdir = $(GLUSTERD_WORKDIR)/hooks/1/stop/pre/ +if WITH_SERVER +hook_SCRIPTS = $(EXTRA_DIST) +endif diff --git a/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh b/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh index 9125030bb7e..0975a00f18d 100755 --- a/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh +++ b/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh @@ -1,11 +1,10 @@ #! /bin/bash -#non-portable - RHS-2.0 only -SMB_CONF=/etc/samba/smb.conf CTDB_MNT=/gluster/lock PROGNAME="ctdb" -OPTSPEC="volname:" +OPTSPEC="volname:,last:" VOL= +LAST= # $META is the volume that will be used by CTDB as a shared filesystem. # It is not desirable to use this volume for storing 'data' as well. # META is set to 'all' (viz. a keyword and hence not a legal volume name) @@ -13,18 +12,8 @@ VOL= # User needs to set META to the volume that serves CTDB lockfile. META="all" -function sighup_samba () { - pid=`cat /var/run/smbd.pid` - if [ "$pid" != "" ] - then - kill -HUP $pid; - else - /etc/init.d/smb start - fi -} - function parse_args () { - ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) + ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") eval set -- "$ARGS" while true; do @@ -32,31 +21,21 @@ function parse_args () { --volname) shift VOL=$1 - ;; - + ;; + --last) + shift + LAST=$1 + ;; *) - shift - break - ;; - + shift + break + ;; esac - shift done } -function remove_ctdb_options () { - IFS=$'\n' - GLUSTER_CTDB_CONFIG=$'# ctdb config for glusterfs\n\tclustering = yes\n\tidmap backend = tdb2\n' - - for line in $GLUSTER_CTDB_CONFIG - do - sed -i /"$line"/d $SMB_CONF - done - unset IFS -} - function remove_fstab_entry () { mntpt=$1 fstab="/etc/fstab" @@ -74,12 +53,10 @@ function remove_fstab_entry () { fi } -parse_args $@ +parse_args "$@" if [ "$META" = "$VOL" ] then umount "$CTDB_MNT" chkconfig ctdb off remove_fstab_entry $CTDB_MNT - remove_ctdb_options - sighup_samba fi diff --git a/extras/hook-scripts/stop/pre/S30samba-stop.sh b/extras/hook-scripts/stop/pre/S30samba-stop.sh index 8950eea436e..ea799381d62 100755 --- a/extras/hook-scripts/stop/pre/S30samba-stop.sh +++ b/extras/hook-scripts/stop/pre/S30samba-stop.sh @@ -16,27 +16,33 @@ #event by removing the volume related entries(if any) in smb.conf file. PROGNAME="Ssamba-stop" -OPTSPEC="volname:" +OPTSPEC="volname:,last:" VOL= CONFIGFILE= PIDDIR= +LAST= function parse_args () { - ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) + ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") eval set -- "$ARGS" while true; do - case $1 in - --volname) - shift - VOL=$1 - ;; - *) - shift - break - ;; - esac - shift + case $1 in + --volname) + shift + VOL=$1 + ;; + --last) + shift + LAST=$1 + ;; + *) + shift + break + ;; + esac + + shift done } @@ -46,13 +52,13 @@ function find_config_info () { echo "Samba is not installed" exit 1 fi - CONFIGFILE=`echo $cmdout | awk {'print $2'}` - PIDDIR=`smbd -b | grep PIDDIR | awk {'print $2'}` + CONFIGFILE=`echo $cmdout | awk '{print $2}'` + PIDDIR=`smbd -b | grep PIDDIR | awk '{print $2}'` } -function del_samba_share () { +function deactivate_samba_share () { volname=$1 - sed -i "/\[gluster-$volname\]/,/^$/d" ${CONFIGFILE} + sed -i -e '/^\[gluster-'"$volname"'\]/{ :a' -e 'n; /available = no/H; /^$/!{$!ba;}; x; /./!{ s/^/available = no/; $!{G;x}; $H; }; s/.*//; x; };' ${CONFIGFILE} } function sighup_samba () { @@ -61,11 +67,11 @@ function sighup_samba () { then kill -HUP $pid; else - /etc/init.d/smb condrestart + service smb condrestart fi } -parse_args $@ +parse_args "$@" find_config_info -del_samba_share $VOL +deactivate_samba_share $VOL sighup_samba diff --git a/extras/identify-hangs.sh b/extras/identify-hangs.sh new file mode 100755 index 00000000000..ebc6bf144aa --- /dev/null +++ b/extras/identify-hangs.sh @@ -0,0 +1,53 @@ +#!/bin/bash +function get_statedump_fnames_without_timestamps +{ + ls | grep -E "[.]dump[.][0-9][0-9]*" | cut -f1-3 -d'.' | sort -u +} + +function get_non_uniq_fields +{ + local statedump_fname_prefix=$1 + print_stack_lkowner_unique_in_one_line "$statedump_fname_prefix" | sort | uniq -c | grep -vE "^\s*1 " | awk '{$1="repeats="$1; print $0}' +} + +function print_stack_lkowner_unique_in_one_line +{ + local statedump_fname_prefix=$1 + sed -e '/./{H;$!d;}' -e 'x;/unique=/!d;/stack=/!d;/lk-owner=/!d;/pid=/!d;' "${statedump_fname_prefix}"* | grep -E "(stack|lk-owner|unique|pid)=" | paste -d " " - - - - +} + +function get_stacks_that_appear_in_multiple_statedumps +{ + #If a stack with same 'unique/lk-owner/stack' appears in multiple statedumps + #print the stack + local statedump_fname_prefix=$1 + while read -r non_uniq_stack; + do + if [ -z "$printed" ]; + then + printed="1" + fi + echo "$statedump_fname_prefix" "$non_uniq_stack" + done < <(get_non_uniq_fields "$statedump_fname_prefix") +} + +statedumpdir=${1} +if [ -z "$statedumpdir" ]; +then + echo "Usage: $0 <statedump-dir>" + exit 1 +fi + +if [ ! -d "$statedumpdir" ]; +then + echo "$statedumpdir: Is not a directory" + echo "Usage: $0 <statedump-dir>" + exit 1 +fi + +cd "$statedumpdir" || exit 1 +for statedump_fname_prefix in $(get_statedump_fnames_without_timestamps); +do + get_stacks_that_appear_in_multiple_statedumps "$statedump_fname_prefix" +done | column -t +echo "NOTE: stacks with lk-owner=\"\"/lk-owner=0000000000000000/unique=0 may not be hung frames and need further inspection" >&2 diff --git a/extras/init.d/Makefile.am b/extras/init.d/Makefile.am index 347545fe520..8d8cc69571a 100644 --- a/extras/init.d/Makefile.am +++ b/extras/init.d/Makefile.am @@ -1,5 +1,7 @@ -EXTRA_DIST = glusterd-Debian glusterd-FreeBSD glusterd-Redhat glusterd-SuSE glusterd.plist rhel5-load-fuse.modules +EXTRA_DIST = glusterd-Debian glusterd-FreeBSD glusterd-Redhat \ + glusterd-SuSE glusterd.plist glustereventsd-FreeBSD \ + glustereventsd-Redhat glustereventsd-Debian CLEANFILES = @@ -8,15 +10,23 @@ SYSTEMD_DIR = @systemddir@ LAUNCHD_DIR = @launchddir@ $(GF_DISTRIBUTION): +if WITH_SERVER @if [ ! -d $(SYSTEMD_DIR) ]; then \ - $(MKDIR_P) $(DESTDIR)$(INIT_DIR); \ + $(mkdir_p) $(DESTDIR)$(INIT_DIR); \ $(INSTALL_PROGRAM) glusterd-$(GF_DISTRIBUTION) $(DESTDIR)$(INIT_DIR)/glusterd; \ fi +endif +if BUILD_EVENTS + @if [ ! -d $(SYSTEMD_DIR) ]; then \ + $(mkdir_p) $(DESTDIR)$(INIT_DIR); \ + $(INSTALL_PROGRAM) glustereventsd-$(GF_DISTRIBUTION) $(DESTDIR)$(INIT_DIR)/glustereventsd; \ + fi +endif install-exec-local: $(GF_DISTRIBUTION) install-data-local: if GF_DARWIN_HOST_OS - $(MKDIR_P) $(DESTDIR)$(LAUNCHD_DIR) + $(mkdir_p) $(DESTDIR)$(LAUNCHD_DIR) $(INSTALL_PROGRAM) glusterd.plist $(DESTDIR)$(LAUNCHD_DIR)/org.gluster.glusterd.plist endif diff --git a/extras/init.d/glusterd-Redhat.in b/extras/init.d/glusterd-Redhat.in index c6254a066ad..94801fe31a5 100755 --- a/extras/init.d/glusterd-Redhat.in +++ b/extras/init.d/glusterd-Redhat.in @@ -76,6 +76,7 @@ stop() killproc $BASE fi RETVAL=$? + echo [ $RETVAL -eq 0 ] && rm -f $LOCKFILE return $RETVAL } diff --git a/extras/init.d/glustereventsd-Debian.in b/extras/init.d/glustereventsd-Debian.in new file mode 100644 index 00000000000..6eebdb2b8d8 --- /dev/null +++ b/extras/init.d/glustereventsd-Debian.in @@ -0,0 +1,91 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: glustereventsd +# Required-Start: $local_fs $network +# Required-Stop: $local_fs $network +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Gluster Events Server +# Description: Gluster Events Server +### END INIT INFO + +# Author: Chris AtLee <chris@atlee.ca> +# Patched by: Matthias Albert < matthias@linux4experts.de> + +PATH=/sbin:/usr/sbin:/bin:/usr/bin +NAME=glustereventsd +SCRIPTNAME=/etc/init.d/$NAME +DAEMON=@prefix@/sbin/$NAME +PIDFILE=/var/run/$NAME.pid +GLUSTEREVENTSD_OPTS="" +PID=`test -f $PIDFILE && cat $PIDFILE` + + +# Gracefully exit if the package has been removed. +test -x $DAEMON || exit 0 + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +. /lib/lsb/init-functions + + +do_start() +{ + pidofproc -p $PIDFILE $DAEMON >/dev/null + status=$? + if [ $status -eq 0 ]; then + log_success_msg "glustereventsd service is already running with pid $PID" + else + log_daemon_msg "Starting glustereventsd service" "glustereventsd" + start-stop-daemon --start --quiet --oknodo --pidfile $PIDFILE --startas $DAEMON -- -p $PIDFILE $GLUSTEREVENTSD_OPTS + log_end_msg $? + start_daemon -p $PIDFILE $DAEMON -f $CONFIGFILE + return $? + fi +} + +do_stop() +{ + log_daemon_msg "Stopping glustereventsd service" "glustereventsd" + start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE + log_end_msg $? + rm -f $PIDFILE + killproc -p $PIDFILE $DAEMON + return $? +} + +do_status() +{ + pidofproc -p $PIDFILE $DAEMON >/dev/null + status=$? + if [ $status -eq 0 ]; then + log_success_msg "glustereventsd service is running with pid $PID" + else + log_failure_msg "glustereventsd service is not running." + fi + exit $status +} + +case "$1" in + start) + do_start + ;; + stop) + do_stop + ;; + status) + do_status; + ;; + restart|force-reload) + do_stop + sleep 2 + do_start + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + diff --git a/extras/init.d/glustereventsd-FreeBSD.in b/extras/init.d/glustereventsd-FreeBSD.in new file mode 100644 index 00000000000..2e8303ec6c6 --- /dev/null +++ b/extras/init.d/glustereventsd-FreeBSD.in @@ -0,0 +1,19 @@ +#!/bin/sh +# +# $FreeBSD$ +# + +# PROVIDE: glustereventsd + +. /etc/rc.subr + +name="glustereventsd" +rcvar=`set_rcvar` +command=@prefix@/sbin/${name} +command_interpreter=/usr/local/bin/python +pidfile="/var/run/${name}.pid" +glustereventsd_flags="-p /var/run/${name}.pid" +start_cmd="/usr/sbin/daemon $command ${glustereventsd_flags}" + +load_rc_config $name +run_rc_command "$1" diff --git a/extras/init.d/glustereventsd-Redhat.in b/extras/init.d/glustereventsd-Redhat.in new file mode 100644 index 00000000000..d23ce4c244f --- /dev/null +++ b/extras/init.d/glustereventsd-Redhat.in @@ -0,0 +1,129 @@ +#!/bin/bash +# +# glustereventsd Startup script for the glusterfs Events server +# +# chkconfig: - 20 80 +# description: Gluster Events Server + +### BEGIN INIT INFO +# Provides: glustereventsd +# Required-Start: $local_fs $network +# Required-Stop: $local_fs $network +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: glusterfs Events server +# Description: GlusterFS Events Server +### END INIT INFO +# + +# Source function library. +. /etc/rc.d/init.d/functions + +BASE=glustereventsd + +# Fedora File System Layout dictates /run +[ -e /run ] && RUNDIR="/run" +PIDFILE="${RUNDIR:-/var/run}/${BASE}.pid" + +PID=`test -f $PIDFILE && cat $PIDFILE` + +GLUSTEREVENTSD_BIN=@prefix@/sbin/$BASE +GLUSTEREVENTSD_OPTS="--pid-file=$PIDFILE" +GLUSTEREVENTSD="$GLUSTEREVENTSD_BIN $GLUSTEREVENTSD_OPTS" +RETVAL=0 + +LOCKFILE=/var/lock/subsys/${BASE} + +# Start the service $BASE +start() +{ + if pidofproc -p $PIDFILE $GLUSTEREVENTSD_BIN &> /dev/null; then + echo "glustereventsd service is already running with pid $PID" + return 0 + else + echo -n $"Starting $BASE:" + daemon $GLUSTEREVENTSD & + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch $LOCKFILE + return $RETVAL + fi +} + +# Stop the service $BASE +stop() +{ + echo -n $"Stopping $BASE:" + if pidofproc -p $PIDFILE $GLUSTEREVENTSD_BIN &> /dev/null; then + killproc -p $PIDFILE $BASE + else + killproc $BASE + fi + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && rm -f $LOCKFILE + return $RETVAL +} + +restart() +{ + stop + start +} + +reload() +{ + restart +} + +force_reload() +{ + restart +} + +rh_status() +{ + status $BASE +} + +rh_status_q() +{ + rh_status &>/dev/null +} + + +### service arguments ### +case $1 in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + reload) + rh_status_q || exit 7 + $1 + ;; + force-reload) + force_reload + ;; + status) + rh_status + ;; + condrestart|try-restart) + rh_status_q || exit 0 + restart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" + exit 1 +esac + +exit $? diff --git a/extras/init.d/rhel5-load-fuse.modules b/extras/init.d/rhel5-load-fuse.modules deleted file mode 100755 index ee194db99b8..00000000000 --- a/extras/init.d/rhel5-load-fuse.modules +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -# -# fusermount-glusterfs requires the /dev/fuse character device. The fuse module -# provides this and is loaded on demand in newer Linux distributions. -# - -[ -c /dev/fuse ] || /sbin/modprobe fuse diff --git a/extras/mount-shared-storage.sh b/extras/mount-shared-storage.sh new file mode 100755 index 00000000000..cc40e13c3e3 --- /dev/null +++ b/extras/mount-shared-storage.sh @@ -0,0 +1,39 @@ +#!/bin/bash +#Post reboot there is a chance in which mounting of shared storage will fail +#This will impact starting of features like NFS-Ganesha. So this script will +#try to mount the shared storage if it fails + +exitStatus=0 + +while IFS= read -r glm +do + IFS=$' \t' read -r -a arr <<< "$glm" + + #Validate storage type is glusterfs + if [ "${arr[2]}" == "glusterfs" ] + then + + #check whether shared storage is mounted + #if it is mounted then mountpoint -q will return a 0 success code + if mountpoint -q "${arr[1]}" + then + echo "${arr[1]} is already mounted" + continue + fi + + mount -t glusterfs -o "${arr[3]}" "${arr[0]}" "${arr[1]}" + #wait for few seconds + sleep 10 + + #recheck mount got succeed + if mountpoint -q "${arr[1]}" + then + echo "${arr[1]} has been mounted" + continue + else + echo "${arr[1]} failed to mount" + exitStatus=1 + fi + fi +done <<< "$(sed '/^#/ d' </etc/fstab | grep 'glusterfs')" +exit $exitStatus diff --git a/extras/ocf/volume.in b/extras/ocf/volume.in index 72fd1213af2..76cc649e55f 100755 --- a/extras/ocf/volume.in +++ b/extras/ocf/volume.in @@ -6,6 +6,7 @@ # HA resource # # Authors: Florian Haas (hastexo Professional Services GmbH) +# Jiri Lunacek (Hosting90 Systems s.r.o.) # # License: GNU General Public License (GPL) @@ -54,6 +55,14 @@ must have clone ordering enabled. <shortdesc lang="en">gluster executable</shortdesc> <content type="string" default="$OCF_RESKEY_binary_default"/> </parameter> + <parameter name="peer_map"> + <longdesc lang="en"> + Mapping of hostname - peer name in the gluster cluster + in format hostname1:peername1,hostname2:peername2,... + </longdesc> + <shortdesc lang="en">gluster peer map</shortdesc> + <content type="string" default=""/> + </parameter> </parameters> <actions> <action name="start" timeout="20" /> @@ -68,9 +77,13 @@ EOF } +if [ -n "${OCF_RESKEY_peer_map}" ]; then + SHORTHOSTNAME=`echo "${OCF_RESKEY_peer_map}" | egrep -o "$SHORTHOSTNAME\:[^,]+" | awk -F: '{print $2}'` +fi + volume_getdir() { local voldir - voldir="@sysconfdir@/glusterd/vols/${OCF_RESKEY_volname}" + voldir="@GLUSTERD_WORKDIR@/vols/${OCF_RESKEY_volname}" [ -d ${voldir} ] || return 1 @@ -78,6 +91,16 @@ volume_getdir() { return 0 } +volume_getpid_dir() { + local volpid_dir + volpid_dir="/var/run/gluster/vols/${OCF_RESKEY_volname}" + + [ -d ${volpid_dir} ] || return 1 + + echo "${volpid_dir}" + return 0 +} + volume_getbricks() { local infofile local voldir @@ -92,17 +115,19 @@ volume_getbricks() { volume_getpids() { local bricks - local piddir local pidfile local infofile - local voldir + local volpid_dir - voldir=`volume_getdir` + volpid_dir=`volume_getpid_dir` bricks=`volume_getbricks` - piddir="${voldir}/run" + + if [ -z "$bricks" ]; then + return 1 + fi for brick in ${bricks}; do - pidfile="${piddir}/${SHORTHOSTNAME}${brick}.pid" + pidfile="${volpid_dir}/${SHORTHOSTNAME}${brick}.pid" [ -e $pidfile ] || return 1 cat $pidfile done @@ -206,6 +231,11 @@ volume_validate_all() { # Test for required binaries check_binary $OCF_RESKEY_binary + + if [ -z "$SHORTHOSTNAME" ]; then + ocf_log err 'Unable to get host in node map' + return $OCF_ERR_CONFIGURED + fi return $OCF_SUCCESS } diff --git a/extras/peer_add_secret_pub.in b/extras/peer_add_secret_pub.in new file mode 100644 index 00000000000..c9674af353d --- /dev/null +++ b/extras/peer_add_secret_pub.in @@ -0,0 +1,70 @@ +#!/bin/bash + +user=$1 +pub_file=$2 + +if [ "$user" == "" ]; then + echo "Invalid User"; + exit 1; +fi + +if [ "$pub_file" == "" ]; then + echo "Invalid pub file"; + exit 1; +fi + +home_dir=`getent passwd $user | cut -d ':' -f 6`; + +if [ "$home_dir" == "" ]; then + echo "Invalid home dir"; + exit 1; +fi + +authorized_keys_file=$(cat /etc/ssh/sshd_config | \ + grep -e "^AuthorizedKeysFile" | \ + awk '{print $2}' | tail -1); + +# If not set, use default location +if [ "x$authorized_keys_file" == "x" ]; then + authorized_keys_file="%h/.ssh/authorized_keys" +fi + +# If default location +if [ "$authorized_keys_file" == ".ssh/authorized_keys" ]; then + authorized_keys_file="%h/$authorized_keys_file" +fi + +# Replace %u with user name (ex: /etc/ssh/keys/%u/authorized_keys) +authorized_keys_file="${authorized_keys_file//%u/$user}"; + +# Replace %h with home dir (ex: %h/.ssh/authorized_keys) +authorized_keys_file="${authorized_keys_file//%h/$home_dir}"; +ssh_dir=$(dirname $authorized_keys_file); + +if [ ! -d $ssh_dir ]; then + mkdir $ssh_dir; + chmod 700 $ssh_dir; + chown $user: $ssh_dir; +fi + +if [ ! -d $authorized_keys_file ]; then + touch $authorized_keys_file; + chmod 600 $authorized_keys_file; + chown $user: $authorized_keys_file; +fi + +# Restore SELinux security contexts. This is required +# for passwdless SSH to work. + +if type restorecon >/dev/null 2>&1; then + restorecon -F $ssh_dir $authorized_keys_file; +fi + +# Add to authorized_keys file only if not exists already +while read line +do + grep -Fxq "$line" $authorized_keys_file; + [ $? -ne 0 ] && echo "$line" >> $authorized_keys_file; +done < "$GLUSTERD_WORKDIR"/$pub_file; + +exit 0; diff --git a/extras/profiler/glusterfs-profiler b/extras/profiler/glusterfs-profiler index 65d445864aa..aaafd088648 100755 --- a/extras/profiler/glusterfs-profiler +++ b/extras/profiler/glusterfs-profiler @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com> # This file is part of GlusterFS. @@ -291,7 +291,7 @@ class Texttable: s = "%s%s%s" % (horiz, [horiz, self._char_corner][self._has_vlines()], horiz) # build the line - l = string.join([horiz*n for n in self._width], s) + l = s.join([horiz*n for n in self._width]) # add border if needed if self._has_border(): l = "%s%s%s%s%s\n" % (self._char_corner, horiz, l, horiz, diff --git a/extras/prot_filter.py b/extras/prot_filter.py deleted file mode 100755 index 7dccacf155e..00000000000 --- a/extras/prot_filter.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/python - -""" - Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com> - This file is part of GlusterFS. - - This file is licensed to you under your choice of the GNU Lesser - General Public License, version 3 or any later version (LGPLv3 or - later), or the GNU General Public License, version 2 (GPLv2), in all - cases as published by the Free Software Foundation. -""" - -""" - INSTRUCTIONS - Put this in /usr/lib64/glusterfs/$version/filter to have it run automatically, - or else you'll have to run it by hand every time you change the volume - configuration. Give it a list of volume names on which to enable the - protection functionality; it will deliberately ignore client volfiles for - other volumes, and all server volfiles. It *will* include internal client - volfiles such as those used for NFS or rebalance/self-heal; this is a - deliberate choice so that it will catch deletions from those sources as well. -""" - -volume_list = [ "jdtest" ] - -import copy -import string -import sys -import types - -class Translator: - def __init__ (self, name): - self.name = name - self.xl_type = "" - self.opts = {} - self.subvols = [] - self.dumped = False - def __repr__ (self): - return "<Translator %s>" % self.name - -def load (path): - # If it's a string, open it; otherwise, assume it's already a - # file-like object (most notably from urllib*). - if type(path) in types.StringTypes: - fp = file(path,"r") - else: - fp = path - all_xlators = {} - xlator = None - last_xlator = None - while True: - text = fp.readline() - if text == "": - break - text = text.split() - if not len(text): - continue - if text[0] == "volume": - if xlator: - raise RuntimeError, "nested volume definition" - xlator = Translator(text[1]) - continue - if not xlator: - raise RuntimeError, "text outside volume definition" - if text[0] == "type": - xlator.xl_type = text[1] - continue - if text[0] == "option": - xlator.opts[text[1]] = string.join(text[2:]) - continue - if text[0] == "subvolumes": - for sv in text[1:]: - xlator.subvols.append(all_xlators[sv]) - continue - if text[0] == "end-volume": - all_xlators[xlator.name] = xlator - last_xlator = xlator - xlator = None - continue - raise RuntimeError, "unrecognized keyword %s" % text[0] - if xlator: - raise RuntimeError, "unclosed volume definition" - return all_xlators, last_xlator - -def generate (graph, last, stream=sys.stdout): - for sv in last.subvols: - if not sv.dumped: - generate(graph,sv,stream) - print >> stream, "" - sv.dumped = True - print >> stream, "volume %s" % last.name - print >> stream, " type %s" % last.xl_type - for k, v in last.opts.iteritems(): - print >> stream, " option %s %s" % (k, v) - if last.subvols: - print >> stream, " subvolumes %s" % string.join( - [ sv.name for sv in last.subvols ]) - print >> stream, "end-volume" - -def push_filter (graph, old_xl, filt_type, opts={}): - new_type = "-" + filt_type.split("/")[1] - old_type = "-" + old_xl.xl_type.split("/")[1] - pos = old_xl.name.find(old_type) - if pos >= 0: - new_name = old_xl.name - old_name = new_name[:pos] + new_type + new_name[len(old_type)+pos:] - else: - new_name = old_xl.name + old_type - old_name = old_xl.name + new_type - new_xl = Translator(new_name) - new_xl.xl_type = old_xl.xl_type - new_xl.opts = old_xl.opts - new_xl.subvols = old_xl.subvols - graph[new_xl.name] = new_xl - old_xl.name = old_name - old_xl.xl_type = filt_type - old_xl.opts = opts - old_xl.subvols = [new_xl] - graph[old_xl.name] = old_xl - -if __name__ == "__main__": - path = sys.argv[1] - # Alow an override for debugging. - for extra in sys.argv[2:]: - volume_list.append(extra) - graph, last = load(path) - for v in volume_list: - if graph.has_key(v): - break - else: - print "No configured volumes found - aborting." - sys.exit(0) - for v in graph.values(): - if v.xl_type == "cluster/distribute": - push_filter(graph,v,"features/prot_dht") - elif v.xl_type == "protocol/client": - push_filter(graph,v,"features/prot_client") - # We push debug/trace so that every fop gets a real frame, because DHT - # gets confused if STACK_WIND_TAIL causes certain fops to be invoked - # from anything other than a direct child. - for v in graph.values(): - if v.xl_type == "features/prot_client": - push_filter(graph,v,"debug/trace") - generate(graph,last,stream=open(path,"w")) diff --git a/extras/python/Makefile.am b/extras/python/Makefile.am new file mode 100644 index 00000000000..7d81fa0319b --- /dev/null +++ b/extras/python/Makefile.am @@ -0,0 +1,7 @@ +if HAVE_PYTHON +# Install __init__.py into the Python site-packages area +pypkgdir = @BUILD_PYTHON_SITE_PACKAGES@/gluster +pypkg_PYTHON = __init__.py +endif + +EXTRA_DIST = __init__.py diff --git a/extras/python/__init__.py b/extras/python/__init__.py new file mode 100644 index 00000000000..3ad9513f40e --- /dev/null +++ b/extras/python/__init__.py @@ -0,0 +1,2 @@ +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/extras/contri-add.sh b/extras/quota/contri-add.sh index 7db5edd5d20..7db5edd5d20 100755 --- a/extras/contri-add.sh +++ b/extras/quota/contri-add.sh diff --git a/extras/quota/log_accounting.sh b/extras/quota/log_accounting.sh new file mode 100755 index 00000000000..e2dd87b84d7 --- /dev/null +++ b/extras/quota/log_accounting.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# The script does an accounting of all directories using command 'du' and +# using gluster. We can then compare the two to identify accounting mismatch +# THere can be minor mismatch because gluster only accounts for the size of +# files. Direcotries can take up upto 4kB space on FS per directory. THis +# size is accounted by du and not by gluster. However the difference would +# not be significant. + +mountpoint=$1 +volname=$2 + +usage () +{ + echo >&2 "usage: $0 <mountpoint> <volume name>" + exit +} + +[ $# -lt 2 ] && usage + +cd $mountpoint +du -h | head -n -1 | tr -d '.' |awk '{ for (i = 2; i <= NF; i++) { printf("%s ", $i);} print "" }' > /tmp/gluster_quota_1 +cat /tmp/gluster_quota_1 | sed 's/ $//' | sed 's/ /\\ /g' | sed 's/(/\\(/g' | sed 's/)/\\)/g' |xargs gluster v quota $volname list > /tmp/gluster_quota_2 +du -h | head -n -1 |awk '{ for (i = 2; i <= NF; i++) { printf("%s %s", $i, $1);} print "" }' | tr -d '.' > /tmp/gluster_quota_3 +cat /tmp/gluster_quota_2 /tmp/gluster_quota_3 | sort > /tmp/gluster_quota_4 +find . -type d > /tmp/gluster_quota_5 +tar -cvf /tmp/gluster_quota_files.tar /tmp/gluster_quota_* diff --git a/extras/quota/quota_fsck.py b/extras/quota/quota_fsck.py new file mode 100755 index 00000000000..e62f7fc52a3 --- /dev/null +++ b/extras/quota/quota_fsck.py @@ -0,0 +1,377 @@ +#!/usr/bin/python3 +# The following script enables, Detecting, Reporting and Fixing +# anomalies in quota accounting. Run this script with -h option +# for further details. + +''' + Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +''' +from __future__ import print_function +import os, sys, re +from stat import * +import subprocess +import argparse +import xattr + +aggr_size = {} +verbose_mode = False +mnt_path = None +brick_path = None +obj_fix_count = 0 +file_count = 0 +dir_count = 0 + +#CONSTANTS +KB = 1024 +MB = 1048576 +GB = 1048576 * 1024 +TB = 1048576 * 1048576 + +QUOTA_VERBOSE = 0 +QUOTA_META_ABSENT = 1 +QUOTA_SIZE_MISMATCH = 2 + +IS_DIRTY ='0x3100' +IS_CLEAN ='0x3000' + + +epilog_msg=''' + The script attempts to find any gluster accounting issues in the + filesystem at the given subtree. The script crawls the given + subdirectory tree doing a stat for all files and compares the + size reported by gluster quota with the size reported by stat + calls. Any mismatch is reported. In addition integrity of marker + xattrs are verified. + ''' + +def print_msg(log_type, path, xattr_dict = {}, stbuf = "", dir_size = None): + if log_type == QUOTA_VERBOSE: + print('%-24s %-60s\nxattr_values: %s\n%s\n' % ("Verbose", path, xattr_dict, stbuf)) + elif log_type == QUOTA_META_ABSENT: + print('%-24s %-60s\n%s\n' % ("Quota-Meta Absent", path, xattr_dict)) + elif log_type == QUOTA_SIZE_MISMATCH: + print("mismatch") + if dir_size is not None: + print('%24s %60s %12s %12s' % ("Size Mismatch", path, + xattr_dict, dir_size)) + else: + print('%-24s %-60s %-12s %-12s' % ("Size Mismatch", path, xattr_dict, + stbuf.st_size)) + +def size_differs_lot(s1, s2): + ''' + There could be minor accounting differences between the stat based + accounting and gluster accounting. To avoid these from throwing lot + of false positives in our logs. using a threshold of 1M for now. + TODO: For a deeply nested directory, at higher levels in hierarchy + differences may not be significant, hence this check needs to be improved. + ''' + if abs(s1-s2) > 0: + return True + else: + return False + +def fix_hardlink_accounting(curr_dict, accounted_dict, curr_size): + ''' + Hard links are messy.. we have to account them for their parent + directory. But, stop accounting at the most common ancestor. + Eg: + say we have 3 hardlinks : /d1/d2/h1, /d1/d3/h2 and /d1/h3 + + suppose we encounter the hard links h1 first , then h2 and then h3. + while accounting for h1, we account the size until root(d2->d1->/) + while accounting for h2, we need to account only till d3. (as d1 + and / are accounted for this inode). + while accounting for h3 we should not account at all.. as all + its ancestors are already accounted for same inode. + + curr_dict : dict of hardlinks that were seen and + accounted by the current iteration. + accounted_dict : dict of hardlinks that has already been + accounted for. + + size : size of the object as accounted by the + curr_iteration. + + Return vale: + curr_size : size reduced by hardlink sizes for those + hardlinks that has already been accounted + in current subtree. + Also delete the duplicate link from curr_dict. + ''' + + dual_accounted_links = set(curr_dict.keys()) & set(accounted_dict.keys()) + for link in dual_accounted_links: + curr_size = curr_size - curr_dict[link] + del curr_dict[link] + return curr_size + + +def fix_xattr(file_name, mark_dirty): + global obj_fix_count + global mnt_path + + if mnt_path is None: + return + if mark_dirty: + print("MARKING DIRTY: " + file_name) + out = subprocess.check_output (["/usr/bin/setfattr", "-n", + "trusted.glusterfs.quota.dirty", + "-v", IS_DIRTY, file_name]) + rel_path = os.path.relpath(file_name, brick_path) + print("stat on " + mnt_path + "/" + rel_path) + stbuf = os.lstat(mnt_path + "/" + rel_path) + + obj_fix_count += 1 + +def get_quota_xattr_brick(dpath): + out = subprocess.check_output (["/usr/bin/getfattr", "--no-dereference", + "-d", "-m.", "-e", "hex", dpath]) + pairs = out.splitlines() + + ''' + Sample output to be parsed: + [root@dhcp35-100 mnt]# getfattr -d -m. -e hex /export/b1/B0/d14/d13/ + # file: export/b1/B0/d14/d13/ + security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a7573725f743a733000 + trusted.gfid=0xbae5e0d2d05043de9fd851d91ecf63e8 + trusted.glusterfs.dht=0x000000010000000000000000ffffffff + trusted.glusterfs.dht.mds=0x00000000 + trusted.glusterfs.quota.6a7675a3-b85a-40c5-830b-de9229d702ce.contri.39=0x00000000000000000000000000000000000000000000000e + trusted.glusterfs.quota.dirty=0x3000 + trusted.glusterfs.quota.size.39=0x00000000000000000000000000000000000000000000000e + ''' + + ''' + xattr_dict dictionary holds quota related xattrs + eg: + ''' + + xattr_dict = {} + xattr_dict['parents'] = {} + + for xattr in pairs[1:]: + xattr = xattr.decode("utf-8") + xattr_key = xattr.split("=")[0] + if xattr_key == "": + # skip any empty lines + continue + elif not re.search("quota", xattr_key): + # skip all non quota xattr. + continue + + xattr_value = xattr.split("=")[1] + if re.search("contri", xattr_key): + + xattr_version = xattr_key.split(".")[5] + if 'version' not in xattr_dict: + xattr_dict['version'] = xattr_version + else: + if xattr_version != xattr_dict['version']: + print("Multiple xattr version found") + + + cur_parent = xattr_key.split(".")[3] + if cur_parent not in xattr_dict['parents']: + xattr_dict['parents'][cur_parent] = {} + + contri_dict = xattr_dict['parents'][cur_parent] + if len(xattr_value) == 34: + # 34 bytes implies file contri xattr + # contri format =0x< 16bytes file size><16bytes file count> + # size is obtained in iatt, file count = 1, dir count=0 + contri_dict['contri_size'] = int(xattr_value[2:18], 16) + contri_dict['contri_file_count'] = int(xattr_value[18:34], 16) + contri_dict['contri_dir_count'] = 0 + else: + # This is a directory contri. + contri_dict['contri_size'] = int(xattr_value[2:18], 16) + contri_dict['contri_file_count'] = int(xattr_value[18:34], 16) + contri_dict['contri_dir_count'] = int(xattr_value[34:], 16) + + elif re.search("size", xattr_key): + xattr_dict['size'] = int(xattr_value[2:18], 16) + xattr_dict['file_count'] = int(xattr_value[18:34], 16) + xattr_dict['dir_count'] = int(xattr_value[34:], 16) + elif re.search("dirty", xattr_key): + if xattr_value == IS_CLEAN: + xattr_dict['dirty'] = False + elif xattr_value == IS_DIRTY: + xattr_dict['dirty'] = True + elif re.search("limit_objects", xattr_key): + xattr_dict['limit_objects'] = int(xattr_value[2:18], 16) + elif re.search("limit_set", xattr_key): + xattr_dict['limit_set'] = int(xattr_value[2:18], 16) + + return xattr_dict + +def verify_file_xattr(path, stbuf = None): + + global file_count + file_count += 1 + + if stbuf is None: + stbuf = os.lstat(path) + + xattr_dict = get_quota_xattr_brick(path) + + for parent in xattr_dict['parents']: + contri_dict = xattr_dict['parents'][parent] + + if 'contri_size' not in contri_dict or \ + 'contri_file_count' not in contri_dict or \ + 'contri_dir_count' not in contri_dict: + print_msg(QUOTA_META_ABSENT, path, xattr_dict, stbuf) + fix_xattr(path, False) + return + elif size_differs_lot(contri_dict['contri_size'], stbuf.st_size): + print_msg(QUOTA_SIZE_MISMATCH, path, xattr_dict, stbuf) + fix_xattr(path, False) + return + + if verbose_mode is True: + print_msg(QUOTA_VERBOSE, path, xattr_dict, stbuf) + + +def verify_dir_xattr(path, dir_size): + + global dir_count + dir_count += 1 + xattr_dict = get_quota_xattr_brick(path) + + stbuf = os.lstat(path) + + for parent in xattr_dict['parents']: + contri_dict = xattr_dict['parents'][parent] + + if 'size' not in xattr_dict or 'contri_size' not in contri_dict: + print_msg(QUOTA_META_ABSENT, path) + fix_xattr(path, True) + return + elif size_differs_lot(dir_size, xattr_dict['size']) or \ + size_differs_lot(contri_dict['contri_size'], xattr_dict['size']): + print_msg(QUOTA_SIZE_MISMATCH, path, xattr_dict, stbuf, dir_size) + fix_xattr(path, True) + return + + if verbose_mode is True: + print_msg("VERBOSE", path, xattr_dict, stbuf, dir_size) + + +def walktree(t_dir, hard_link_dict): + '''recursively descend the directory tree rooted at dir, + aggregating the size + t_dir : directory to walk over. + hard_link_dict : dict of inodes with multiple hard_links under t_dir + ''' + global aggr_size + aggr_size[t_dir] = 0 + + for entry in os.listdir(t_dir): + pathname = os.path.join(t_dir, entry) + stbuf = os.lstat(pathname) + if S_ISDIR(stbuf.st_mode): + # It's a directory, recurse into it + if entry == '.glusterfs': + print("skipping " + pathname) + continue + descendent_hardlinks = {} + subtree_size = walktree(pathname, descendent_hardlinks) + + subtree_size = fix_hardlink_accounting(descendent_hardlinks, + hard_link_dict, + subtree_size) + + aggr_size[t_dir] = aggr_size[t_dir] + subtree_size + + elif S_ISREG(stbuf.st_mode) or S_ISLNK(stbuf.st_mode): + # Even a symbolic link file may have multiple hardlinks. + + file_size = stbuf.st_size + if stbuf.st_nlink > 2: + # send a single element dict to check if file is accounted. + file_size = fix_hardlink_accounting({stbuf.st_ino:stbuf.st_size}, + hard_link_dict, + stbuf.st_size) + + if file_size == 0: + print_msg("HARD_LINK (skipped)", pathname, "", + stbuf) + else: + print_msg("HARD_LINK (accounted)", pathname, "", + stbuf) + hard_link_dict[stbuf.st_ino] = stbuf.st_size + + if t_dir in aggr_size: + aggr_size[t_dir] = aggr_size[t_dir] + file_size + else: + aggr_size[t_dir] = file_size + verify_file_xattr(pathname, stbuf) + + else: + # Unknown file type, print a message + print('Skipping %s, due to file mode' % (pathname)) + + if t_dir not in aggr_size: + aggr_size[t_dir] = 0 + + verify_dir_xattr(t_dir, aggr_size[t_dir]) + # du also accounts for t_directory sizes + # aggr_size[t_dir] += 4096 + + #cleanup + ret = aggr_size[t_dir] + del aggr_size[t_dir] + return ret + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Diagnose quota accounting issues.', epilog=epilog_msg) + parser.add_argument('brick_path', nargs=1, + help='The brick path (or any descendent sub-directory of brick path)', + ) + parser.add_argument('--full-logs', dest='verbose', action='store_true', + help=''' + log all the xattr values and stat values reported + for analysis. [CAUTION: This can give lot of output + depending on FS depth. So one has to make sure enough + disk space exists if redirecting to file] + ''' + ) + parser.add_argument('--fix-issues', metavar='mount_path', dest='mnt', action='store', + help=''' + fix accounting issues where the xattr values disagree + with stat sizes reported by gluster. A mount is also + required for this option to be used. + [CAUTION: This will directly modify backend xattr] + ''' + ) + parser.add_argument('--sub-dir', metavar='sub_dir', dest='sub_dir', action='store', + help=''' + limit the crawling and accounting verification/correction + to a specific subdirectory. + ''' + ) + + args = parser.parse_args() + verbose_mode = args.verbose + brick_path = args.brick_path[0] + sub_dir = args.sub_dir + mnt_path = args.mnt + hard_link_dict = {} + if sub_dir is not None: + walktree(os.path.join(brick_path, sub_dir), hard_link_dict) + else: + walktree(brick_path, hard_link_dict) + + print("Files verified : " + str(file_count)) + print("Directories verified : " + str(dir_count)) + if mnt_path is not None: + print("Objects Fixed : " + str(obj_fix_count)) diff --git a/extras/quota/xattr_analysis.py b/extras/quota/xattr_analysis.py new file mode 100755 index 00000000000..7bd7d96374c --- /dev/null +++ b/extras/quota/xattr_analysis.py @@ -0,0 +1,73 @@ +#!/usr/bin/python3 +# Below script has two purposes +# 1. Display xattr of entire FS tree in a human readable form +# 2. Display all the directory where contri and size mismatch. +# (If there are any directory with contri and size mismatch that are not dirty +# then that highlights a propagation issue) +# The script takes only one input LOG _FILE generated from the command, +# find <brick_path> | xargs getfattr -d -m. -e hex > log_gluster_xattr + +from __future__ import print_function +import re +import subprocess +import sys +from hurry.filesize import size + +if len(sys.argv) < 2: + sys.exit('Usage: %s log_gluster_xattr \n' + 'to generate log_gluster_xattr use: \n' + 'find <brick_path> | xargs getfattr -d -m. -e hex > log_gluster_xattr' + % sys.argv[0]) +LOG_FILE=sys.argv[1] + +def get_quota_xattr_brick(): + out = subprocess.check_output (["/usr/bin/cat", LOG_FILE]) + pairs = out.splitlines() + + xdict = {} + mismatch_size = [('====contri_size===', '====size====')] + for xattr in pairs: + k = xattr.split("=")[0] + if re.search("# file:", k): + print(xdict) + filename=k + print("=====" + filename + "=======") + xdict = {} + elif k is "": + pass + else: + print(xattr) + v = xattr.split("=")[1] + if re.search("contri", k): + if len(v) == 34: + # for files size is obtained in iatt, file count should be 1, dir count=0 + xdict['contri_file_count'] = int(v[18:34], 16) + xdict['contri_dir_count'] = 0 + else: + xdict['contri_size'] = size(int(v[2:18], 16)) + xdict['contri_file_count'] = int(v[18:34], 16) + xdict['contri_dir_count'] = int(v[34:], 16) + elif re.search("size", k): + xdict['size'] = size(int(v[2:18], 16)) + xdict['file_count'] = int(v[18:34], 16) + xdict['dir_count'] = int(v[34:], 16) + elif re.search("dirty", k): + if v == '0x3000': + xdict['dirty'] = False + elif v == '0x3100': + xdict['dirty'] = True + elif re.search("limit_objects", k): + xdict['limit_objects'] = int(v[2:18], 16) + elif re.search("limit_set", k): + xdict['limit_set'] = size(int(v[2:18], 16)) + + if 'size' in xdict and 'contri_size' in xdict and xdict['size'] != xdict['contri_size']: + mismatch_size.append((xdict['contri_size'], xdict['size'], filename)) + + for values in mismatch_size: + print(values) + + +if __name__ == '__main__': + get_quota_xattr_brick() + diff --git a/extras/rebalance.py b/extras/rebalance.py index 80c614c5dfe..37c68ebbb42 100755 --- a/extras/rebalance.py +++ b/extras/rebalance.py @@ -1,4 +1,6 @@ -#!/usr/bin/python +#!/usr/bin/python3 + +from __future__ import print_function import atexit import copy @@ -11,6 +13,7 @@ import subprocess import sys import tempfile import volfilter +import platform # It's just more convenient to have named fields. class Brick: @@ -37,20 +40,20 @@ class Brick: def get_bricks (host, vol): t = pipes.Template() - t.prepend("gluster --remote-host=%s system getspec %s"%(host,vol),".-") - return t.open(None,"r") + t.prepend("gluster --remote-host=%s system getspec %s"%(host, vol), ".-") + return t.open(None, "r") def generate_stanza (vf, all_xlators, cur_subvol): sv_list = [] for sv in cur_subvol.subvols: - generate_stanza(vf,all_xlators,sv) + generate_stanza(vf, all_xlators, sv) sv_list.append(sv.name) - vf.write("volume %s\n"%cur_subvol.name) - vf.write(" type %s\n"%cur_subvol.type) - for kvpair in cur_subvol.opts.iteritems(): - vf.write(" option %s %s\n"%kvpair) + vf.write("volume %s\n" % cur_subvol.name) + vf.write(" type %s\n" % cur_subvol.type) + for kvpair in cur_subvol.opts.items(): + vf.write(" option %s %s\n" % kvpair) if sv_list: - vf.write(" subvolumes %s\n"%string.join(sv_list)) + vf.write(" subvolumes %s\n" % ''.join(sv_list)) vf.write("end-volume\n\n") @@ -58,14 +61,14 @@ def mount_brick (localpath, all_xlators, dht_subvol): # Generate a volfile. vf_name = localpath + ".vol" - vf = open(vf_name,"w") - generate_stanza(vf,all_xlators,dht_subvol) + vf = open(vf_name, "w") + generate_stanza(vf, all_xlators, dht_subvol) vf.flush() vf.close() # Create a brick directory and mount the brick there. os.mkdir(localpath) - subprocess.call(["glusterfs","-f",vf_name,localpath]) + subprocess.call(["glusterfs", "-f", vf_name, localpath]) # We use the command-line tools because there's no getxattr support in the # Python standard library (which is ridiculous IMO). Adding the xattr package @@ -79,16 +82,16 @@ def mount_brick (localpath, all_xlators, dht_subvol): def get_range (brick): t = pipes.Template() cmd = "getfattr -e hex -n trusted.glusterfs.dht %s 2> /dev/null" - t.prepend(cmd%brick,".-") - t.append("grep ^trusted.glusterfs.dht=","--") - f = t.open(None,"r") + t.prepend(cmd%brick, ".-") + t.append("grep ^trusted.glusterfs.dht=", "--") + f = t.open(None, "r") try: value = f.readline().rstrip().split('=')[1][2:] except: - print "could not get layout for %s (might be OK)" % brick + print("could not get layout for %s (might be OK)" % brick) return None - v_start = int("0x"+value[16:24],16) - v_end = int("0x"+value[24:32],16) + v_start = int("0x"+value[16:24], 16) + v_end = int("0x"+value[24:32], 16) return (v_start, v_end) def calc_sizes (bricks, total): @@ -125,7 +128,7 @@ def normalize (in_bricks): curr_hash = b.r_end + 1 break else: - print "gap found at 0x%08x" % curr_hash + print("gap found at 0x%08x" % curr_hash) sys.exit(1) return out_bricks + in_bricks, used @@ -153,8 +156,8 @@ def get_score (bricks): if __name__ == "__main__": - my_usage = "%prog [options] server volume [directory]" - parser = optparse.OptionParser(usage=my_usage) + my_usage = "%prog [options] server volume [directory]" + parser = optparse.OptionParser(usage=my_usage) parser.add_option("-f", "--free-space", dest="free_space", default=False, action="store_true", help="use free space instead of total space") @@ -164,7 +167,7 @@ if __name__ == "__main__": parser.add_option("-v", "--verbose", dest="verbose", default=False, action="store_true", help="verbose output") - options, args = parser.parse_args() + options, args = parser.parse_args() if len(args) == 3: fix_dir = args[2] @@ -182,9 +185,9 @@ if __name__ == "__main__": def cleanup_workdir (): os.chdir(orig_dir) if options.verbose: - print "Cleaning up %s" % work_dir + print("Cleaning up %s" % work_dir) for b in bricks: - subprocess.call(["umount",b.path]) + subprocess.call(["umount", b.path]) shutil.rmtree(work_dir) if not options.leave_mounted: atexit.register(cleanup_workdir) @@ -192,44 +195,51 @@ if __name__ == "__main__": # Mount each brick individually, so we can issue brick-specific calls. if options.verbose: - print "Mounting subvolumes..." + print("Mounting subvolumes...") index = 0 - volfile_pipe = get_bricks(hostname,volname) + volfile_pipe = get_bricks(hostname, volname) all_xlators, last_xlator = volfilter.load(volfile_pipe) for dht_vol in all_xlators.itervalues(): if dht_vol.type == "cluster/distribute": break else: - print "no DHT volume found" + print("no DHT volume found") sys.exit(1) for sv in dht_vol.subvols: #print "found subvol %s" % sv.name lpath = "%s/brick%s" % (work_dir, index) index += 1 - mount_brick(lpath,all_xlators,sv) - bricks.append(Brick(lpath,sv.name)) + mount_brick(lpath, all_xlators, sv) + bricks.append(Brick(lpath, sv.name)) if index == 0: - print "no bricks" + print("no bricks") sys.exit(1) # Collect all of the sizes. if options.verbose: - print "Collecting information..." + print("Collecting information...") total = 0 for b in bricks: info = os.statvfs(b.path) + # On FreeBSD f_bsize (info[0]) contains the optimal I/O size, + # not the block size as it's found on Linux. In this case we + # use f_frsize (info[1]). + if platform.system() == 'FreeBSD': + bsize = info[1] + else: + bsize = info[0] # We want a standard unit even if different bricks use # different block sizes. The size is chosen to avoid overflows # for very large bricks with very small block sizes, but also # accommodate filesystems which use very large block sizes to # cheat on benchmarks. - blocksper100mb = 104857600 / info[0] + blocksper100mb = 104857600 / bsize if options.free_space: size = info[3] / blocksper100mb else: size = info[2] / blocksper100mb if size <= 0: - print "brick %s has invalid size %d" % (b.path, size) + print("brick %s has invalid size %d" % (b.path, size)) sys.exit(1) b.set_size(size) total += size @@ -240,13 +250,13 @@ if __name__ == "__main__": if hash_range is not None: rs, re = hash_range if rs > re: - print "%s has backwards hash range" % b.path + print("%s has backwards hash range" % b.path) sys.exit(1) - b.set_range(hash_range[0],hash_range[1]) + b.set_range(hash_range[0], hash_range[1]) if options.verbose: - print "Calculating new layouts..." - calc_sizes(bricks,total) + print("Calculating new layouts...") + calc_sizes(bricks, total) bricks, used = normalize(bricks) # We can't afford O(n!) here, but O(n^2) should be OK and the result @@ -254,10 +264,10 @@ if __name__ == "__main__": while used < len(bricks): best_place = used best_score = get_score(bricks) - for i in xrange(used): + for i in range(used): new_bricks = bricks[:] del new_bricks[used] - new_bricks.insert(i,bricks[used]) + new_bricks.insert(i, bricks[used]) new_score = get_score(new_bricks) if new_score > best_score: best_place = i @@ -265,7 +275,7 @@ if __name__ == "__main__": if best_place != used: nb = bricks[used] del bricks[used] - bricks.insert(best_place,nb) + bricks.insert(best_place, nb) used += 1 # Finalize whatever we decided on. @@ -275,25 +285,25 @@ if __name__ == "__main__": curr_hash += b.good_size b.r_end = curr_hash - 1 - print "Here are the xattr values for your size-weighted layout:" + print("Here are the xattr values for your size-weighted layout:") for b in bricks: - print " %s: 0x0000000200000000%08x%08x" % ( - b.sv_name, b.r_start, b.r_end) + print(" %s: 0x0000000200000000%08x%08x" % ( + b.sv_name, b.r_start, b.r_end)) if fix_dir: if options.verbose: - print "Fixing layout for %s" % fix_dir + print("Fixing layout for %s" % fix_dir) for b in bricks: value = "0x0000000200000000%08x%08x" % ( b.r_start, b.r_end) path = "%s/%s" % (b.path, fix_dir) cmd = "setfattr -n trusted.glusterfs.dht -v %s %s" % ( value, path) - print cmd + print(cmd) if options.leave_mounted: - print "The following subvolumes are still mounted:" + print("The following subvolumes are still mounted:") for b in bricks: - print "%s on %s" % (b.sv_name, b.path) - print "Don't forget to clean up when you're done." + print("%s on %s" % (b.sv_name, b.path)) + print("Don't forget to clean up when you're done.") diff --git a/extras/run-gluster.tmpfiles.in b/extras/run-gluster.tmpfiles.in new file mode 100644 index 00000000000..329f2dde6db --- /dev/null +++ b/extras/run-gluster.tmpfiles.in @@ -0,0 +1,2 @@ +# hardcoding /run for now, should be detected while building from source? +d /run/gluster 0775 gluster gluster - diff --git a/extras/snap_scheduler/Makefile.am b/extras/snap_scheduler/Makefile.am new file mode 100644 index 00000000000..782f139016f --- /dev/null +++ b/extras/snap_scheduler/Makefile.am @@ -0,0 +1,9 @@ +snap_schedulerdir = $(sbindir)/ + +if WITH_SERVER +snap_scheduler_SCRIPTS = gcron.py snap_scheduler.py conf.py +endif + +EXTRA_DIST = gcron.py snap_scheduler.py conf.py + +CLEANFILES = diff --git a/extras/snap_scheduler/README.md b/extras/snap_scheduler/README.md new file mode 100644 index 00000000000..1316bb76469 --- /dev/null +++ b/extras/snap_scheduler/README.md @@ -0,0 +1,125 @@ +Snapshot Scheduler +============================== + +SUMMARY +------- + +GlusterFS volume snapshot provides point-in-time copy of a GlusterFS volume. Currently, GlusterFS volume snapshots can be easily scheduled by setting up cron jobs on one of the nodes in the GlusterFS trusted storage pool. This has a single point failure (SPOF), as scheduled jobs can be missed if the node running the cron jobs dies. + +We can avoid the SPOF by distributing the cron jobs to all nodes of the trusted storage pool. + +DETAILED DESCRIPTION +-------------------- + +The solution to the above problems involves the usage of: + +* A shared storage - This can be any shared storage (another gluster volume, a NFS mount, etc.) that will be used to share the schedule configuration and will help in the coordination of the jobs. +* An agent - This agent will perform the actual snapshot commands, instead of cron. It will contain the logic to perform coordinated snapshots. +* A helper script - This script will allow the user to initialise the scheduler on the local node, enable/disable scheduling, add/edit/list/delete snapshot schedules. +* cronie - It is the default cron daemon shipped with RHEL. It invokes the agent at the appropriate intervals as mentioned by the user to perform the snapshot operation on the volume as mentioned by the user in the schedule. + +INITIAL SETUP +------------- + +The administrator needs to create a shared storage that can be available to nodes across the cluster. A GlusterFS volume can also be used for the same. It is preferable that the *shared volume* be a replicate volume to avoid SPOF. + +Once the shared storage is created, it should be mounted on all nodes in the trusted storage pool which will be participating in the scheduling. The location where the shared_storage should be mounted (/var/run/gluster/snaps/shared_storage) in these nodes is fixed and is not configurable. Each node participating in the scheduling then needs to perform an initialisation of the snapshot scheduler by invoking the following: + +snap_scheduler.py init + +NOTE: This command needs to be run on all the nodes participating in the scheduling + +HELPER SCRIPT +------------- + +The helper script(snap_scheduler.py) will initialise the scheduler on the local node, enable/disable scheduling, add/edit/list/delete snapshot schedules. + +a) snap_scheduler.py init + +This command initialises the snap_scheduler and interfaces it with the crond running on the local node. This is the first step, before executing any scheduling related commands from a node. + +NOTE: The helper script needs to be run with this option on all the nodes participating in the scheduling. Other options of the helper script can be run independently from any node, where initialisation has been successfully completed. + +b) snap_scheduler.py enable + +The snap scheduler is disabled by default after initialisation. This command enables the snap scheduler. + +c) snap_scheduler.py disable + +This command disables the snap scheduler. + +d) snap_scheduler.py status + +This command displays the current status(Enabled/Disabled) of the snap scheduler. + +e) snap_scheduler.py add "Job Name" "Schedule" "Volume Name" + +This command adds a new snapshot schedule. All the arguments must be provided within double-quotes(""). It takes three arguments: + +-> Job Name: This name uniquely identifies this particular schedule, and can be used to reference this schedule for future events like edit/delete. If a schedule already exists for the specified Job Name, the add command will fail. + +-> Schedule: The schedules are accepted in the format crond understands:- + +Example of job definition: +.---------------- minute (0 - 59) +| .------------- hour (0 - 23) +| | .---------- day of month (1 - 31) +| | | .------- month (1 - 12) OR jan,feb,mar,apr ... +| | | | .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat +| | | | | +* * * * * user-name command to be executed +Although we accept all valid cron schedules, currently we support granularity of snapshot schedules to a maximum of half-hourly snapshots. + +-> Volume Name: The name of the volume on which the scheduled snapshot operation will be performed. + +f) snap_scheduler.py edit "Job Name" "Schedule" "Volume Name" + +This command edits an existing snapshot schedule. It takes the same three arguments that the add option takes. All the arguments must be provided within double-quotes(""). If a schedule does not exists for the specified Job Name, the edit command will fail. + +g) snap_scheduler.py delete "Job Name" + +This command deletes an existing snapshot schedule. It takes the job name of the schedule as argument. The argument must be provided within double-quotes(""). If a schedule does not exists for the specified Job Name, the delete command will fail. + +h) snap_scheduler.py list + +This command lists the existing snapshot schedules in the following manner: Pseudocode: + +# snap_scheduler.py list +JOB_NAME SCHEDULE OPERATION VOLUME NAME +-------------------------------------------------------------------- +Job0 * * * * * Snapshot Create test_vol + +THE AGENT +--------- + +The snapshots scheduled with the help of the helper script, are read by crond which then invokes the agent(gcron.py) at the scheduled intervals to perform the snapshot operations on the specified volumes. It then performs the scheduled snapshots using the following algorithm to coordinate. + +start_time = get current time +lock_file = job_name passed as an argument +vol_name = volume name psased as an argument +try POSIX locking the $lock_file + if lock is obtained, then + mod_time = Get modification time of $entry + if $mod_time < $start_time, then + Take snapshot of $entry.name (Volume name) + if snapshot failed, then + log the failure + Update modification time of $entry to current time + unlock the $entry + +The coordination with other scripts running on other nodes, is handled by the use of POSIX locks. All the instances of the script will attempt to lock the lock_file which is essentialy an empty file with the job name, and one which gets the lock will take the snapshot. + +To prevent redoing a done task, the script will make use of the mtime attribute of the entry. At the beginning execution, the script would have saved its start time. Once the script obtains the lock on the lock_file, before taking the snapshot, it compares the mtime of the entry with the start time. The snapshot will only be taken if the mtime is smaller than start time. Once the snapshot command completes, the script will update the mtime of the lock_file to the current time before unlocking. + +If a snapshot command fails, the script will log the failure (in syslog) and continue with its operation. It will not attempt to retry the failed snapshot in the current schedule, but will attempt it again in the next schedules. It is left to the administrator to monitor the logs and decide what to do after a failure. + +ASSUMPTIONS AND LIMITATIONS +--------------------------- + +It is assumed that all nodes in the have their times synced using NTP or any other mechanism. This is a hard requirement for this feature to work. + +The administrator needs to have python2.7 or higher installed, as well as the argparse module installed, to be able to use the helper script(snap_scheduler.py). + +There is a latency of one minute, between providing a command by the helper script and that command taking effect. Hence, currently we do not support snapshot schedules with per minute granularity. + +The administrator can however leverage the scheduler to schedule snapshots with granularity of half-hourly/hourly/daily/weekly/monthly/yearly periodic intervals. They can also schedule snapshots, which are customised mentioning which minute of the hour, which day of the week, which week of the month, and which month of the year, they want to schedule the snapshot operation. diff --git a/extras/snap_scheduler/conf.py.in b/extras/snap_scheduler/conf.py.in new file mode 100644 index 00000000000..6dcca0534a7 --- /dev/null +++ b/extras/snap_scheduler/conf.py.in @@ -0,0 +1,11 @@ +# +# Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. + +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. +# + +GLUSTERFS_LIBEXECDIR = '@GLUSTERFS_LIBEXECDIR@' diff --git a/extras/snap_scheduler/gcron.py b/extras/snap_scheduler/gcron.py new file mode 100755 index 00000000000..0e4df77d481 --- /dev/null +++ b/extras/snap_scheduler/gcron.py @@ -0,0 +1,190 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +from __future__ import print_function +import subprocess +import os +import os.path +import sys +import time +import logging +import logging.handlers +import fcntl + + +GCRON_TASKS = "/run/gluster/shared_storage/snaps/glusterfs_snap_cron_tasks" +GCRON_CROND_TASK = "/etc/cron.d/glusterfs_snap_cron_tasks" +GCRON_RELOAD_FLAG = "/var/run/gluster/crond_task_reload_flag" +LOCK_FILE_DIR = "/run/gluster/shared_storage/snaps/lock_files/" +log = logging.getLogger("gcron-logger") +start_time = 0.0 + + +def initLogger(script_name): + log.setLevel(logging.DEBUG) + logFormat = "[%(asctime)s %(filename)s:%(lineno)s %(funcName)s] "\ + "%(levelname)s %(message)s" + formatter = logging.Formatter(logFormat) + + sh = logging.handlers.SysLogHandler() + sh.setLevel(logging.ERROR) + sh.setFormatter(formatter) + + process = subprocess.Popen(["gluster", "--print-logdir"], + stdout=subprocess.PIPE, + universal_newlines=True) + out, err = process.communicate() + if process.returncode == 0: + logfile = os.path.join(out.strip(), script_name[:-3]+".log") + + fh = logging.FileHandler(logfile) + fh.setLevel(logging.DEBUG) + fh.setFormatter(formatter) + + log.addHandler(sh) + log.addHandler(fh) + + +def takeSnap(volname="", snapname=""): + success = True + if volname == "": + log.debug("No volname given") + return False + if snapname == "": + log.debug("No snapname given") + return False + + cli = ["gluster", + "snapshot", + "create", + snapname, + volname] + log.debug("Running command '%s'", " ".join(cli)) + + p = subprocess.Popen(cli, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate() + rv = p.returncode + + log.debug("Command '%s' returned '%d'", " ".join(cli), rv) + + if rv: + log.error("Snapshot of %s failed", volname) + log.error("Command output:") + log.error(err) + success = False + else: + log.info("Snapshot of %s successful", volname) + + return success + + +def doJob(name, lockFile, jobFunc, volname): + success = True + try: + f = os.open(lockFile, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK) + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + mtime = os.path.getmtime(lockFile) + global start_time + log.debug("%s last modified at %s", lockFile, time.ctime(mtime)) + if mtime < start_time: + log.debug("Processing job %s", name) + if jobFunc(volname, name): + log.info("Job %s succeeded", name) + else: + log.error("Job %s failed", name) + success = False + os.utime(lockFile, None) + else: + log.info("Job %s has been processed already", name) + fcntl.flock(f, fcntl.LOCK_UN) + except (OSError, IOError): + log.info("Job %s is being processed by another agent", name) + os.close(f) + except (OSError, IOError) as e: + log.debug("Failed to open lock file %s : %s", lockFile, e) + log.error("Failed to process job %s", name) + success = False + + return success + + +def main(): + script_name = os.path.basename(__file__) + initLogger(script_name) + global start_time + if sys.argv[1] == "--update": + if not os.path.exists(GCRON_TASKS): + # Create a flag in /var/run/gluster which indicates that this + # node doesn't have access to GCRON_TASKS right now, so that + # when the mount is available and GCRON_TASKS is available + # the flag will tell this routine to reload GCRON_CROND_TASK + try: + f = os.open(GCRON_RELOAD_FLAG, + os.O_CREAT | os.O_NONBLOCK, 0o644) + os.close(f) + except OSError as e: + if errno != EEXIST: + log.error("Failed to create %s : %s", + GCRON_RELOAD_FLAG, e) + output("Failed to create %s. Error: %s" + % (GCRON_RELOAD_FLAG, e)) + return + + if not os.path.exists(GCRON_CROND_TASK): + return + + # As GCRON_TASKS exists now, we should check if GCRON_RELOAD_FLAG + # also exists. If so we should touch GCRON_CROND_TASK and remove + # the GCRON_RELOAD_FLAG + if os.path.exists(GCRON_RELOAD_FLAG): + try: + os.remove(GCRON_RELOAD_FLAG); + process = subprocess.Popen(["touch", "-h", GCRON_CROND_TASK], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = process.communicate() + if process.returncode != 0: + log.error("Failed to touch %s. Error: %s.", + GCRON_CROND_TASK, err) + except (IOError, OSError) as e: + log.error("Failed to touch %s. Error: %s.", + GCRON_CROND_TASK, e) + return + if os.lstat(GCRON_TASKS).st_mtime > \ + os.lstat(GCRON_CROND_TASK).st_mtime: + try: + process = subprocess.Popen(["touch", "-h", GCRON_CROND_TASK], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = process.communicate() + if process.returncode != 0: + log.error("Failed to touch %s. Error: %s.", + GCRON_CROND_TASK, err) + except IOError as e: + log.error("Failed to touch %s. Error: %s.", + GCRON_CROND_TASK, e) + return + + volname = sys.argv[1] + jobname = sys.argv[2] + locking_file = os.path.join(LOCK_FILE_DIR, jobname) + log.debug("locking_file = %s", locking_file) + log.debug("volname = %s", volname) + log.debug("jobname = %s", jobname) + + start_time = int(time.time()) + + doJob("Scheduled-" + jobname + "-" + volname, locking_file, takeSnap, volname) + + +if __name__ == "__main__": + main() diff --git a/extras/snap_scheduler/snap_scheduler.py b/extras/snap_scheduler/snap_scheduler.py new file mode 100755 index 00000000000..e8fcc449a9b --- /dev/null +++ b/extras/snap_scheduler/snap_scheduler.py @@ -0,0 +1,941 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +from __future__ import print_function +import subprocess +import os +import os.path +import logging +import argparse +import fcntl +import logging.handlers +import sys +import shutil +from errno import EEXIST +from conf import GLUSTERFS_LIBEXECDIR +sys.path.insert(1, GLUSTERFS_LIBEXECDIR) + +EVENTS_ENABLED = True +try: + from events.eventtypes import SNAPSHOT_SCHEDULER_INITIALISED \ + as EVENT_SNAPSHOT_SCHEDULER_INITIALISED + from events.eventtypes import SNAPSHOT_SCHEDULER_INIT_FAILED \ + as EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED + from events.eventtypes import SNAPSHOT_SCHEDULER_DISABLED \ + as EVENT_SNAPSHOT_SCHEDULER_DISABLED + from events.eventtypes import SNAPSHOT_SCHEDULER_DISABLE_FAILED \ + as EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED + from events.eventtypes import SNAPSHOT_SCHEDULER_ENABLED \ + as EVENT_SNAPSHOT_SCHEDULER_ENABLED + from events.eventtypes import SNAPSHOT_SCHEDULER_ENABLE_FAILED \ + as EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED + from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_ADDED \ + as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED + from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED \ + as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED + from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_DELETED \ + as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED + from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED \ + as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED + from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_EDITED \ + as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED + from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED \ + as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED +except ImportError: + # Events APIs not installed, dummy eventtypes with None + EVENTS_ENABLED = False + EVENT_SNAPSHOT_SCHEDULER_INITIALISED = None + EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED = None + EVENT_SNAPSHOT_SCHEDULER_DISABLED = None + EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED = None + EVENT_SNAPSHOT_SCHEDULER_ENABLED = None + EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED = None + EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED = None + EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED = None + EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED = None + EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED = None + EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED = None + EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED = None + +SCRIPT_NAME = "snap_scheduler" +scheduler_enabled = False +log = logging.getLogger(SCRIPT_NAME) +SHARED_STORAGE_DIR="/run/gluster/shared_storage" +GCRON_DISABLED = SHARED_STORAGE_DIR+"/snaps/gcron_disabled" +GCRON_ENABLED = SHARED_STORAGE_DIR+"/snaps/gcron_enabled" +GCRON_TASKS = SHARED_STORAGE_DIR+"/snaps/glusterfs_snap_cron_tasks" +GCRON_CROND_TASK = "/etc/cron.d/glusterfs_snap_cron_tasks" +LOCK_FILE_DIR = SHARED_STORAGE_DIR+"/snaps/lock_files/" +LOCK_FILE = LOCK_FILE_DIR+"lock_file" +TMP_FILE = SHARED_STORAGE_DIR+"/snaps/tmp_file" +GCRON_UPDATE_TASK = "/etc/cron.d/gcron_update_task" +CURRENT_SCHEDULER = SHARED_STORAGE_DIR+"/snaps/current_scheduler" +tasks = {} +longest_field = 12 +current_scheduler = "" + +INTERNAL_ERROR = 2 +SHARED_STORAGE_DIR_DOESNT_EXIST = 3 +SHARED_STORAGE_NOT_MOUNTED = 4 +ANOTHER_TRANSACTION_IN_PROGRESS = 5 +INIT_FAILED = 6 +SCHEDULING_ALREADY_DISABLED = 7 +SCHEDULING_ALREADY_ENABLED = 8 +NODE_NOT_INITIALISED = 9 +ANOTHER_SCHEDULER_ACTIVE = 10 +JOB_ALREADY_EXISTS = 11 +JOB_NOT_FOUND = 12 +INVALID_JOBNAME = 13 +INVALID_VOLNAME = 14 +INVALID_SCHEDULE = 15 +INVALID_ARG = 16 +VOLUME_DOES_NOT_EXIST = 17 + +def print_error (error_num): + if error_num == INTERNAL_ERROR: + return "Internal Error" + elif error_num == SHARED_STORAGE_DIR_DOESNT_EXIST: + return "The shared storage directory ("+SHARED_STORAGE_DIR+")" \ + " does not exist." + elif error_num == SHARED_STORAGE_NOT_MOUNTED: + return "The shared storage directory ("+SHARED_STORAGE_DIR+")" \ + " is not mounted." + elif error_num == ANOTHER_TRANSACTION_IN_PROGRESS: + return "Another transaction is in progress." + elif error_num == INIT_FAILED: + return "Initialisation failed." + elif error_num == SCHEDULING_ALREADY_DISABLED: + return "Snapshot scheduler is already disabled." + elif error_num == SCHEDULING_ALREADY_ENABLED: + return "Snapshot scheduler is already enabled." + elif error_num == NODE_NOT_INITIALISED: + return "The node is not initialised." + elif error_num == ANOTHER_SCHEDULER_ACTIVE: + return "Another scheduler is active." + elif error_num == JOB_ALREADY_EXISTS: + return "The job already exists." + elif error_num == JOB_NOT_FOUND: + return "The job cannot be found." + elif error_num == INVALID_JOBNAME: + return "The job name is invalid." + elif error_num == INVALID_VOLNAME: + return "The volume name is invalid." + elif error_num == INVALID_SCHEDULE: + return "The schedule is invalid." + elif error_num == INVALID_ARG: + return "The argument is invalid." + elif error_num == VOLUME_DOES_NOT_EXIST: + return "The volume does not exist." + +def output(msg): + print("%s: %s" % (SCRIPT_NAME, msg)) + + +def initLogger(): + log.setLevel(logging.DEBUG) + logFormat = "[%(asctime)s %(filename)s:%(lineno)s %(funcName)s] "\ + "%(levelname)s %(message)s" + formatter = logging.Formatter(logFormat) + + sh = logging.handlers.SysLogHandler() + sh.setLevel(logging.ERROR) + sh.setFormatter(formatter) + + process = subprocess.Popen(["gluster", "--print-logdir"], + stdout=subprocess.PIPE, universal_newlines=True) + logfile = os.path.join(process.stdout.read()[:-1], SCRIPT_NAME + ".log") + + fh = logging.FileHandler(logfile) + fh.setLevel(logging.DEBUG) + fh.setFormatter(formatter) + + log.addHandler(sh) + log.addHandler(fh) + + +def scheduler_status(): + ret = INTERNAL_ERROR + global scheduler_enabled + try: + f = os.path.realpath(GCRON_TASKS) + if f != os.path.realpath(GCRON_ENABLED) or not os.path.exists(GCRON_ENABLED): + log.info("Snapshot scheduler is currently disabled.") + scheduler_enabled = False + else: + log.info("Snapshot scheduler is currently enabled.") + scheduler_enabled = True + ret = 0 + except: + log.error("Failed to enable snapshot scheduling. Error: " + "Failed to check the status of %s.", GCRON_DISABLED) + + return ret + +def enable_scheduler(): + ret = scheduler_status() + if ret == 0: + if not scheduler_enabled: + + # Check if another scheduler is active. + ret = get_current_scheduler() + if ret == 0: + if (current_scheduler != "none"): + print_str = "Failed to enable snapshot scheduling. " \ + "Error: Another scheduler is active." + log.error(print_str) + output(print_str) + ret = ANOTHER_SCHEDULER_ACTIVE + return ret + else: + print_str = "Failed to get current scheduler info." + log.error(print_str) + output(print_str) + return ret + + log.info("Enabling snapshot scheduler.") + try: + if os.path.exists(GCRON_DISABLED): + os.remove(GCRON_DISABLED) + if os.path.lexists(GCRON_TASKS): + os.remove(GCRON_TASKS) + try: + f = os.open(GCRON_ENABLED, os.O_CREAT | os.O_NONBLOCK, + 0o644) + os.close(f) + except OSError as e: + log.error("Failed to open %s. Error: %s.", + GCRON_ENABLED, e) + ret = INTERNAL_ERROR + return ret + os.symlink(GCRON_ENABLED, GCRON_TASKS) + update_current_scheduler("cli") + log.info("Snapshot scheduling is enabled") + output("Snapshot scheduling is enabled") + ret = 0 + except OSError as e: + print_str = ("Failed to enable snapshot scheduling." + "Error: {{}}" + e) + log.error(print_str) + output(print_str) + ret = INTERNAL_ERROR + else: + print_str = "Failed to enable snapshot scheduling. " \ + "Error: Snapshot scheduling is already enabled." + log.error(print_str) + output(print_str) + ret = SCHEDULING_ALREADY_ENABLED + else: + print_str = "Failed to enable snapshot scheduling. " \ + "Error: Failed to check scheduler status." + log.error(print_str) + output(print_str) + + return ret + + +def disable_scheduler(): + ret = scheduler_status() + if ret == 0: + if scheduler_enabled: + log.info("Disabling snapshot scheduler.") + try: + # Check if another scheduler is active. If not, then + # update current scheduler to "none". Else do nothing. + ret = get_current_scheduler() + if ret == 0: + if (current_scheduler == "cli"): + update_current_scheduler("none") + else: + print_str = "Failed to disable snapshot scheduling. " \ + "Error: Failed to get current scheduler info." + log.error(print_str) + output(print_str) + return ret + + if os.path.exists(GCRON_DISABLED): + os.remove(GCRON_DISABLED) + if os.path.lexists(GCRON_TASKS): + os.remove(GCRON_TASKS) + f = os.open(GCRON_DISABLED, os.O_CREAT, 0o644) + os.close(f) + os.symlink(GCRON_DISABLED, GCRON_TASKS) + log.info("Snapshot scheduling is disabled") + output("Snapshot scheduling is disabled") + ret = 0 + except OSError as e: + print_str = ("Failed to disable snapshot scheduling. Error: " + + e) + log.error(print_str) + output(print_str) + ret = INTERNAL_ERROR + else: + print_str = "Failed to disable scheduling. " \ + "Error: Snapshot scheduling is already disabled." + log.error(print_str) + output(print_str) + ret = SCHEDULING_ALREADY_DISABLED + else: + print_str = "Failed to disable snapshot scheduling. " \ + "Error: Failed to check scheduler status." + log.error(print_str) + output(print_str) + ret = INTERNAL_ERROR + + return ret + + +def load_tasks_from_file(): + global tasks + global longest_field + try: + with open(GCRON_ENABLED, 'r') as f: + for line in f: + line = line.rstrip('\n') + if not line: + break + line = line.split("gcron.py") + schedule = line[0].split("root")[0].rstrip(' ') + line = line[1].split(" ") + volname = line[1] + jobname = line[2] + longest_field = max(longest_field, len(jobname), len(volname), + len(schedule)) + tasks[jobname] = schedule+":"+volname + f.close() + ret = 0 + except IOError as e: + log.error("Failed to open %s. Error: %s.", GCRON_ENABLED, e) + ret = INTERNAL_ERROR + + return ret + + +def get_current_scheduler(): + global current_scheduler + try: + with open(CURRENT_SCHEDULER, 'r') as f: + current_scheduler = f.readline().rstrip('\n') + f.close() + ret = 0 + except IOError as e: + log.error("Failed to open %s. Error: %s.", CURRENT_SCHEDULER, e) + ret = INTERNAL_ERROR + + return ret + + +def list_schedules(): + log.info("Listing snapshot schedules.") + ret = load_tasks_from_file() + if ret == 0: + if len(tasks) == 0: + output("No snapshots scheduled") + else: + jobname = "JOB_NAME".ljust(longest_field+5) + schedule = "SCHEDULE".ljust(longest_field+5) + operation = "OPERATION".ljust(longest_field+5) + volname = "VOLUME NAME".ljust(longest_field+5) + hyphens = "".ljust((longest_field+5) * 4, '-') + print(jobname+schedule+operation+volname) + print(hyphens) + for key in sorted(tasks): + jobname = key.ljust(longest_field+5) + schedule = tasks[key].split(":")[0].ljust( + longest_field + 5) + volname = tasks[key].split(":")[1].ljust( + longest_field + 5) + operation = "Snapshot Create".ljust(longest_field+5) + print(jobname+schedule+operation+volname) + ret = 0 + else: + print_str = "Failed to list snapshot schedules. " \ + "Error: Failed to load tasks from "+GCRON_ENABLED + log.error(print_str) + output(print_str) + + return ret + + +def write_tasks_to_file(): + try: + with open(TMP_FILE, "w", 0o644) as f: + # If tasks is empty, just create an empty tmp file + if len(tasks) != 0: + for key in sorted(tasks): + jobname = key + schedule = tasks[key].split(":")[0] + volname = tasks[key].split(":")[1] + f.write("%s root PATH=$PATH:/usr/local/sbin:/usr/sbin " + "gcron.py %s %s\n" % (schedule, volname, jobname)) + f.write("\n") + f.flush() + os.fsync(f.fileno()) + f.close() + except IOError as e: + log.error("Failed to open %s. Error: %s.", TMP_FILE, e) + ret = INTERNAL_ERROR + return ret + + shutil.move(TMP_FILE, GCRON_ENABLED) + ret = 0 + + return ret + +def update_current_scheduler(data): + try: + with open(TMP_FILE, "w", 0o644) as f: + f.write("%s" % data) + f.flush() + os.fsync(f.fileno()) + f.close() + except IOError as e: + log.error("Failed to open %s. Error: %s.", TMP_FILE, e) + ret = INTERNAL_ERROR + return ret + + shutil.move(TMP_FILE, CURRENT_SCHEDULER) + ret = 0 + + return ret + + +def isVolumePresent(volname): + success = False + if volname == "": + log.debug("No volname given") + return success + + cli = ["gluster", + "volume", + "info", + volname] + log.debug("Running command '%s'", " ".join(cli)) + + p = subprocess.Popen(cli, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate() + rv = p.returncode + + log.debug("Command '%s' returned '%d'", " ".join(cli), rv) + + if rv: + log.error("Command output:") + log.error(err) + else: + success = True; + + return success + + +def add_schedules(jobname, schedule, volname): + log.info("Adding snapshot schedules.") + ret = load_tasks_from_file() + if ret == 0: + if jobname in tasks: + print_str = ("%s already exists in schedule. Use " + "'edit' to modify %s" % (jobname, jobname)) + log.error(print_str) + output(print_str) + ret = JOB_ALREADY_EXISTS + else: + if not isVolumePresent(volname): + print_str = ("Volume %s does not exist. Create %s and retry." % + (volname, volname)) + log.error(print_str) + output(print_str) + ret = VOLUME_DOES_NOT_EXIST + else: + tasks[jobname] = schedule + ":" + volname + ret = write_tasks_to_file() + if ret == 0: + # Create a LOCK_FILE for the job + job_lockfile = LOCK_FILE_DIR + jobname + try: + f = os.open(job_lockfile, os.O_CREAT | os.O_NONBLOCK, + 0o644) + os.close(f) + except OSError as e: + log.error("Failed to open %s. Error: %s.", + job_lockfile, e) + ret = INTERNAL_ERROR + return ret + log.info("Successfully added snapshot schedule %s" % + jobname) + output("Successfully added snapshot schedule") + ret = 0 + else: + print_str = "Failed to add snapshot schedule. " \ + "Error: Failed to load tasks from "+GCRON_ENABLED + log.error(print_str) + output(print_str) + + return ret + + +def delete_schedules(jobname): + log.info("Delete snapshot schedules.") + ret = load_tasks_from_file() + if ret == 0: + if jobname in tasks: + del tasks[jobname] + ret = write_tasks_to_file() + if ret == 0: + # Delete the LOCK_FILE for the job + job_lockfile = LOCK_FILE_DIR+jobname + try: + os.remove(job_lockfile) + except OSError as e: + log.error("Failed to open %s. Error: %s.", + job_lockfile, e) + ret = INTERNAL_ERROR + return ret + log.info("Successfully deleted snapshot schedule %s" + % jobname) + output("Successfully deleted snapshot schedule") + ret = 0 + else: + print_str = ("Failed to delete %s. Error: No such " + "job scheduled" % jobname) + log.error(print_str) + output(print_str) + ret = JOB_NOT_FOUND + else: + print_str = "Failed to delete snapshot schedule. " \ + "Error: Failed to load tasks from "+GCRON_ENABLED + log.error(print_str) + output(print_str) + + return ret + + +def edit_schedules(jobname, schedule, volname): + log.info("Editing snapshot schedules.") + ret = load_tasks_from_file() + if ret == 0: + if jobname in tasks: + if not isVolumePresent(volname): + print_str = ("Volume %s does not exist. Create %s and retry." % + (volname, volname)) + log.error(print_str) + output(print_str) + ret = VOLUME_DOES_NOT_EXIST + else: + tasks[jobname] = schedule+":"+volname + ret = write_tasks_to_file() + if ret == 0: + log.info("Successfully edited snapshot schedule %s" % + jobname) + output("Successfully edited snapshot schedule") + else: + print_str = ("Failed to edit %s. Error: No such " + "job scheduled" % jobname) + log.error(print_str) + output(print_str) + ret = JOB_NOT_FOUND + else: + print_str = "Failed to edit snapshot schedule. " \ + "Error: Failed to load tasks from "+GCRON_ENABLED + log.error(print_str) + output(print_str) + + return ret + +def get_bool_val(): + getsebool_cli = ["getsebool", + "-a"] + p1 = subprocess.Popen(getsebool_cli, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + grep_cmd = ["grep", + "cron_system_cronjob_use_shares"] + p2 = subprocess.Popen(grep_cmd, stdin=p1.stdout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + p1.stdout.close() + output, err = p2.communicate() + rv = p2.returncode + + if rv: + log.error("Command output:") + log.error(err) + return -1 + + bool_val = output.split()[2] + log.debug("Bool value = '%s'", bool_val) + + return bool_val + +def get_selinux_status(): + getenforce_cli = ["getenforce"] + log.debug("Running command '%s'", " ".join(getenforce_cli)) + + try: + p1 = subprocess.Popen(getenforce_cli, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError as oserr: + log.error("Failed to run the command \"getenforce\". Error: %s" %\ + oserr) + return -1 + + output, err = p1.communicate() + rv = p1.returncode + + if rv: + log.error("Command output:") + log.error(err) + return -1 + else: + selinux_status=output.rstrip() + log.debug("selinux status: %s", selinux_status) + + return selinux_status + +def set_cronjob_user_share(): + selinux_status = get_selinux_status() + if (selinux_status == -1): + log.error("Failed to get selinux status") + return -1 + elif (selinux_status == "Disabled"): + return 0 + + bool_val = get_bool_val() + # In case of a failure (where the boolean value is not) + # present in the system, we should not proceed further + # We should only proceed when the value is "off" + if (bool_val == -1 or bool_val != "off"): + return 0 + + setsebool_cli = ["setsebool", "-P", + "cron_system_cronjob_use_shares", + "on"] + log.debug("Running command '%s'", " ".join(setsebool_cli)) + + p1 = subprocess.Popen(setsebool_cli, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + output, err = p1.communicate() + rv = p1.returncode + + if rv: + log.error("Command output:") + log.error(err) + return rv + + bool_val = get_bool_val() + if (bool_val == "on"): + return 0 + else: + # In case of an error or if boolean is not on + # we return a failure here + return -1 + +def initialise_scheduler(): + ret = set_cronjob_user_share() + if ret: + log.error("Failed to set selinux boolean " + "cron_system_cronjob_use_shares to 'on'") + return ret + + try: + with open(TMP_FILE, "w+", 0o644) as f: + updater = ("* * * * * root PATH=$PATH:/usr/local/sbin:" + "/usr/sbin gcron.py --update\n") + f.write("%s\n" % updater) + f.flush() + os.fsync(f.fileno()) + f.close() + except IOError as e: + log.error("Failed to open %s. Error: %s.", TMP_FILE, e) + ret = INIT_FAILED + return ret + + shutil.move(TMP_FILE, GCRON_UPDATE_TASK) + + if not os.path.lexists(GCRON_TASKS): + try: + f = open(GCRON_TASKS, "w", 0o644) + f.close() + except IOError as e: + log.error("Failed to open %s. Error: %s.", GCRON_TASKS, e) + ret = INIT_FAILED + return ret + + if os.path.lexists(GCRON_CROND_TASK): + os.remove(GCRON_CROND_TASK) + + os.symlink(GCRON_TASKS, GCRON_CROND_TASK) + + log.info("Successfully initialised snapshot scheduler for this node") + output("Successfully initialised snapshot scheduler for this node") + gf_event (EVENT_SNAPSHOT_SCHEDULER_INITIALISED, status="Success") + + ret = 0 + return ret + + +def syntax_checker(args): + if hasattr(args, 'jobname'): + if (len(args.jobname.split()) != 1): + output("Invalid Jobname. Jobname should not be empty and should not contain \" \" character.") + ret = INVALID_JOBNAME + return ret + args.jobname=args.jobname.strip() + + if hasattr(args, 'volname'): + if (len(args.volname.split()) != 1): + output("Invalid Volname. Volname should not be empty and should not contain \" \" character.") + ret = INVALID_VOLNAME + return ret + args.volname=args.volname.strip() + + if hasattr(args, 'schedule'): + if (len(args.schedule.split()) != 5): + output("Invalid Schedule. Please refer to the following for adding a valid cron schedule") + print ("* * * * *") + print ("| | | | |") + print ("| | | | +---- Day of the Week (range: 1-7, 1 standing for Monday)") + print ("| | | +------ Month of the Year (range: 1-12)") + print ("| | +-------- Day of the Month (range: 1-31)") + print ("| +---------- Hour (range: 0-23)") + print ("+------------ Minute (range: 0-59)") + ret = INVALID_SCHEDULE + return ret + + ret = 0 + return ret + + +def perform_operation(args): + if not os.path.exists(CURRENT_SCHEDULER): + update_current_scheduler("none") + + # Initialise snapshot scheduler on local node + if args.action == "init": + ret = initialise_scheduler() + if ret != 0: + output("Failed to initialise snapshot scheduling") + gf_event (EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED, + error=print_error(ret)) + return ret + + # Disable snapshot scheduler + if args.action == "disable_force": + ret = disable_scheduler() + if ret == 0: + subprocess.Popen(["touch", "-h", GCRON_TASKS]) + gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLED, + status="Successfully Disabled") + else: + gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED, + error=print_error(ret)) + return ret + + # Check if the symlink to GCRON_TASKS is properly set in the shared storage + if (not os.path.lexists(GCRON_UPDATE_TASK) or + not os.path.lexists(GCRON_CROND_TASK) or + os.readlink(GCRON_CROND_TASK) != GCRON_TASKS): + print_str = ("Please run 'snap_scheduler.py' init to initialise " + "the snap scheduler for the local node.") + log.error(print_str) + output(print_str) + ret = NODE_NOT_INITIALISED + return ret + + # Check status of snapshot scheduler. + if args.action == "status": + ret = scheduler_status() + if ret == 0: + if scheduler_enabled: + output("Snapshot scheduling status: Enabled") + else: + output("Snapshot scheduling status: Disabled") + else: + output("Failed to check status of snapshot scheduler") + return ret + + # Enable snapshot scheduler + if args.action == "enable": + ret = enable_scheduler() + if ret == 0: + subprocess.Popen(["touch", "-h", GCRON_TASKS]) + gf_event (EVENT_SNAPSHOT_SCHEDULER_ENABLED, + status="Successfully Enabled") + else: + gf_event (EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED, + error=print_error(ret)) + return ret + + # Disable snapshot scheduler + if args.action == "disable": + ret = disable_scheduler() + if ret == 0: + subprocess.Popen(["touch", "-h", GCRON_TASKS]) + gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLED, + status="Successfully Disabled") + else: + gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED, + error=print_error(ret)) + return ret + + # List snapshot schedules + if args.action == "list": + ret = list_schedules() + return ret + + # Add snapshot schedules + if args.action == "add": + ret = syntax_checker(args) + if ret != 0: + return ret + ret = add_schedules(args.jobname, args.schedule, args.volname) + if ret == 0: + subprocess.Popen(["touch", "-h", GCRON_TASKS]) + gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED, + status="Successfully added job "+args.jobname) + else: + gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED, + status="Failed to add job "+args.jobname, + error=print_error(ret)) + return ret + + # Delete snapshot schedules + if args.action == "delete": + ret = syntax_checker(args) + if ret != 0: + return ret + ret = delete_schedules(args.jobname) + if ret == 0: + subprocess.Popen(["touch", "-h", GCRON_TASKS]) + gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED, + status="Successfully deleted job "+args.jobname) + else: + gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED, + status="Failed to delete job "+args.jobname, + error=print_error(ret)) + return ret + + # Edit snapshot schedules + if args.action == "edit": + ret = syntax_checker(args) + if ret != 0: + return ret + ret = edit_schedules(args.jobname, args.schedule, args.volname) + if ret == 0: + subprocess.Popen(["touch", "-h", GCRON_TASKS]) + gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED, + status="Successfully edited job "+args.jobname) + else: + gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED, + status="Failed to edit job "+args.jobname, + error=print_error(ret)) + return ret + + ret = INVALID_ARG + return ret + +def gf_event(event_type, **kwargs): + if EVENTS_ENABLED: + from events.gf_event import gf_event as gfevent + gfevent(event_type, **kwargs) + + +def main(argv): + initLogger() + ret = -1 + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="action", + metavar=('{init, status, enable,' + ' disable, list, add,' + ' delete, edit}')) + subparsers.add_parser('init', + help="Initialise the node for snapshot scheduling") + + subparsers.add_parser("status", + help="Check if snapshot scheduling is " + "enabled or disabled") + subparsers.add_parser("enable", + help="Enable snapshot scheduling") + subparsers.add_parser("disable", + help="Disable snapshot scheduling") + subparsers.add_parser("disable_force") + subparsers.add_parser("list", + help="List snapshot schedules") + parser_add = subparsers.add_parser("add", + help="Add snapshot schedules") + parser_add.add_argument("jobname", help="Job Name") + parser_add.add_argument("schedule", help="Schedule") + parser_add.add_argument("volname", help="Volume Name") + + parser_delete = subparsers.add_parser("delete", + help="Delete snapshot schedules") + parser_delete.add_argument("jobname", help="Job Name") + parser_edit = subparsers.add_parser("edit", + help="Edit snapshot schedules") + parser_edit.add_argument("jobname", help="Job Name") + parser_edit.add_argument("schedule", help="Schedule") + parser_edit.add_argument("volname", help="Volume Name") + + args = parser.parse_args(argv) + + if not os.path.exists(SHARED_STORAGE_DIR): + output("Failed: "+SHARED_STORAGE_DIR+" does not exist.") + return SHARED_STORAGE_DIR_DOESNT_EXIST + + if not os.path.ismount(SHARED_STORAGE_DIR): + output("Failed: Shared storage is not mounted at "+SHARED_STORAGE_DIR) + return SHARED_STORAGE_NOT_MOUNTED + + if not os.path.exists(SHARED_STORAGE_DIR+"/snaps/"): + try: + os.makedirs(SHARED_STORAGE_DIR+"/snaps/") + except OSError as e: + if errno != EEXIST: + log.error("Failed to create %s : %s", SHARED_STORAGE_DIR+"/snaps/", e) + output("Failed to create %s. Error: %s" + % (SHARED_STORAGE_DIR+"/snaps/", e)) + return INTERNAL_ERROR + + if not os.path.exists(GCRON_ENABLED): + f = os.open(GCRON_ENABLED, os.O_CREAT | os.O_NONBLOCK, 0o644) + os.close(f) + + if not os.path.exists(LOCK_FILE_DIR): + try: + os.makedirs(LOCK_FILE_DIR) + except OSError as e: + if errno != EEXIST: + log.error("Failed to create %s : %s", LOCK_FILE_DIR, e) + output("Failed to create %s. Error: %s" + % (LOCK_FILE_DIR, e)) + return INTERNAL_ERROR + + try: + f = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK, 0o644) + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + ret = perform_operation(args) + fcntl.flock(f, fcntl.LOCK_UN) + except IOError: + log.info("%s is being processed by another agent.", LOCK_FILE) + output("Another snap_scheduler command is running. " + "Please try again after some time.") + return ANOTHER_TRANSACTION_IN_PROGRESS + os.close(f) + except OSError as e: + log.error("Failed to open %s : %s", LOCK_FILE, e) + output("Failed to open %s. Error: %s" % (LOCK_FILE, e)) + return INTERNAL_ERROR + + return ret + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/extras/statedumpparse.rb b/extras/statedumpparse.rb new file mode 100755 index 00000000000..1aff43377db --- /dev/null +++ b/extras/statedumpparse.rb @@ -0,0 +1,208 @@ +#!/usr/bin/env ruby + +require 'time' +require 'optparse' + +unless Array.instance_methods.include? :to_h + class Array + def to_h + h = {} + each { |k,v| h[k]=v } + h + end + end +end + +# statedump.c:gf_proc_dump_mempool_info uses a five-dash record separator, +# client.c:client_fd_lk_ctx_dump uses a six-dash record separator. +ARRSEP = /^(-{5,6}=-{5,6})?$/ +HEAD = /^\[(.*)\]$/ +INPUT_FORMATS = %w[statedump json] + +format = 'json' +input_format = 'statedump' +tz = '+0000' +memstat_select,memstat_reject = //,/\Z./ +OptionParser.new do |op| + op.banner << " [<] <STATEDUMP>" + op.on("-f", "--format=F", "json/yaml/memstat(-[plain|human|json])") { |s| format = s } + op.on("--input-format=F", INPUT_FORMATS.join(?/)) { |s| input_format = s } + op.on("--timezone=T", + "time zone to apply to zoneless timestamps [default UTC]") { |s| tz = s } + op.on("--memstat-select=RX", "memstat: select memory types matching RX") { |s| + memstat_select = Regexp.new s + } + op.on("--memstat-reject=RX", "memstat: reject memory types matching RX") { |s| + memstat_reject = Regexp.new s + } +end.parse! + + +if format =~ /\Amemstat(?:-(.*))?/ + memstat_type = $1 || 'plain' + unless %w[plain human json].include? memstat_type + raise "unknown memstat type #{memstat_type.dump}" + end + format = 'memstat' +end + +repr, logsep = case format +when 'yaml' + require 'yaml' + + [proc { |e| e.to_yaml }, "\n"] +when 'json', 'memstat' + require 'json' + + [proc { |e| e.to_json }, " "] +else + raise "unkonwn format '#{format}'" +end +formatter = proc { |e| puts repr.call(e) } + +INPUT_FORMATS.include? input_format or raise "unkwown input format '#{input_format}'" + +dumpinfo = {} + +# parse a statedump entry +elem_cbk = proc { |s,&cbk| + arraylike = false + s.grep(/\S/).empty? and next + head = nil + while s.last =~ /^\s*$/ + s.pop + end + body = catch { |misc2| + s[0] =~ HEAD ? (head = $1) : (throw misc2) + body = [[]] + s[1..-1].each { |l| + if l =~ ARRSEP + arraylike = true + body << [] + next + end + body.last << l + } + + body.reject(&:empty?).map { |e| + ea = e.map { |l| + k,v = l.split("=",2) + m = /\A(0|-?[1-9]\d*)(\.\d+)?\Z/.match v + [k, m ? (m[2] ? Float(v) : Integer(v)) : v] + } + begin + ea.to_h + rescue + throw misc2 + end + } + } + + if body + cbk.call [head, arraylike ? body : (body.empty? ? {} : body[0])] + else + STDERR.puts ["WARNING: failed to parse record:", repr.call(s)].join(logsep) + end +} + +# aggregator routine +aggr = case format +when 'memstat' + meminfo = {} + # commit memory-related entries to meminfo + proc { |k,r| + case k + when /memusage/ + (meminfo["GF_MALLOC"]||={})[k] ||= r["size"] if k =~ memstat_select and k !~ memstat_reject + when "mempool" + r.each {|e| + kk = "mempool:#{e['pool-name']}" + (meminfo["mempool"]||={})[kk] ||= e["size"] if kk =~ memstat_select and kk !~ memstat_reject + } + end + } +else + # just format data, don't actually aggregate anything + proc { |pair| formatter.call pair } +end + +# processing the data +case input_format +when 'statedump' + acc = [] + $<.each { |l| + l = l.strip + if l =~ /^(DUMP-(?:START|END)-TIME):\s+(.*)/ + dumpinfo["_meta"]||={} + (dumpinfo["_meta"]["date"]||={})[$1] = Time.parse([$2, tz].join " ") + next + end + + if l =~ HEAD + elem_cbk.call(acc, &aggr) + acc = [l] + next + end + + acc << l + } + elem_cbk.call(acc, &aggr) +when 'json' + $<.each { |l| + r = JSON.load l + case r + when Array + aggr[r] + when Hash + dumpinfo.merge! r + end + } +end + +# final actions: output aggregated data +case format +when 'memstat' + ma = meminfo.values.map(&:to_a).inject(:+) + totals = meminfo.map { |coll,h| [coll, h.values.inject(:+)] }.to_h + tt = ma.transpose[1].inject(:+) + + summary_sep,showm = case memstat_type + when 'json' + ["", proc { |k,v| puts({type: k, value: v}.to_json) }] + when 'plain', 'human' + # human-friendly number representation + hr = proc { |n| + qa = %w[B kB MB GB] + q = ((1...qa.size).find {|i| n < (1 << i*10)} || qa.size) - 1 + "%.2f%s" % [n.to_f / (1 << q*10), qa[q]] + } + + templ = "%{val} %{key}" + tft = proc { |t| t } + nft = if memstat_type == 'human' + nw = [ma.transpose[1], totals.values, tt].flatten.map{|n| hr[n].size}.max + proc { |n| + hn = hr[n] + " " * (nw - hn.size) + hn + } + else + nw = tt.to_s.size + proc { |n| "%#{nw}d" % n } + end + ## Alternative template, key first: + # templ = "%{key} %{val}" + # tw = ma.transpose[0].map(&:size).max + # tft = proc { |t| t + " " * [tw - t.size, 0].max } + # nft = (memstat_type == 'human') ? hr : proc { |n| n } + ["\n", proc { |k,v| puts templ % {key: tft[k], val: nft[v]} }] + else + raise 'this should be impossible' + end + + ma.sort_by { |k,v| v }.each(&showm) + print summary_sep + totals.each { |coll,t| showm.call "Total #{coll}", t } + showm.call "TOTAL", tt +else + formatter.call dumpinfo +end diff --git a/extras/stop-all-gluster-processes.sh b/extras/stop-all-gluster-processes.sh index 6c81a301c72..710aaf5fd3c 100755 --- a/extras/stop-all-gluster-processes.sh +++ b/extras/stop-all-gluster-processes.sh @@ -1,33 +1,193 @@ -#! /bin/sh +#!/bin/bash +# +# Kill all the processes/services except glusterd +# +# Usage: ./extras/stop-all-gluster-processes.sh [-g] [-h] +# options: +# -g Terminate in graceful mode +# -h Show this message, then exit +# +# eg: +# 1. ./extras/stop-all-gluster-processes.sh +# 2. ./extras/stop-all-gluster-processes.sh -g +# +# By default, this script executes in force mode, i.e. all of brick, gsyncd +# and other glustershd services/processes are killed without checking for +# ongoing tasks such as geo-rep, self-heal, rebalance and etc. which may lead +# to inconsistency after the node is brought back. +# +# On specifying '-g' option this script works in graceful mode, to maintain +# data consistency the script fails with a valid exit code incase if any of +# the gluster processes are busy in doing their jobs. +# +# The author of page [1] proposes user-defined exit codes to the range 64 - 113 +# Find the better explanation behind the choice in the link +# +# The exit code returned by stop-all-gluster-processes.sh: +# 0 No errors/Success +# 64 Rebalance is in progress +# 65 Self-Heal is in progress +# 66 Tier daemon running on this node +# 127 option not found +# +# [1] http://www.tldp.org/LDP/abs/html/exitcodes.html -function main() + +# global +errors=0 + +# find the mounts and return their pids +get_mount_pids() { - for pidfile in $(find /var/lib/glusterd/ -iname '*pid'); + local opts + local pid + + for opts in $(grep -w fuse.glusterfs /proc/mounts| awk '{print $1":/"$2}'); do - pid=$(cat ${pidfile}); - echo "sending SIGTERM to process $pid"; - kill -TERM $pid; + IFS=' ' read -r -a volinfo <<< $(echo "${opts}" | sed 's/:\// /g') + pid+="$(ps -Ao pid,args | grep -w "volfile-server=${volinfo[0]}" | + grep -w "volfile-id=/${volinfo[1]}" | grep -w "${volinfo[2]}" | + awk '{print $1}') " done + echo "${pid}" +} - # for geo-replication, only 'monitor' has pid file written, other - # processes are not having a pid file, so get it through 'ps' and - # handle these processes - gsyncpid=`ps aux | grep gluster | grep gsync | awk '{print $2}'`; - test -n $gsyncpid && kill -TERM $gsyncpid; +# handle mount processes i.e. 'glusterfs' +kill_mounts() +{ + local signal=${1} + local pid - sleep 5; + for pid in $(get_mount_pids); + do + echo "sending SIG${signal} to mount process with pid: ${pid}"; + kill -${signal} ${pid}; + done +} + +# handle brick processes and node services +kill_bricks_and_services() +{ + local signal=${1} + local pidfile + local pid + + for pidfile in $(find /var/run/gluster/ -name '*.pid'); + do + local pid=$(cat ${pidfile}); + echo "sending SIG${signal} to pid: ${pid}"; + kill -${signal} ${pid}; + done +} + +# for geo-replication, only 'monitor' has pid file written, other +# processes are not having a pid file, so get it through 'ps' and +# handle these processes +kill_georep_gsync() +{ + local signal=${1} + + # FIXME: add strick/better check + local gsyncpid=$(ps -Ao pid,args | grep gluster | grep gsync | + awk '{print $1}'); + if [ -n "${gsyncpid}" ] + then + echo "sending SIG${signal} to geo-rep gsync process ${gsyncpid}"; + kill -${signal} ${gsyncpid} || errors=$((${errors} + 1)); + fi +} - # if pid file still exists, its something to KILL - for pidfile in $(find /var/lib/glusterd/ -iname '*pid'); +# check if all processes are ready to die +check_background_tasks() +{ + volumes=$(gluster vol list) + quit=0 + for volname in ${volumes}; do - pid=$(cat ${pidfile}); - echo "sending SIGKILL to process $pid"; - kill -KILL $pid; + # tiering + if [[ $(gluster volume tier ${volname} status 2> /dev/null | + grep "localhost" | grep -c "in progress") -gt 0 ]] + then + quit=66 + break; + fi + + # rebalance + if [[ $(gluster volume rebalance ${volname} status 2> /dev/null | + grep -c "in progress") -gt 0 ]] + then + quit=64 + break; + fi + + # self heal + if [[ $(gluster volume heal ${volname} info | grep "Number of entries" | + awk '{ sum+=$4} END {print sum}') -gt 0 ]]; + then + quit=65 + break; + fi + + # geo-rep, snapshot and quota doesn't need grace checks, + # as they ensures the consistancy on force kills + done + + echo ${quit} +} + +usage() +{ + cat <<EOM +Usage: $0 [-g] [-h] + options: + -g Terminate in graceful mode + -h Show this message, then exit + +eg: + 1. $0 + 2. $0 -g +EOM +} + +main() +{ + while getopts "gh" opt; do + case $opt in + g) + # graceful mode + quit=$(check_background_tasks) + if [[ ${quit} -ne 0 ]] + then + exit ${quit}; + fi + # else safe to kill + ;; + h) + usage + exit 0; + ;; + *) + usage + exit 127; + ;; + esac done + # remove all the options that have been parsed by getopts + shift $((OPTIND-1)) + + kill_mounts TERM + kill_georep_gsync TERM + kill_bricks_and_services TERM + + sleep 5; + echo "" + + # still not Terminated? let's pass SIGKILL + kill_mounts KILL + kill_georep_gsync KILL + kill_bricks_and_services KILL - # handle 'KILL' of geo-replication - gsyncpid=`ps aux | grep gluster | grep gsync | awk '{print $2}'`; - test -n $gsyncpid && kill -KILL $gsyncpid; + exit ${errors}; } -main "$@"; +main "$@" diff --git a/extras/stripe-merge.c b/extras/stripe-merge.c index 74bd47e303e..e013a6e6e8a 100644 --- a/extras/stripe-merge.c +++ b/extras/stripe-merge.c @@ -28,7 +28,7 @@ #include <stdint.h> #include <errno.h> #include <string.h> -#include <attr/xattr.h> +#include <sys/xattr.h> #include <fnmatch.h> #define ATTRNAME_STRIPE_INDEX "trusted.*.stripe-index" @@ -40,33 +40,33 @@ #define INVALID_MODE UINT32_MAX struct file_stripe_info { - int stripe_count; - int stripe_size; - int coalesce; - mode_t mode; - int fd[0]; + int stripe_count; + int stripe_size; + int coalesce; + mode_t mode; + int fd[0]; }; -static int close_files(struct file_stripe_info *); +static int +close_files(struct file_stripe_info *); -static struct -file_stripe_info *alloc_file_stripe_info(int count) +static struct file_stripe_info * +alloc_file_stripe_info(int count) { - int i; - struct file_stripe_info *finfo; + int i; + struct file_stripe_info *finfo; - finfo = calloc(1, sizeof(struct file_stripe_info) + - (sizeof(int) * count)); - if (!finfo) - return NULL; + finfo = calloc(1, sizeof(struct file_stripe_info) + (sizeof(int) * count)); + if (!finfo) + return NULL; - for (i = 0; i < count; i++) - finfo->fd[i] = INVALID_FD; + for (i = 0; i < count; i++) + finfo->fd[i] = INVALID_FD; - finfo->mode = INVALID_MODE; - finfo->coalesce = INVALID_FD; + finfo->mode = INVALID_MODE; + finfo->coalesce = INVALID_FD; - return finfo; + return finfo; } /* @@ -77,39 +77,39 @@ file_stripe_info *alloc_file_stripe_info(int count) static int get_stripe_attr_name(const char *path, const char *pattern, char **attrname) { - char attrbuf[4096]; - char *ptr, *match = NULL; - int len, r, match_count = 0; - - if (!path || !pattern || !attrname) - return -1; - - len = listxattr(path, attrbuf, sizeof(attrbuf)); - if (len < 0) - return len; - - ptr = attrbuf; - while (ptr) { - r = fnmatch(pattern, ptr, 0); - if (!r) { - if (!match) - match = ptr; - match_count++; - } else if (r != FNM_NOMATCH) { - return -1; - } - - len -= strlen(ptr) + 1; - if (len > 0) - ptr += strlen(ptr) + 1; - else - ptr = NULL; - } - - if (match) - *attrname = strdup(match); - - return match_count; + char attrbuf[4096]; + char *ptr, *match = NULL; + int len, r, match_count = 0; + + if (!path || !pattern || !attrname) + return -1; + + len = listxattr(path, attrbuf, sizeof(attrbuf)); + if (len < 0) + return len; + + ptr = attrbuf; + while (ptr) { + r = fnmatch(pattern, ptr, 0); + if (!r) { + if (!match) + match = ptr; + match_count++; + } else if (r != FNM_NOMATCH) { + return -1; + } + + len -= strlen(ptr) + 1; + if (len > 0) + ptr += strlen(ptr) + 1; + else + ptr = NULL; + } + + if (match) + *attrname = strdup(match); + + return match_count; } /* @@ -118,19 +118,19 @@ get_stripe_attr_name(const char *path, const char *pattern, char **attrname) static int get_stripe_attr_val(const char *path, const char *attr, int *val) { - char attrbuf[4096]; - int len; + char attrbuf[4096]; + int len; - if (!path || !attr || !val) - return -1; + if (!path || !attr || !val) + return -1; - len = getxattr(path, attr, attrbuf, sizeof(attrbuf)); - if (len < 0) - return len; + len = getxattr(path, attr, attrbuf, sizeof(attrbuf)); + if (len < 0) + return len; - *val = atoi(attrbuf); + *val = atoi(attrbuf); - return 0; + return 0; } /* @@ -145,29 +145,31 @@ get_stripe_attr_val(const char *path, const char *attr, int *val) static int get_attr(const char *path, const char *pattern, char **buf, int *val) { - int count = 1; - - if (!buf) - return -1; - - if (!*buf) { - count = get_stripe_attr_name(path, pattern, buf); - if (count > 1) { - /* pattern isn't good enough */ - fprintf(stderr, "ERROR: duplicate attributes found " - "matching pattern: %s\n", pattern); - free(*buf); - *buf = NULL; - return count; - } else if (count < 1) { - return count; - } - } - - if (get_stripe_attr_val(path, *buf, val) < 0) - return -1; - - return count; + int count = 1; + + if (!buf) + return -1; + + if (!*buf) { + count = get_stripe_attr_name(path, pattern, buf); + if (count > 1) { + /* pattern isn't good enough */ + fprintf(stderr, + "ERROR: duplicate attributes found " + "matching pattern: %s\n", + pattern); + free(*buf); + *buf = NULL; + return count; + } else if (count < 1) { + return count; + } + } + + if (get_stripe_attr_val(path, *buf, val) < 0) + return -1; + + return count; } /* @@ -178,164 +180,168 @@ get_attr(const char *path, const char *pattern, char **buf, int *val) * print a warning if any files are missing. We proceed without error in the * latter case to support partial recovery. */ -static struct -file_stripe_info *validate_and_open_files(char *paths[], int count) +static struct file_stripe_info * +validate_and_open_files(char *paths[], int count) { - int i, val, tmp; - struct stat sbuf; - char *stripe_count_attr = NULL; - char *stripe_size_attr = NULL; - char *stripe_index_attr = NULL; - char *stripe_coalesce_attr = NULL; - struct file_stripe_info *finfo = NULL; - - for (i = 0; i < count; i++) { - if (!paths[i]) - goto err; - - /* - * Check the stripe count first so we can allocate the info - * struct with the appropriate number of fds. - */ - if (get_attr(paths[i], ATTRNAME_STRIPE_COUNT, - &stripe_count_attr, &val) != 1) { - fprintf(stderr, "ERROR: %s: attribute: '%s'\n", - paths[i], ATTRNAME_STRIPE_COUNT); - goto err; - } - if (!finfo) { - finfo = alloc_file_stripe_info(val); - if (!finfo) - goto err; - - if (val != count) - fprintf(stderr, "WARNING: %s: stripe-count " - "(%d) != file count (%d). Result may " - "be incomplete.\n", paths[i], val, - count); - - finfo->stripe_count = val; - } else if (val != finfo->stripe_count) { - fprintf(stderr, "ERROR %s: invalid stripe count: %d " - "(expected %d)\n", paths[i], val, - finfo->stripe_count); - goto err; - } - - /* - * Get and validate the chunk size. - */ - if (get_attr(paths[i], ATTRNAME_STRIPE_SIZE, &stripe_size_attr, - &val) != 1) { - fprintf(stderr, "ERROR: %s: attribute: '%s'\n", - paths[i], ATTRNAME_STRIPE_SIZE); - goto err; - } - - if (!finfo->stripe_size) { - finfo->stripe_size = val; - } else if (val != finfo->stripe_size) { - fprintf(stderr, "ERROR: %s: invalid stripe size: %d " - "(expected %d)\n", paths[i], val, - finfo->stripe_size); - goto err; - } - - /* - * stripe-coalesce is a backward compatible attribute. If the - * attribute does not exist, assume a value of zero for the - * traditional stripe format. - */ - tmp = get_attr(paths[i], ATTRNAME_STRIPE_COALESCE, - &stripe_coalesce_attr, &val); - if (!tmp) { - val = 0; - } else if (tmp != 1) { - fprintf(stderr, "ERROR: %s: attribute: '%s'\n", - paths[i], ATTRNAME_STRIPE_COALESCE); - goto err; - } - - if (finfo->coalesce == INVALID_FD) { - finfo->coalesce = val; - } else if (val != finfo->coalesce) { - fprintf(stderr, "ERROR: %s: invalid coalesce flag\n", - paths[i]); - goto err; - } - - /* - * Get/validate the stripe index and open the file in the - * appropriate fd slot. - */ - if (get_attr(paths[i], ATTRNAME_STRIPE_INDEX, - &stripe_index_attr, &val) != 1) { - fprintf(stderr, "ERROR: %s: attribute: '%s'\n", - paths[i], ATTRNAME_STRIPE_INDEX); - goto err; - } - if (finfo->fd[val] != INVALID_FD) { - fprintf(stderr, "ERROR: %s: duplicate stripe index: " - "%d\n", paths[i], val); - goto err; - } - - finfo->fd[val] = open(paths[i], O_RDONLY); - if (finfo->fd[val] < 0) - goto err; - - /* - * Get the creation mode for the file. - */ - if (fstat(finfo->fd[val], &sbuf) < 0) - goto err; - if (finfo->mode == INVALID_MODE) { - finfo->mode = sbuf.st_mode; - } else if (sbuf.st_mode != finfo->mode) { - fprintf(stderr, "ERROR: %s: invalid mode\n", paths[i]); - goto err; - } - } - - free(stripe_count_attr); - free(stripe_size_attr); - free(stripe_index_attr); - free(stripe_coalesce_attr); - - return finfo; + int i, val, tmp; + struct stat sbuf; + char *stripe_count_attr = NULL; + char *stripe_size_attr = NULL; + char *stripe_index_attr = NULL; + char *stripe_coalesce_attr = NULL; + struct file_stripe_info *finfo = NULL; + + for (i = 0; i < count; i++) { + if (!paths[i]) + goto err; + + /* + * Check the stripe count first so we can allocate the info + * struct with the appropriate number of fds. + */ + if (get_attr(paths[i], ATTRNAME_STRIPE_COUNT, &stripe_count_attr, + &val) != 1) { + fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i], + ATTRNAME_STRIPE_COUNT); + goto err; + } + if (!finfo) { + finfo = alloc_file_stripe_info(val); + if (!finfo) + goto err; + + if (val != count) + fprintf(stderr, + "WARNING: %s: stripe-count " + "(%d) != file count (%d). Result may " + "be incomplete.\n", + paths[i], val, count); + + finfo->stripe_count = val; + } else if (val != finfo->stripe_count) { + fprintf(stderr, + "ERROR %s: invalid stripe count: %d " + "(expected %d)\n", + paths[i], val, finfo->stripe_count); + goto err; + } + + /* + * Get and validate the chunk size. + */ + if (get_attr(paths[i], ATTRNAME_STRIPE_SIZE, &stripe_size_attr, &val) != + 1) { + fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i], + ATTRNAME_STRIPE_SIZE); + goto err; + } + + if (!finfo->stripe_size) { + finfo->stripe_size = val; + } else if (val != finfo->stripe_size) { + fprintf(stderr, + "ERROR: %s: invalid stripe size: %d " + "(expected %d)\n", + paths[i], val, finfo->stripe_size); + goto err; + } + + /* + * stripe-coalesce is a backward compatible attribute. If the + * attribute does not exist, assume a value of zero for the + * traditional stripe format. + */ + tmp = get_attr(paths[i], ATTRNAME_STRIPE_COALESCE, + &stripe_coalesce_attr, &val); + if (!tmp) { + val = 0; + } else if (tmp != 1) { + fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i], + ATTRNAME_STRIPE_COALESCE); + goto err; + } + + if (finfo->coalesce == INVALID_FD) { + finfo->coalesce = val; + } else if (val != finfo->coalesce) { + fprintf(stderr, "ERROR: %s: invalid coalesce flag\n", paths[i]); + goto err; + } + + /* + * Get/validate the stripe index and open the file in the + * appropriate fd slot. + */ + if (get_attr(paths[i], ATTRNAME_STRIPE_INDEX, &stripe_index_attr, + &val) != 1) { + fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i], + ATTRNAME_STRIPE_INDEX); + goto err; + } + if (finfo->fd[val] != INVALID_FD) { + fprintf(stderr, + "ERROR: %s: duplicate stripe index: " + "%d\n", + paths[i], val); + goto err; + } + + finfo->fd[val] = open(paths[i], O_RDONLY); + if (finfo->fd[val] < 0) + goto err; + + /* + * Get the creation mode for the file. + */ + if (fstat(finfo->fd[val], &sbuf) < 0) + goto err; + if (finfo->mode == INVALID_MODE) { + finfo->mode = sbuf.st_mode; + } else if (sbuf.st_mode != finfo->mode) { + fprintf(stderr, "ERROR: %s: invalid mode\n", paths[i]); + goto err; + } + } + + free(stripe_count_attr); + free(stripe_size_attr); + free(stripe_index_attr); + free(stripe_coalesce_attr); + + return finfo; err: - free(stripe_count_attr); - free(stripe_size_attr); - free(stripe_index_attr); - free(stripe_coalesce_attr); + free(stripe_count_attr); + free(stripe_size_attr); + free(stripe_index_attr); + free(stripe_coalesce_attr); - if (finfo) { - close_files(finfo); - free(finfo); - } + if (finfo) { + close_files(finfo); + free(finfo); + } - return NULL; + return NULL; } static int close_files(struct file_stripe_info *finfo) { - int i, ret; + int i, ret; - if (!finfo) - return -1; + if (!finfo) + return -1; - for (i = 0; i < finfo->stripe_count; i++) { - if (finfo->fd[i] == INVALID_FD) - continue; + for (i = 0; i < finfo->stripe_count; i++) { + if (finfo->fd[i] == INVALID_FD) + continue; - ret = close(finfo->fd[i]); - if (ret < 0) - return ret; - } + ret = close(finfo->fd[i]); + if (ret < 0) + return ret; + } - return ret; + return ret; } /* @@ -351,43 +357,43 @@ close_files(struct file_stripe_info *finfo) static int generate_file_coalesce(int target, struct file_stripe_info *finfo) { - char *buf; - int ret = 0; - int r, w, i; - - buf = malloc(finfo->stripe_size); - if (!buf) - return -1; - - i = 0; - while (1) { - if (finfo->fd[i] == INVALID_FD) { - if (lseek(target, finfo->stripe_size, SEEK_CUR) < 0) - break; - - i = (i + 1) % finfo->stripe_count; - continue; - } - - r = read(finfo->fd[i], buf, finfo->stripe_size); - if (r < 0) { - ret = r; - break; - } - if (!r) - break; - - w = write(target, buf, r); - if (w < 0) { - ret = w; - break; - } - - i = (i + 1) % finfo->stripe_count; - } - - free(buf); - return ret; + char *buf; + int ret = 0; + int r, w, i; + + buf = malloc(finfo->stripe_size); + if (!buf) + return -1; + + i = 0; + while (1) { + if (finfo->fd[i] == INVALID_FD) { + if (lseek(target, finfo->stripe_size, SEEK_CUR) < 0) + break; + + i = (i + 1) % finfo->stripe_count; + continue; + } + + r = read(finfo->fd[i], buf, finfo->stripe_size); + if (r < 0) { + ret = r; + break; + } + if (!r) + break; + + w = write(target, buf, r); + if (w < 0) { + ret = w; + break; + } + + i = (i + 1) % finfo->stripe_count; + } + + free(buf); + return ret; } /* @@ -398,97 +404,100 @@ generate_file_coalesce(int target, struct file_stripe_info *finfo) static int generate_file_traditional(int target, struct file_stripe_info *finfo) { - int i, j, max_ret, ret; - char buf[finfo->stripe_count][4096]; - - do { - char newbuf[4096] = {0, }; - - max_ret = 0; - for (i = 0; i < finfo->stripe_count; i++) { - memset(buf[i], 0, 4096); - ret = read(finfo->fd[i], buf[i], 4096); - if (ret > max_ret) - max_ret = ret; - } - for (i = 0; i < max_ret; i++) - for (j = 0; j < finfo->stripe_count; j++) - newbuf[i] |= buf[j][i]; - write(target, newbuf, max_ret); - } while (max_ret); - - return 0; + int i, j, max_ret, ret; + char buf[finfo->stripe_count][4096]; + + do { + char newbuf[4096] = { + 0, + }; + + max_ret = 0; + for (i = 0; i < finfo->stripe_count; i++) { + memset(buf[i], 0, 4096); + ret = read(finfo->fd[i], buf[i], 4096); + if (ret > max_ret) + max_ret = ret; + } + for (i = 0; i < max_ret; i++) + for (j = 0; j < finfo->stripe_count; j++) + newbuf[i] |= buf[j][i]; + write(target, newbuf, max_ret); + } while (max_ret); + + return 0; } static int generate_file(int target, struct file_stripe_info *finfo) { - if (finfo->coalesce) - return generate_file_coalesce(target, finfo); + if (finfo->coalesce) + return generate_file_coalesce(target, finfo); - return generate_file_traditional(target, finfo); + return generate_file_traditional(target, finfo); } static void usage(char *name) { - fprintf(stderr, "Usage: %s [-o <outputfile>] <inputfile1> " - "<inputfile2> ...\n", name); + fprintf(stderr, + "Usage: %s [-o <outputfile>] <inputfile1> " + "<inputfile2> ...\n", + name); } int main(int argc, char *argv[]) { - int file_count, opt; - char *opath = NULL; - int targetfd; - struct file_stripe_info *finfo; - - while ((opt = getopt(argc, argv, "o:")) != -1) { - switch (opt) { - case 'o': - opath = optarg; - break; - default: - usage(argv[0]); - return -1; - } - } - - file_count = argc - optind; - - if (!opath || !file_count) { - usage(argv[0]); - return -1; - } - - finfo = validate_and_open_files(&argv[optind], file_count); - if (!finfo) - goto err; - - targetfd = open(opath, O_RDWR|O_CREAT, finfo->mode); - if (targetfd < 0) - goto err; - - if (generate_file(targetfd, finfo) < 0) - goto err; - - if (fsync(targetfd) < 0) - fprintf(stderr, "ERROR: %s\n", strerror(errno)); - if (close(targetfd) < 0) - fprintf(stderr, "ERROR: %s\n", strerror(errno)); - - close_files(finfo); - free(finfo); - - return 0; + int file_count, opt; + char *opath = NULL; + int targetfd; + struct file_stripe_info *finfo; + + while ((opt = getopt(argc, argv, "o:")) != -1) { + switch (opt) { + case 'o': + opath = optarg; + break; + default: + usage(argv[0]); + return -1; + } + } + + file_count = argc - optind; + + if (!opath || !file_count) { + usage(argv[0]); + return -1; + } + + finfo = validate_and_open_files(&argv[optind], file_count); + if (!finfo) + goto err; + + targetfd = open(opath, O_RDWR | O_CREAT, finfo->mode); + if (targetfd < 0) + goto err; + + if (generate_file(targetfd, finfo) < 0) + goto err; + + if (fsync(targetfd) < 0) + fprintf(stderr, "ERROR: %s\n", strerror(errno)); + if (close(targetfd) < 0) + fprintf(stderr, "ERROR: %s\n", strerror(errno)); + + close_files(finfo); + free(finfo); + + return 0; err: - if (finfo) { - close_files(finfo); - free(finfo); - } + if (finfo) { + close_files(finfo); + free(finfo); + } - return -1; + return -1; } - diff --git a/extras/systemd/Makefile.am b/extras/systemd/Makefile.am index 1fbb74593bb..61446a9b84a 100644 --- a/extras/systemd/Makefile.am +++ b/extras/systemd/Makefile.am @@ -1,11 +1,17 @@ +CLEANFILES = glusterd.service glustereventsd.service glusterfssharedstorage.service gluster-ta-volume.service +EXTRA_DIST = glusterd.service.in glustereventsd.service.in glusterfssharedstorage.service.in gluster-ta-volume.service.in -CLEANFILES = +if USE_SYSTEMD +systemd_DATA = gluster-ta-volume.service +endif -SYSTEMD_DIR = @systemddir@ - -install-exec-local: - @if [ -d $(SYSTEMD_DIR) ]; then \ - $(MKDIR_P) $(DESTDIR)$(SYSTEMD_DIR); \ - $(INSTALL_PROGRAM) glusterd.service $(DESTDIR)$(SYSTEMD_DIR)/; \ - fi +if WITH_SERVER +if USE_SYSTEMD +# systemddir is already defined through configure.ac +systemd_DATA += glusterd.service glusterfssharedstorage.service +if BUILD_EVENTS +systemd_DATA += glustereventsd.service +endif +endif +endif diff --git a/extras/systemd/gluster-ta-volume.service.in b/extras/systemd/gluster-ta-volume.service.in new file mode 100644 index 00000000000..2802bca05bf --- /dev/null +++ b/extras/systemd/gluster-ta-volume.service.in @@ -0,0 +1,13 @@ +[Unit] +Description=GlusterFS, Thin-arbiter process to maintain quorum for replica volume +After=network.target + +[Service] +Environment="LOG_LEVEL=WARNING" +ExecStart=@prefix@/sbin/glusterfsd -N --volfile-id ta -f @GLUSTERD_WORKDIR@/thin-arbiter/thin-arbiter.vol --brick-port 24007 --xlator-option ta-server.transport.socket.listen-port=24007 -LWARNING +Restart=always +KillMode=process +SuccessExitStatus=15 + +[Install] +WantedBy=multi-user.target diff --git a/extras/systemd/glusterd.service.in b/extras/systemd/glusterd.service.in index a5d260d86e7..abb0d82911f 100644 --- a/extras/systemd/glusterd.service.in +++ b/extras/systemd/glusterd.service.in @@ -1,14 +1,26 @@ [Unit] Description=GlusterFS, a clustered file-system server -After=network.target rpcbind.service +Documentation=man:glusterd(8) +StartLimitBurst=6 +StartLimitIntervalSec=3600 +Requires=@RPCBIND_SERVICE@ +After=network.target @RPCBIND_SERVICE@ Before=network-online.target [Service] Type=forking PIDFile=@localstatedir@/run/glusterd.pid LimitNOFILE=65536 -ExecStart=@prefix@/sbin/glusterd -p @localstatedir@/run/glusterd.pid +Environment="LOG_LEVEL=INFO" +EnvironmentFile=-@SYSCONF_DIR@/sysconfig/glusterd +ExecStart=@prefix@/sbin/glusterd -p @localstatedir@/run/glusterd.pid --log-level $LOG_LEVEL $GLUSTERD_OPTIONS KillMode=process +TimeoutSec=300 +SuccessExitStatus=15 +Restart=on-abnormal +RestartSec=60 +StartLimitBurst=6 +StartLimitInterval=3600 [Install] WantedBy=multi-user.target diff --git a/extras/systemd/glustereventsd.service.in b/extras/systemd/glustereventsd.service.in new file mode 100644 index 00000000000..f80b78199f6 --- /dev/null +++ b/extras/systemd/glustereventsd.service.in @@ -0,0 +1,16 @@ +[Unit] +Description=Gluster Events Notifier +After=network.target +Documentation=man:glustereventsd(8) + + +[Service] +Environment=PYTHONPATH=@BUILD_PYTHON_SITE_PACKAGES_EXPANDED@:$PYTHONPATH +Type=simple +ExecStart=@SBIN_DIR@/glustereventsd --pid-file @localstatedir@/run/glustereventsd.pid +ExecReload=/bin/kill -SIGUSR2 $MAINPID +KillMode=control-group +PIDFile=@localstatedir@/run/glustereventsd.pid + +[Install] +WantedBy=multi-user.target diff --git a/extras/systemd/glusterfssharedstorage.service.in b/extras/systemd/glusterfssharedstorage.service.in new file mode 100644 index 00000000000..723ff49afb7 --- /dev/null +++ b/extras/systemd/glusterfssharedstorage.service.in @@ -0,0 +1,13 @@ +[Unit] +Description=Mount glusterfs sharedstorage +Requires=glusterd.service remote-fs-pre.target local-fs.target + +[Service] +Type=forking +ExecStart=@GLUSTERFS_LIBEXECDIR@/mount-shared-storage.sh +Restart=on-failure +RestartSec=3 +RestartForceExitStatus=1 + +[Install] +WantedBy=multi-user.target diff --git a/extras/test/ld-preload-test/ld-preload-lib.c b/extras/test/ld-preload-test/ld-preload-lib.c index 88afd14c3e7..d120c053a69 100644 --- a/extras/test/ld-preload-test/ld-preload-lib.c +++ b/extras/test/ld-preload-test/ld-preload-lib.c @@ -34,594 +34,582 @@ #include <fcntl.h> #include <sys/stat.h> #include <dirent.h> -#include <attr/xattr.h> +#include <sys/xattr.h> #include <sys/sendfile.h> /* Err number that is assigned to errno so that test application can * verify that the function was intercepted correctly. */ -#define PRELOAD_ERRNO_VERF 6449 -#define set_errno() (errno = PRELOAD_ERRNO_VERF) +#define PRELOAD_ERRNO_VERF 6449 +#define set_errno() (errno = PRELOAD_ERRNO_VERF) -inline void -intercept (char *call, int tabs) +void +intercept(char *call, int tabs) { - while (tabs > 0) { - fprintf (stdout, "\t"); - --tabs; - } + while (tabs > 0) { + fprintf(stdout, "\t"); + --tabs; + } - fprintf (stdout, "Intercepted by %s", call); + fprintf(stdout, "Intercepted by %s", call); } int -creat64 (const char *pathname, mode_t mode) +creat64(const char *pathname, mode_t mode) { - intercept ("creat64", 2); - set_errno (); - return -1; + intercept("creat64", 2); + set_errno(); + return -1; } int -creat (const char *pathname, mode_t mode) +creat(const char *pathname, mode_t mode) { - intercept ("creat", 2); - set_errno (); - return -1; + intercept("creat", 2); + set_errno(); + return -1; } - int -close (int fd) +close(int fd) { - intercept ("close", 2); - set_errno (); - return -1; + intercept("close", 2); + set_errno(); + return -1; } int -open64 (const char *pathname, int flags, ...) +open64(const char *pathname, int flags, ...) { - intercept ("open64", 2); - set_errno (); - return -1; + intercept("open64", 2); + set_errno(); + return -1; } - int -open (const char *pathname, int flags, ...) +open(const char *pathname, int flags, ...) { - intercept ("open", 2); - set_errno (); - return -1; + intercept("open", 2); + set_errno(); + return -1; } ssize_t -read (int fd, void *buf, size_t count) +read(int fd, void *buf, size_t count) { - intercept ("read", 2); - set_errno (); - return -1; + intercept("read", 2); + set_errno(); + return -1; } ssize_t -readv (int fd, const struct iovec *vector, int count) +readv(int fd, const struct iovec *vector, int count) { - intercept ("readv", 2); - set_errno (); - return -1; + intercept("readv", 2); + set_errno(); + return -1; } ssize_t -pread (int fd, void *buf, size_t count, unsigned long offset) +pread(int fd, void *buf, size_t count, unsigned long offset) { - intercept ("pread", 2); - set_errno (); - return -1; + intercept("pread", 2); + set_errno(); + return -1; } - ssize_t -pread64 (int fd, void *buf, size_t count, uint64_t offset) +pread64(int fd, void *buf, size_t count, uint64_t offset) { - intercept ("pread64", 2); - set_errno (); - return -1; + intercept("pread64", 2); + set_errno(); + return -1; } ssize_t -write (int fd, const void *buf, size_t count) +write(int fd, const void *buf, size_t count) { - intercept ("write", 2); - set_errno (); - return -1; + intercept("write", 2); + set_errno(); + return -1; } ssize_t -writev (int fd, const struct iovec *vector, int count) +writev(int fd, const struct iovec *vector, int count) { - intercept ("writev", 2); - set_errno (); - return -1; + intercept("writev", 2); + set_errno(); + return -1; } ssize_t -pwrite (int fd, const void *buf, size_t count, unsigned long offset) +pwrite(int fd, const void *buf, size_t count, unsigned long offset) { - intercept ("pwrite", 2); - set_errno (); - return -1; + intercept("pwrite", 2); + set_errno(); + return -1; } ssize_t -pwrite64 (int fd, const void *buf, size_t count, uint64_t offset) +pwrite64(int fd, const void *buf, size_t count, uint64_t offset) { - intercept ("pwrite64", 2); - set_errno (); - return -1; + intercept("pwrite64", 2); + set_errno(); + return -1; } - off_t -lseek (int fildes, unsigned long offset, int whence) +lseek(int fildes, unsigned long offset, int whence) { - intercept ("lseek", 2); - set_errno (); - return -1; + intercept("lseek", 2); + set_errno(); + return -1; } off_t -lseek64 (int fildes, uint64_t offset, int whence) +lseek64(int fildes, uint64_t offset, int whence) { - intercept ("lseek64", 2); - set_errno (); - return -1; + intercept("lseek64", 2); + set_errno(); + return -1; } - int -dup (int fd) +dup(int fd) { - intercept ("dup", 2); - set_errno (); - return -1; + intercept("dup", 2); + set_errno(); + return -1; } int -dup2 (int oldfd, int newfd) +dup2(int oldfd, int newfd) { - intercept ("dup2", 2); - set_errno (); - return -1; + intercept("dup2", 2); + set_errno(); + return -1; } int -mkdir (const char *pathname, mode_t mode) +mkdir(const char *pathname, mode_t mode) { - intercept ("mkdir", 2); - set_errno (); - return -1; + intercept("mkdir", 2); + set_errno(); + return -1; } int -rmdir (const char *pathname) +rmdir(const char *pathname) { - intercept ("rmdir", 2); - set_errno (); - return -1; + intercept("rmdir", 2); + set_errno(); + return -1; } int -chmod (const char *pathname, mode_t mode) +chmod(const char *pathname, mode_t mode) { - intercept ("chmod", 2); - set_errno (); - return -1; + intercept("chmod", 2); + set_errno(); + return -1; } int -chown (const char *pathname, uid_t owner, gid_t group) +chown(const char *pathname, uid_t owner, gid_t group) { - intercept ("chown", 2); - set_errno (); - return -1; + intercept("chown", 2); + set_errno(); + return -1; } int -fchmod (int fd, mode_t mode) +fchmod(int fd, mode_t mode) { - intercept ("fchmod", 2); - set_errno (); - return -1; + intercept("fchmod", 2); + set_errno(); + return -1; } int -fchown (int fd, uid_t uid, gid_t gid) +fchown(int fd, uid_t uid, gid_t gid) { - intercept ("fchown", 2); - set_errno (); - return -1; + intercept("fchown", 2); + set_errno(); + return -1; } -int fsync (int fd) +int +fsync(int fd) { - intercept ("fsync", 2); - set_errno (); - return -1; + intercept("fsync", 2); + set_errno(); + return -1; } - int -ftruncate (int fd, off_t length) +ftruncate(int fd, off_t length) { - intercept ("ftruncate", 1); - set_errno (); - return -1; + intercept("ftruncate", 1); + set_errno(); + return -1; } - int -ftruncate64 (int fd, off_t length) +ftruncate64(int fd, off_t length) { - intercept ("ftruncate64", 1); - set_errno (); - return -1; + intercept("ftruncate64", 1); + set_errno(); + return -1; } int -link (const char *oldpath, const char *newname) +link(const char *oldpath, const char *newname) { - intercept ("link", 2); - set_errno (); - return -1; + intercept("link", 2); + set_errno(); + return -1; } int -rename (const char *oldpath, const char *newpath) +rename(const char *oldpath, const char *newpath) { - intercept ("rename", 2); - set_errno (); - return -1; + intercept("rename", 2); + set_errno(); + return -1; } int -utimes (const char *path, const struct timeval times[2]) +utimes(const char *path, const struct timeval times[2]) { - intercept ("utimes", 2); - set_errno (); - return -1; + intercept("utimes", 2); + set_errno(); + return -1; } int -utime (const char *path, const struct utimbuf *buf) +futimes(int fd, const struct timeval times[2]) { - intercept ("utime", 2); - set_errno (); - return -1; + intercept("futimes", 2); + set_errno(); + return -1; } - int -mknod (const char *path, mode_t mode, dev_t dev) +utime(const char *path, const struct utimbuf *buf) { - intercept ("mknod", 2); - set_errno (); - return -1; + intercept("utime", 2); + set_errno(); + return -1; } int -__xmknod (int ver, const char *path, mode_t mode, dev_t *dev) +mknod(const char *path, mode_t mode, dev_t dev) { - intercept ("__xmknod", 2); - set_errno (); - return -1; + intercept("mknod", 2); + set_errno(); + return -1; } int -mkfifo (const char *path, mode_t mode) +__xmknod(int ver, const char *path, mode_t mode, dev_t *dev) { - intercept ("mkfifo", 2); - set_errno (); - return -1; + intercept("__xmknod", 2); + set_errno(); + return -1; } int -unlink (const char *path) +mkfifo(const char *path, mode_t mode) { - intercept ("unlink", 2); - set_errno (); - return -1; + intercept("mkfifo", 2); + set_errno(); + return -1; } - int -symlink (const char *oldpath, const char *newpath) +unlink(const char *path) { - intercept ("symlink", 2); - set_errno (); - return -1; + intercept("unlink", 2); + set_errno(); + return -1; } int -readlink (const char *path, char *buf, size_t bufsize) +symlink(const char *oldpath, const char *newpath) { - intercept ("readlink", 1); - set_errno (); - return -1; + intercept("symlink", 2); + set_errno(); + return -1; } +int +readlink(const char *path, char *buf, size_t bufsize) +{ + intercept("readlink", 1); + set_errno(); + return -1; +} char * -realpath (const char *path, char *resolved) +realpath(const char *path, char *resolved) { - intercept ("realpath", 1); - set_errno (); - return NULL; + intercept("realpath", 1); + set_errno(); + return NULL; } - DIR * -opendir (const char *path) +opendir(const char *path) { - intercept ("opendir", 2); - set_errno (); - return NULL; + intercept("opendir", 2); + set_errno(); + return NULL; } - struct dirent * -readdir (DIR *dir) +readdir(DIR *dir) { - intercept ("readdir\t", 2); - set_errno (); - return NULL; + intercept("readdir\t", 2); + set_errno(); + return NULL; } struct dirent * -readdir64 (DIR *dir) +readdir64(DIR *dir) { - intercept ("readdir64", 2); - set_errno (); - return NULL; + intercept("readdir64", 2); + set_errno(); + return NULL; } - int -readdir_r (DIR *dir, struct dirent *entry, struct dirent **result) +readdir_r(DIR *dir, struct dirent *entry, struct dirent **result) { - intercept ("readdir_r", 1); - set_errno (); - return -1; + intercept("readdir_r", 1); + set_errno(); + return -1; } int -readdir64_r (DIR *dir, struct dirent *entry, struct dirent **result) +readdir64_r(DIR *dir, struct dirent *entry, struct dirent **result) { - intercept ("readdir64_r", 1); - set_errno (); - return -1; + intercept("readdir64_r", 1); + set_errno(); + return -1; } - int -closedir (DIR *dh) +closedir(DIR *dh) { - intercept ("closedir", 1); - set_errno (); - return -1; + intercept("closedir", 1); + set_errno(); + return -1; } int -__xstat (int ver, const char *path, struct stat *buf) +__xstat(int ver, const char *path, struct stat *buf) { - intercept ("__xstat\t", 2); - set_errno (); - return -1; + intercept("__xstat\t", 2); + set_errno(); + return -1; } - int -__xstat64 (int ver, const char *path, struct stat *buf) +__xstat64(int ver, const char *path, struct stat *buf) { - intercept ("__xstat64", 2); - set_errno (); - return -1; + intercept("__xstat64", 2); + set_errno(); + return -1; } int -stat (const char *path, struct stat *buf) +stat(const char *path, struct stat *buf) { - intercept ("stat", 2); - set_errno (); - return -1; + intercept("stat", 2); + set_errno(); + return -1; } int -stat64 (const char *path, struct stat *buf) +stat64(const char *path, struct stat *buf) { - intercept ("stat64", 2); - set_errno (); - return -1; + intercept("stat64", 2); + set_errno(); + return -1; } int -__fxstat (int ver, int fd, struct stat *buf) +__fxstat(int ver, int fd, struct stat *buf) { - intercept ("__fxstat\t", 2); - set_errno (); - return -1; + intercept("__fxstat\t", 2); + set_errno(); + return -1; } - int -__fxstat64 (int ver, int fd, struct stat *buf) +__fxstat64(int ver, int fd, struct stat *buf) { - intercept ("__fxstat64", 2); - set_errno (); - return -1; + intercept("__fxstat64", 2); + set_errno(); + return -1; } int -fstat (int fd, struct stat *buf) +fstat(int fd, struct stat *buf) { - intercept ("fstat", 2); - set_errno (); - return -1; + intercept("fstat", 2); + set_errno(); + return -1; } int -fstat64 (int fd , struct stat *buf) +fstat64(int fd, struct stat *buf) { - intercept ("fstat64", 2); - set_errno (); - return -1; + intercept("fstat64", 2); + set_errno(); + return -1; } int -__lxstat (int ver, const char *path, struct stat *buf) +__lxstat(int ver, const char *path, struct stat *buf) { - intercept ("__lxstat\t", 2); - set_errno (); - return -1; + intercept("__lxstat\t", 2); + set_errno(); + return -1; } int -__lxstat64 (int ver, const char *path, struct stat *buf) +__lxstat64(int ver, const char *path, struct stat *buf) { - intercept ("__lxstat64", 2); - set_errno (); - return -1; + intercept("__lxstat64", 2); + set_errno(); + return -1; } int -lstat (const char *path, struct stat *buf) +lstat(const char *path, struct stat *buf) { - intercept ("lstat", 2); - set_errno (); - return -1; + intercept("lstat", 2); + set_errno(); + return -1; } int -lstat64 (const char *path, struct stat *buf) +lstat64(const char *path, struct stat *buf) { - intercept ("lstat64", 2); - set_errno (); - return -1; + intercept("lstat64", 2); + set_errno(); + return -1; } int -statfs (const char *path, struct statfs *buf) +statfs(const char *path, struct statfs *buf) { - intercept ("statfs", 2); - set_errno (); - return -1; + intercept("statfs", 2); + set_errno(); + return -1; } - int -statfs64 (const char *path, struct statfs *buf) +statfs64(const char *path, struct statfs *buf) { - intercept ("statfs64", 2); - set_errno (); - return -1; + intercept("statfs64", 2); + set_errno(); + return -1; } int -statvfs (const char *path, struct statvfs *buf) +statvfs(const char *path, struct statvfs *buf) { - intercept ("statvfs\t", 2); - set_errno (); - return -1; + intercept("statvfs\t", 2); + set_errno(); + return -1; } - int -statvfs64 (const char *path, struct statvfs *buf) +statvfs64(const char *path, struct statvfs *buf) { - intercept ("statvfs64", 2); - set_errno (); - return -1; + intercept("statvfs64", 2); + set_errno(); + return -1; } ssize_t -getxattr (const char *path, const char *name, void *value, size_t size) +getxattr(const char *path, const char *name, void *value, size_t size) { - intercept ("getxattr", 1); - set_errno (); - return -1; + intercept("getxattr", 1); + set_errno(); + return -1; } ssize_t -lgetxattr (const char *path, const char *name, void *value, size_t size) +lgetxattr(const char *path, const char *name, void *value, size_t size) { - intercept ("lgetxattr", 1); - set_errno (); - return -1; + intercept("lgetxattr", 1); + set_errno(); + return -1; } - int -remove (const char* path) +remove(const char *path) { - intercept ("remove", 2); - set_errno (); - return -1; + intercept("remove", 2); + set_errno(); + return -1; } int -lchown (const char *path, uid_t owner, gid_t group) +lchown(const char *path, uid_t owner, gid_t group) { - intercept ("lchown", 2); - set_errno (); - return -1; + intercept("lchown", 2); + set_errno(); + return -1; } void -rewinddir (DIR *dirp) +rewinddir(DIR *dirp) { - intercept ("rewinddir", 1); - set_errno (); - return; + intercept("rewinddir", 1); + set_errno(); + return; } void -seekdir (DIR *dirp, off_t offset) +seekdir(DIR *dirp, off_t offset) { - intercept ("seekdir", 2); - set_errno (); - return; + intercept("seekdir", 2); + set_errno(); + return; } off_t -telldir (DIR *dirp) +telldir(DIR *dirp) { - intercept ("telldir", 2); - set_errno (); - return -1; + intercept("telldir", 2); + set_errno(); + return -1; } ssize_t -sendfile (int out_fd, int in_fd, off_t *offset, size_t count) +sendfile(int out_fd, int in_fd, off_t *offset, size_t count) { - intercept ("sendfile\t", 1); - set_errno (); - return -1; + intercept("sendfile\t", 1); + set_errno(); + return -1; } ssize_t -sendfile64 (int out_fd, int in_fd, off_t *offset, size_t count) +sendfile64(int out_fd, int in_fd, off_t *offset, size_t count) { - intercept ("sendfile64", 1); - set_errno (); - return -1; + intercept("sendfile64", 1); + set_errno(); + return -1; } - int -fcntl (int fd, int cmd, ...) +fcntl(int fd, int cmd, ...) { - intercept ("fcntl", 2); - set_errno (); - return -1; + intercept("fcntl", 2); + set_errno(); + return -1; } - diff --git a/extras/test/ld-preload-test/ld-preload-test.c b/extras/test/ld-preload-test/ld-preload-test.c index 78772f59818..54dde8c7d54 100644 --- a/extras/test/ld-preload-test/ld-preload-test.c +++ b/extras/test/ld-preload-test/ld-preload-test.c @@ -46,322 +46,313 @@ #include <sys/uio.h> #include <utime.h> #include <sys/time.h> -#include <attr/xattr.h> +#include <sys/xattr.h> #include <sys/sendfile.h> - -#define PRELOAD_ERRNO_VERF 6449 -inline void +#define PRELOAD_ERRNO_VERF 6449 +void check_err(int ret, char *call, int tabs) { - while (tabs > 0) { - fprintf (stdout, "\t"); - --tabs; - } - if (ret != -1) { - fprintf (stdout, "Not intercepted: %s\n", call); - return; - } - - if (errno != PRELOAD_ERRNO_VERF) { - fprintf (stdout, "Not intercepted: %s: err: %s\n", call, - strerror (errno)); - return; - } + while (tabs > 0) { + fprintf(stdout, "\t"); + --tabs; + } + if (ret != -1) { + fprintf(stdout, "Not intercepted: %s\n", call); + return; + } - fprintf (stdout, "Intercept verified: %s\n", call); + if (errno != PRELOAD_ERRNO_VERF) { + fprintf(stdout, "Not intercepted: %s: err: %s\n", call, + strerror(errno)); return; + } + + fprintf(stdout, "Intercept verified: %s\n", call); + return; } void -usage (FILE *fp) +usage(FILE *fp) { - fprintf (fp, "Usage: ld-preload-test <Options>\n"); - fprintf (fp, "Options\n"); - fprintf (fp, "\t--path\t\tPathname is used as the file/directory" - " created for the test.\n"); - + fprintf(fp, "Usage: ld-preload-test <Options>\n"); + fprintf(fp, "Options\n"); + fprintf(fp, + "\t--path\t\tPathname is used as the file/directory" + " created for the test.\n"); } - int -run_file_tests (char *testfile) +run_file_tests(char *testfile) { - int ret = -1; - struct stat buf; + int ret = -1; + struct stat buf; - assert (testfile); - fprintf (stdout, "Testing creat"); - ret = creat (testfile, S_IRWXU); - check_err (ret, "creat", 2); + assert(testfile); + fprintf(stdout, "Testing creat"); + ret = creat(testfile, S_IRWXU); + check_err(ret, "creat", 2); - fprintf (stdout, "Testing close"); - ret = close (ret); - check_err (ret, "close", 2); + fprintf(stdout, "Testing close"); + ret = close(ret); + check_err(ret, "close", 2); - fprintf (stdout, "Testing open"); - ret = open (testfile, O_RDONLY); - check_err (ret, "open", 2); + fprintf(stdout, "Testing open"); + ret = open(testfile, O_RDONLY); + check_err(ret, "open", 2); - fprintf (stdout, "Testing read"); - ret = read (0, NULL, 0); - check_err (ret, "read", 2); + fprintf(stdout, "Testing read"); + ret = read(0, NULL, 0); + check_err(ret, "read", 2); - fprintf (stdout, "Testing readv"); - ret = readv (0, NULL, 0); - check_err (ret, "readv", 2); + fprintf(stdout, "Testing readv"); + ret = readv(0, NULL, 0); + check_err(ret, "readv", 2); - fprintf (stdout, "Testing pread"); - ret = pread (0, NULL, 0, 0); - check_err (ret, "pread", 2); + fprintf(stdout, "Testing pread"); + ret = pread(0, NULL, 0, 0); + check_err(ret, "pread", 2); - fprintf (stdout, "Testing write"); - ret = write (0, NULL, 0); - check_err (ret, "write", 2); + fprintf(stdout, "Testing write"); + ret = write(0, NULL, 0); + check_err(ret, "write", 2); - fprintf (stdout, "Testing writev"); - ret = writev (0, NULL, 0); - check_err (ret, "writev", 2); + fprintf(stdout, "Testing writev"); + ret = writev(0, NULL, 0); + check_err(ret, "writev", 2); - fprintf (stdout, "Testing pwrite"); - ret = pwrite (0, NULL, 0, 0); - check_err (ret, "pwrite", 2); + fprintf(stdout, "Testing pwrite"); + ret = pwrite(0, NULL, 0, 0); + check_err(ret, "pwrite", 2); - fprintf (stdout, "Testing lseek"); - ret = lseek (0, 0, 0); - check_err (ret, "lseek", 2); + fprintf(stdout, "Testing lseek"); + ret = lseek(0, 0, 0); + check_err(ret, "lseek", 2); - fprintf (stdout, "Testing dup"); - ret = dup (0); - check_err (ret, "dup", 2); + fprintf(stdout, "Testing dup"); + ret = dup(0); + check_err(ret, "dup", 2); - fprintf (stdout, "Testing dup2"); - ret = dup2 (0, 0); - check_err (ret, "dup2", 2); + fprintf(stdout, "Testing dup2"); + ret = dup2(0, 0); + check_err(ret, "dup2", 2); - fprintf (stdout, "Testing fchmod"); - ret = fchmod (0, 0); - check_err (ret, "fchmod", 2); + fprintf(stdout, "Testing fchmod"); + ret = fchmod(0, 0); + check_err(ret, "fchmod", 2); - fprintf (stdout, "Testing fchown"); - ret = fchown (0, 0, 0); - check_err (ret, "fchown", 2); + fprintf(stdout, "Testing fchown"); + ret = fchown(0, 0, 0); + check_err(ret, "fchown", 2); - fprintf (stdout, "Testing fsync"); - ret = fsync (0); - check_err (ret, "fsync", 2); + fprintf(stdout, "Testing fsync"); + ret = fsync(0); + check_err(ret, "fsync", 2); - fprintf (stdout, "Testing ftruncate"); - ret = ftruncate (0, 0); - check_err (ret, "ftruncate", 1); + fprintf(stdout, "Testing ftruncate"); + ret = ftruncate(0, 0); + check_err(ret, "ftruncate", 1); - fprintf (stdout, "Testing fstat"); - ret = fstat (0, &buf); - check_err (ret, "fstat", 1); + fprintf(stdout, "Testing fstat"); + ret = fstat(0, &buf); + check_err(ret, "fstat", 1); - fprintf (stdout, "Testing sendfile"); - ret = sendfile (0, 0, NULL, 0); - check_err (ret, "sendfile", 1); + fprintf(stdout, "Testing sendfile"); + ret = sendfile(0, 0, NULL, 0); + check_err(ret, "sendfile", 1); - fprintf (stdout, "Testing fcntl"); - ret = fcntl (0, 0, NULL); - check_err (ret, "fcntl", 2); + fprintf(stdout, "Testing fcntl"); + ret = fcntl(0, 0, NULL); + check_err(ret, "fcntl", 2); - fprintf (stdout, "Testing close"); - ret = close (ret); - check_err (ret, "close", 2); + fprintf(stdout, "Testing close"); + ret = close(ret); + check_err(ret, "close", 2); - fprintf (stdout, "Testing remove"); - ret = remove (testfile); - check_err (ret, "remove", 2); + fprintf(stdout, "Testing remove"); + ret = remove(testfile); + check_err(ret, "remove", 2); - return ret; + return ret; } - int -run_attr_tests (char *testfile) +run_attr_tests(char *testfile) { - int ret = -1; - char *res = NULL; - struct stat buf; - struct statfs sbuf; - struct statvfs svbuf; - - assert (testfile); - - fprintf (stdout, "Testing chmod"); - ret = chmod (testfile, 0); - check_err (ret, "chmod", 2); - - fprintf (stdout, "Testing chown"); - ret = chown (testfile, 0, 0); - check_err (ret, "chown", 2); - - fprintf (stdout, "Testing link"); - ret = link (testfile, testfile); - check_err (ret, "link", 2); - - fprintf (stdout, "Testing rename"); - ret = rename (testfile, testfile); - check_err (ret, "rename", 2); - - fprintf (stdout, "Testing utimes"); - ret = utimes (testfile, NULL); - check_err (ret, "utimes", 2); - - fprintf (stdout, "Testing utime"); - ret = utime (testfile, NULL); - check_err (ret, "utime", 2); - - fprintf (stdout, "Testing unlink"); - ret = unlink (testfile); - check_err (ret, "unlink", 2); - - fprintf (stdout, "Testing symlink"); - ret = symlink (testfile, testfile); - check_err (ret, "symlink", 2); - - fprintf (stdout, "Testing readlink"); - ret = readlink (testfile, testfile, 0); - check_err (ret, "readlink", 2); - - fprintf (stdout, "Testing realpath"); - ret = 0; - res = realpath ((const char *)testfile, testfile); - if (!res) - ret = -1; - check_err (ret, "realpath", 2); - - fprintf (stdout, "Testing stat"); - ret = stat (testfile, &buf); - check_err (ret, "stat", 1); - - fprintf (stdout, "Testing lstat"); - ret = lstat (testfile, &buf); - check_err (ret, "lstat", 1); - - fprintf (stdout, "Testing statfs"); - ret = statfs (testfile, &sbuf); - check_err (ret, "statfs", 2); - - fprintf (stdout, "Testing statvfs"); - ret = statvfs (testfile, &svbuf); - check_err (ret, "statvfs", 1); - - fprintf (stdout, "Testing getxattr"); - ret = getxattr (testfile, NULL, NULL, 0); - check_err (ret, "getxattr", 2); - - fprintf (stdout, "Testing lgetxattr"); - ret = lgetxattr (testfile, NULL, NULL, 0); - check_err (ret, "lgetxattr", 1); - - fprintf (stdout, "Testing lchown"); - ret = lchown (testfile, 0, 0); - check_err (ret, "lchown", 2); - return 0; + int ret = -1; + char *res = NULL; + struct stat buf; + struct statfs sbuf; + struct statvfs svbuf; + + assert(testfile); + + fprintf(stdout, "Testing chmod"); + ret = chmod(testfile, 0); + check_err(ret, "chmod", 2); + + fprintf(stdout, "Testing chown"); + ret = chown(testfile, 0, 0); + check_err(ret, "chown", 2); + + fprintf(stdout, "Testing link"); + ret = link(testfile, testfile); + check_err(ret, "link", 2); + + fprintf(stdout, "Testing rename"); + ret = rename(testfile, testfile); + check_err(ret, "rename", 2); + + fprintf(stdout, "Testing utimes"); + ret = utimes(testfile, NULL); + check_err(ret, "utimes", 2); + + fprintf(stdout, "Testing utime"); + ret = utime(testfile, NULL); + check_err(ret, "utime", 2); + + fprintf(stdout, "Testing unlink"); + ret = unlink(testfile); + check_err(ret, "unlink", 2); + + fprintf(stdout, "Testing symlink"); + ret = symlink(testfile, testfile); + check_err(ret, "symlink", 2); + + fprintf(stdout, "Testing readlink"); + ret = readlink(testfile, testfile, 0); + check_err(ret, "readlink", 2); + + fprintf(stdout, "Testing realpath"); + ret = 0; + res = realpath((const char *)testfile, testfile); + if (!res) + ret = -1; + check_err(ret, "realpath", 2); + + fprintf(stdout, "Testing stat"); + ret = stat(testfile, &buf); + check_err(ret, "stat", 1); + + fprintf(stdout, "Testing lstat"); + ret = lstat(testfile, &buf); + check_err(ret, "lstat", 1); + + fprintf(stdout, "Testing statfs"); + ret = statfs(testfile, &sbuf); + check_err(ret, "statfs", 2); + + fprintf(stdout, "Testing statvfs"); + ret = statvfs(testfile, &svbuf); + check_err(ret, "statvfs", 1); + + fprintf(stdout, "Testing getxattr"); + ret = getxattr(testfile, NULL, NULL, 0); + check_err(ret, "getxattr", 2); + + fprintf(stdout, "Testing lgetxattr"); + ret = lgetxattr(testfile, NULL, NULL, 0); + check_err(ret, "lgetxattr", 1); + + fprintf(stdout, "Testing lchown"); + ret = lchown(testfile, 0, 0); + check_err(ret, "lchown", 2); + return 0; } - int -run_dev_tests (char *testfile) +run_dev_tests(char *testfile) { - int ret = -1; + int ret = -1; - assert (testfile); + assert(testfile); - fprintf (stdout, "Testing mknod"); - ret = mknod (testfile, 0, 0); - check_err (ret, "mknod", 2); + fprintf(stdout, "Testing mknod"); + ret = mknod(testfile, 0, 0); + check_err(ret, "mknod", 2); - fprintf (stdout, "Testing mkfifo"); - ret = mkfifo (testfile, 0); - check_err (ret, "mkfifo", 2); - return 0; + fprintf(stdout, "Testing mkfifo"); + ret = mkfifo(testfile, 0); + check_err(ret, "mkfifo", 2); + return 0; } int -run_dir_tests (char *testpath) +run_dir_tests(char *testpath) { - int ret = -1; - DIR *dh = NULL; - struct dirent *dire = NULL; - - assert (testpath); - - fprintf (stdout, "Testing mkdir"); - ret = mkdir (testpath, 0); - check_err (ret, "mkdir", 2); - - fprintf (stdout, "Testing rmdir"); - ret = rmdir (testpath); - check_err (ret, "rmdir", 2); - - fprintf (stdout, "Testing opendir"); - ret = 0; - dh = opendir (testpath); - if (!dh) - ret = -1; - check_err (ret, "opendir", 2); - - fprintf (stdout, "Testing readdir"); - ret = 0; - dire = readdir (dh); - if (!dire) - ret = -1; - check_err (ret, "readdir", 1); - - fprintf (stdout, "Testing readdir_r"); - ret = readdir_r (dh, dire, &dire); - check_err (ret, "readdir_r", 1); - - fprintf (stdout, "Testing rewinddir"); - rewinddir (dh); - check_err (-1, "rewinddir", 1); - - fprintf (stdout, "Testing seekdir"); - seekdir (dh, 0); - check_err (-1, "seekdir", 2); - - fprintf (stdout, "Testing telldir"); - ret = telldir (dh); - check_err (ret, "telldir", 2); - - fprintf (stdout, "Testing closedir"); - ret = closedir (dh); - check_err (ret, "closedir", 2); - return 0; + int ret = -1; + DIR *dh = NULL; + struct dirent *dire = NULL; + + assert(testpath); + + fprintf(stdout, "Testing mkdir"); + ret = mkdir(testpath, 0); + check_err(ret, "mkdir", 2); + + fprintf(stdout, "Testing rmdir"); + ret = rmdir(testpath); + check_err(ret, "rmdir", 2); + + fprintf(stdout, "Testing opendir"); + ret = 0; + dh = opendir(testpath); + if (!dh) + ret = -1; + check_err(ret, "opendir", 2); + + fprintf(stdout, "Testing readdir"); + ret = 0; + dire = readdir(dh); + if (!dire) + ret = -1; + check_err(ret, "readdir", 1); + + fprintf(stdout, "Testing readdir_r"); + ret = readdir_r(dh, dire, &dire); + check_err(ret, "readdir_r", 1); + + fprintf(stdout, "Testing rewinddir"); + rewinddir(dh); + check_err(-1, "rewinddir", 1); + + fprintf(stdout, "Testing seekdir"); + seekdir(dh, 0); + check_err(-1, "seekdir", 2); + + fprintf(stdout, "Testing telldir"); + ret = telldir(dh); + check_err(ret, "telldir", 2); + + fprintf(stdout, "Testing closedir"); + ret = closedir(dh); + check_err(ret, "closedir", 2); + return 0; } - - int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - char *testpath = NULL; - int x = 0; - - for (;x < argc; ++x) { - if (strcmp (argv[x], "--path") == 0) { - testpath = argv[x+1]; - continue; - } + char *testpath = NULL; + int x = 0; + for (; x < argc; ++x) { + if (strcmp(argv[x], "--path") == 0) { + testpath = argv[x + 1]; + continue; } + } - if (!testpath) { - fprintf (stderr, "--path not specified\n"); - usage (stderr); - return -1; - } + if (!testpath) { + fprintf(stderr, "--path not specified\n"); + usage(stderr); + return -1; + } - run_file_tests (testpath); - run_dir_tests (testpath); - run_attr_tests (testpath); - run_dev_tests (testpath); + run_file_tests(testpath); + run_dir_tests(testpath); + run_attr_tests(testpath); + run_dev_tests(testpath); - return 0; + return 0; } - - diff --git a/extras/test/open-fd-tests.c b/extras/test/open-fd-tests.c index 4184079d043..509952b4180 100644 --- a/extras/test/open-fd-tests.c +++ b/extras/test/open-fd-tests.c @@ -4,61 +4,64 @@ #include <unistd.h> #include <sys/types.h> #include <sys/stat.h> -#include <attr/xattr.h> +#include <sys/xattr.h> #include <errno.h> #include <string.h> int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - int ret = -1; - int fd = 0; - char *filename = NULL; - int loop = 0; - struct stat stbuf = {0,}; - char string[1024] = {0,}; + int ret = -1; + int fd = 0; + char *filename = NULL; + int loop = 0; + struct stat stbuf = { + 0, + }; + char string[1024] = { + 0, + }; - if (argc > 1) - filename = argv[1]; + if (argc > 1) + filename = argv[1]; - if (!filename) - filename = "temp-fd-test-file"; + if (!filename) + filename = "temp-fd-test-file"; - fd = open (filename, O_RDWR|O_CREAT|O_TRUNC); - if (fd < 0) { - fd = 0; - fprintf (stderr, "open failed : %s\n", strerror (errno)); - goto out; - } - - while (loop < 1000) { - /* Use it as a mechanism to test time delays */ - memset (string, 0, 1024); - scanf ("%s", string); + fd = open(filename, O_RDWR | O_CREAT | O_TRUNC); + if (fd < 0) { + fd = 0; + fprintf(stderr, "open failed : %s\n", strerror(errno)); + goto out; + } - ret = write (fd, string, strlen (string)); - if (ret != strlen (string)) { - fprintf (stderr, "write failed : %s (%s %d)\n", - strerror (errno), string, loop); - goto out; - } + while (loop < 1000) { + /* Use it as a mechanism to test time delays */ + memset(string, 0, 1024); + scanf("%s", string); - ret = write (fd, "\n", 1); - if (ret != 1) { - fprintf (stderr, "write failed : %s (%d)\n", - strerror (errno), loop); - goto out; - } + ret = write(fd, string, strlen(string)); + if (ret != strlen(string)) { + fprintf(stderr, "write failed : %s (%s %d)\n", strerror(errno), + string, loop); + goto out; + } - loop++; + ret = write(fd, "\n", 1); + if (ret != 1) { + fprintf(stderr, "write failed : %s (%d)\n", strerror(errno), loop); + goto out; } - fprintf (stdout, "finishing the test after %d loops\n", loop); + loop++; + } + + fprintf(stdout, "finishing the test after %d loops\n", loop); - ret = 0; + ret = 0; out: - if (fd) - close (fd); + if (fd) + close(fd); - return ret; + return ret; } diff --git a/extras/test/test-ffop.c b/extras/test/test-ffop.c index 219dd6a2da2..1d9c125db67 100644 --- a/extras/test/test-ffop.c +++ b/extras/test/test-ffop.c @@ -3,777 +3,825 @@ #include <unistd.h> #include <sys/types.h> #include <sys/stat.h> -#include <attr/xattr.h> +#include <sys/xattr.h> #include <errno.h> #include <string.h> #include <dirent.h> -int fd_based_fops_1 (char *filename); //for fd based fops after unlink -int fd_based_fops_2 (char *filename); //for fd based fops before unlink -int dup_fd_based_fops (char *filename); // fops based on fd after dup -int path_based_fops (char *filename); //for fops based on path -int dir_based_fops (char *filename); // for fops which operate on directory -int link_based_fops (char *filename); //for fops which operate in link files (symlinks) -int test_open_modes (char *filename); // to test open syscall with open modes available. -int generic_open_read_write (char *filename, int flag); // generic function which does open write and read. +int +fd_based_fops_1(char *filename); // for fd based fops after unlink +int +fd_based_fops_2(char *filename); // for fd based fops before unlink +int +dup_fd_based_fops(char *filename); // fops based on fd after dup +int +path_based_fops(char *filename); // for fops based on path +int +dir_based_fops(char *filename); // for fops which operate on directory +int +link_based_fops( + char *filename); // for fops which operate in link files (symlinks) +int +test_open_modes( + char *filename); // to test open syscall with open modes available. +int +generic_open_read_write( + char *filename, + int flag); // generic function which does open write and read. int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - int ret = -1; - char filename[255] = {0,}; - - if (argc > 1) - strcpy(filename, argv[1]); - else - strcpy(filename, "temp-xattr-test-file"); - - ret = fd_based_fops_1 (strcat(filename, "_1")); - if (ret < 0) - fprintf (stderr, "fd based file operation 1 failed\n"); - else - fprintf (stdout, "fd based file operation 1 passed\n"); - - ret = fd_based_fops_2 (strcat(filename, "_2")); - if (ret < 0) - fprintf (stderr, "fd based file operation 2 failed\n"); - else - fprintf (stdout, "fd based file operation 2 passed\n"); - - ret = dup_fd_based_fops (strcat (filename, "_3")); - if (ret < 0) - fprintf (stderr, "dup fd based file operation failed\n"); - else - fprintf (stdout, "dup fd based file operation passed\n"); - - ret = path_based_fops (strcat (filename, "_4")); - if (ret < 0) - fprintf (stderr, "path based file operation failed\n"); - else - fprintf (stdout, "path based file operation passed\n"); - - ret = dir_based_fops (strcat (filename, "_5")); - if (ret < 0) - fprintf (stderr, "directory based file operation failed\n"); - else - fprintf (stdout, "directory based file operation passed\n"); - - ret = link_based_fops (strcat (filename, "_5")); - if (ret < 0) - fprintf (stderr, "link based file operation failed\n"); - else - fprintf (stdout, "link based file operation passed\n"); - - ret = test_open_modes (strcat (filename, "_5")); - if (ret < 0) - fprintf (stderr, "testing modes of 'open' call failed\n"); - else - fprintf (stdout, "testing modes of 'open' call passed\n"); + int ret = -1; + char filename[255] = { + 0, + }; + + if (argc > 1) + strcpy(filename, argv[1]); + else + strcpy(filename, "temp-xattr-test-file"); + + ret = fd_based_fops_1(strcat(filename, "_1")); + if (ret < 0) + fprintf(stderr, "fd based file operation 1 failed\n"); + else + fprintf(stdout, "fd based file operation 1 passed\n"); + + ret = fd_based_fops_2(strcat(filename, "_2")); + if (ret < 0) + fprintf(stderr, "fd based file operation 2 failed\n"); + else + fprintf(stdout, "fd based file operation 2 passed\n"); + + ret = dup_fd_based_fops(strcat(filename, "_3")); + if (ret < 0) + fprintf(stderr, "dup fd based file operation failed\n"); + else + fprintf(stdout, "dup fd based file operation passed\n"); + + ret = path_based_fops(strcat(filename, "_4")); + if (ret < 0) + fprintf(stderr, "path based file operation failed\n"); + else + fprintf(stdout, "path based file operation passed\n"); + + ret = dir_based_fops(strcat(filename, "_5")); + if (ret < 0) + fprintf(stderr, "directory based file operation failed\n"); + else + fprintf(stdout, "directory based file operation passed\n"); + + ret = link_based_fops(strcat(filename, "_5")); + if (ret < 0) + fprintf(stderr, "link based file operation failed\n"); + else + fprintf(stdout, "link based file operation passed\n"); + + ret = test_open_modes(strcat(filename, "_5")); + if (ret < 0) + fprintf(stderr, "testing modes of 'open' call failed\n"); + else + fprintf(stdout, "testing modes of 'open' call passed\n"); out: - return ret; + return ret; } int -fd_based_fops_1 (char *filename) +fd_based_fops_1(char *filename) { - int fd = 0; - int ret = -1; - struct stat stbuf = {0,}; - char wstr[50] = {0,}; - char rstr[50] = {0,}; - - fd = open (filename, O_RDWR|O_CREAT); - if (fd < 0) { - fd = 0; - fprintf (stderr, "open failed : %s\n", strerror (errno)); - goto out; - } - - ret = unlink (filename); - if (ret < 0) { - fprintf (stderr, "unlink failed : %s\n", strerror (errno)); - goto out; - } - - strcpy (wstr, "This is my string\n"); - ret = write (fd, wstr, strlen(wstr)); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "write failed: %s\n", strerror (errno)); - goto out; - } - - ret = lseek (fd, 0, SEEK_SET); - if (ret < 0) { - fprintf (stderr, "lseek failed: %s\n", strerror (errno)); - goto out; - } - - ret = read (fd, rstr, strlen(wstr)); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "read failed: %s\n", strerror (errno)); - goto out; - } - - ret = memcmp (rstr, wstr, strlen (wstr)); - if (ret != 0) { - ret = -1; - fprintf (stderr, "read returning junk\n"); - goto out; - } - - ret = ftruncate (fd, 0); - if (ret < 0) { - fprintf (stderr, "ftruncate failed : %s\n", strerror (errno)); - goto out; - } - - ret = fstat (fd, &stbuf); - if (ret < 0) { - fprintf (stderr, "fstat failed : %s\n", strerror (errno)); - goto out; - } - - ret = fchmod (fd, 0640); - if (ret < 0) { - fprintf (stderr, "fchmod failed : %s\n", strerror (errno)); - goto out; - } - - ret = fchown (fd, 10001, 10001); - if (ret < 0) { - fprintf (stderr, "fchown failed : %s\n", strerror (errno)); - goto out; - } - - ret = fsync (fd); - if (ret < 0) { - fprintf (stderr, "fsync failed : %s\n", strerror (errno)); - goto out; - } - - ret = fsetxattr (fd, "trusted.xattr-test", "working", 8, 0); - if (ret < 0) { - fprintf (stderr, "fsetxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fdatasync (fd); - if (ret < 0) { - fprintf (stderr, "fdatasync failed : %s\n", strerror (errno)); - goto out; - } - - ret = flistxattr (fd, NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "flistxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fgetxattr (fd, "trusted.xattr-test", NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "fgetxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fremovexattr (fd, "trusted.xattr-test"); - if (ret < 0) { - fprintf (stderr, "fremovexattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = 0; + int fd = 0; + int ret = -1; + struct stat stbuf = { + 0, + }; + char wstr[50] = { + 0, + }; + char rstr[50] = { + 0, + }; + + fd = open(filename, O_RDWR | O_CREAT); + if (fd < 0) { + fd = 0; + fprintf(stderr, "open failed : %s\n", strerror(errno)); + goto out; + } + + ret = unlink(filename); + if (ret < 0) { + fprintf(stderr, "unlink failed : %s\n", strerror(errno)); + goto out; + } + + strcpy(wstr, "This is my string\n"); + ret = write(fd, wstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "write failed: %s\n", strerror(errno)); + goto out; + } + + ret = lseek(fd, 0, SEEK_SET); + if (ret < 0) { + fprintf(stderr, "lseek failed: %s\n", strerror(errno)); + goto out; + } + + ret = read(fd, rstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "read failed: %s\n", strerror(errno)); + goto out; + } + + ret = memcmp(rstr, wstr, strlen(wstr)); + if (ret != 0) { + ret = -1; + fprintf(stderr, "read returning junk\n"); + goto out; + } + + ret = ftruncate(fd, 0); + if (ret < 0) { + fprintf(stderr, "ftruncate failed : %s\n", strerror(errno)); + goto out; + } + + ret = fstat(fd, &stbuf); + if (ret < 0) { + fprintf(stderr, "fstat failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchmod(fd, 0640); + if (ret < 0) { + fprintf(stderr, "fchmod failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchown(fd, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "fchown failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsync(fd); + if (ret < 0) { + fprintf(stderr, "fsync failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsetxattr(fd, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fdatasync(fd); + if (ret < 0) { + fprintf(stderr, "fdatasync failed : %s\n", strerror(errno)); + goto out; + } + + ret = flistxattr(fd, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "flistxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fgetxattr(fd, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fremovexattr(fd, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = 0; out: - if (fd) - close (fd); + if (fd) + close(fd); - return ret; + return ret; } - int -fd_based_fops_2 (char *filename) +fd_based_fops_2(char *filename) { - int fd = 0; - int ret = -1; - struct stat stbuf = {0,}; - char wstr[50] = {0,}; - char rstr[50] = {0,}; - - fd = open (filename, O_RDWR|O_CREAT); - if (fd < 0) { - fd = 0; - fprintf (stderr, "open failed : %s\n", strerror (errno)); - goto out; - } - - ret = ftruncate (fd, 0); - - if (ret < 0) { - fprintf (stderr, "ftruncate failed : %s\n", strerror (errno)); - goto out; - } - - strcpy (wstr, "This is my second string\n"); - ret = write (fd, wstr, strlen (wstr)); - if (ret < 0) { - ret = -1; - fprintf (stderr, "write failed: %s\n", strerror (errno)); - goto out; - } - - lseek (fd, 0, SEEK_SET); - if (ret < 0) { - fprintf (stderr, "lseek failed: %s\n", strerror (errno)); - goto out; - } - - ret = read (fd, rstr, strlen (wstr)); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "read failed: %s\n", strerror (errno)); - goto out; - } - - ret = memcmp (rstr, wstr, strlen (wstr)); - if (ret != 0) { - ret = -1; - fprintf (stderr, "read returning junk\n"); - goto out; - } - - ret = fstat (fd, &stbuf); - if (ret < 0) { - fprintf (stderr, "fstat failed : %s\n", strerror (errno)); - goto out; - } - - ret = fchmod (fd, 0640); - if (ret < 0) { - fprintf (stderr, "fchmod failed : %s\n", strerror (errno)); - goto out; - } - - ret = fchown (fd, 10001, 10001); - if (ret < 0) { - fprintf (stderr, "fchown failed : %s\n", strerror (errno)); - goto out; - } - - ret = fsync (fd); - if (ret < 0) { - fprintf (stderr, "fsync failed : %s\n", strerror (errno)); - goto out; - } - - ret = fsetxattr (fd, "trusted.xattr-test", "working", 8, 0); - if (ret < 0) { - fprintf (stderr, "fsetxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fdatasync (fd); - if (ret < 0) { - fprintf (stderr, "fdatasync failed : %s\n", strerror (errno)); - goto out; - } - - ret = flistxattr (fd, NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "flistxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fgetxattr (fd, "trusted.xattr-test", NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "fgetxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fremovexattr (fd, "trusted.xattr-test"); - if (ret < 0) { - fprintf (stderr, "fremovexattr failed : %s\n", strerror (errno)); - goto out; - } + int fd = 0; + int ret = -1; + struct stat stbuf = { + 0, + }; + char wstr[50] = { + 0, + }; + char rstr[50] = { + 0, + }; + + fd = open(filename, O_RDWR | O_CREAT); + if (fd < 0) { + fd = 0; + fprintf(stderr, "open failed : %s\n", strerror(errno)); + goto out; + } + + ret = ftruncate(fd, 0); + + if (ret < 0) { + fprintf(stderr, "ftruncate failed : %s\n", strerror(errno)); + goto out; + } + + strcpy(wstr, "This is my second string\n"); + ret = write(fd, wstr, strlen(wstr)); + if (ret < 0) { + ret = -1; + fprintf(stderr, "write failed: %s\n", strerror(errno)); + goto out; + } + + lseek(fd, 0, SEEK_SET); + if (ret < 0) { + fprintf(stderr, "lseek failed: %s\n", strerror(errno)); + goto out; + } + + ret = read(fd, rstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "read failed: %s\n", strerror(errno)); + goto out; + } + + ret = memcmp(rstr, wstr, strlen(wstr)); + if (ret != 0) { + ret = -1; + fprintf(stderr, "read returning junk\n"); + goto out; + } + + ret = fstat(fd, &stbuf); + if (ret < 0) { + fprintf(stderr, "fstat failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchmod(fd, 0640); + if (ret < 0) { + fprintf(stderr, "fchmod failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchown(fd, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "fchown failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsync(fd); + if (ret < 0) { + fprintf(stderr, "fsync failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsetxattr(fd, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fdatasync(fd); + if (ret < 0) { + fprintf(stderr, "fdatasync failed : %s\n", strerror(errno)); + goto out; + } + + ret = flistxattr(fd, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "flistxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fgetxattr(fd, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fremovexattr(fd, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno)); + goto out; + } out: - if (fd) - close (fd); - unlink (filename); + if (fd) + close(fd); + unlink(filename); - return ret; + return ret; } int -path_based_fops (char *filename) +path_based_fops(char *filename) { - int ret = -1; - int fd = 0; - struct stat stbuf = {0,}; - char newfilename[255] = {0,}; - - fd = creat (filename, 0644); - if (fd < 0) { - fprintf (stderr, "creat failed: %s\n", strerror (errno)); - goto out; - } - - ret = truncate (filename, 0); - if (ret < 0) { - fprintf (stderr, "truncate failed: %s\n", strerror (errno)); - goto out; - } - - ret = stat (filename, &stbuf); - if (ret < 0) { - fprintf (stderr, "stat failed: %s\n", strerror (errno)); - goto out; - } - - ret = chmod (filename, 0640); - if (ret < 0) { - fprintf (stderr, "chmod failed: %s\n", strerror (errno)); - goto out; - } - - ret = chown (filename, 10001, 10001); - if (ret < 0) { - fprintf (stderr, "chown failed: %s\n", strerror (errno)); - goto out; - } - - ret = setxattr (filename, "trusted.xattr-test", "working", 8, 0); - if (ret < 0) { - fprintf (stderr, "setxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = listxattr (filename, NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "listxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = getxattr (filename, "trusted.xattr-test", NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "getxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = removexattr (filename, "trusted.xattr-test"); - if (ret < 0) { - fprintf (stderr, "removexattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = access (filename, R_OK|W_OK); - if (ret < 0) { - fprintf (stderr, "access failed: %s\n", strerror (errno)); - goto out; - } - - strcpy (newfilename, filename); - strcat(newfilename, "_new"); - ret = rename (filename, newfilename); - if (ret < 0) { - fprintf (stderr, "rename failed: %s\n", strerror (errno)); - goto out; - } - unlink (newfilename); + int ret = -1; + int fd = 0; + struct stat stbuf = { + 0, + }; + char newfilename[255] = { + 0, + }; + + fd = creat(filename, 0644); + if (fd < 0) { + fprintf(stderr, "creat failed: %s\n", strerror(errno)); + goto out; + } + + ret = truncate(filename, 0); + if (ret < 0) { + fprintf(stderr, "truncate failed: %s\n", strerror(errno)); + goto out; + } + + ret = stat(filename, &stbuf); + if (ret < 0) { + fprintf(stderr, "stat failed: %s\n", strerror(errno)); + goto out; + } + + ret = chmod(filename, 0640); + if (ret < 0) { + fprintf(stderr, "chmod failed: %s\n", strerror(errno)); + goto out; + } + + ret = chown(filename, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "chown failed: %s\n", strerror(errno)); + goto out; + } + + ret = setxattr(filename, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "setxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = listxattr(filename, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "listxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = getxattr(filename, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "getxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = removexattr(filename, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "removexattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = access(filename, R_OK | W_OK); + if (ret < 0) { + fprintf(stderr, "access failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(newfilename, filename); + strcat(newfilename, "_new"); + ret = rename(filename, newfilename); + if (ret < 0) { + fprintf(stderr, "rename failed: %s\n", strerror(errno)); + goto out; + } + unlink(newfilename); out: - if (fd) - close (fd); + if (fd) + close(fd); - unlink (filename); - return ret; + unlink(filename); + return ret; } int -dup_fd_based_fops (char *filename) +dup_fd_based_fops(char *filename) { - int fd = 0; - int newfd = 0; - int ret = -1; - struct stat stbuf = {0,}; - char wstr[50] = {0,}; - char rstr[50] = {0,}; - - fd = open (filename, O_RDWR|O_CREAT); - if (fd < 0) { - fd = 0; - fprintf (stderr, "open failed : %s\n", strerror (errno)); - goto out; - } - - newfd = dup (fd); - if (newfd < 0) { - ret = -1; - fprintf (stderr, "dup failed: %s\n", strerror (errno)); - goto out; - } - - close (fd); - - strcpy (wstr, "This is my string\n"); - ret = write (newfd, wstr, strlen(wstr)); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "write failed: %s\n", strerror (errno)); - goto out; - } - - ret = lseek (newfd, 0, SEEK_SET); - if (ret < 0) { - fprintf (stderr, "lseek failed: %s\n", strerror (errno)); - goto out; - } - - ret = read (newfd, rstr, strlen(wstr)); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "read failed: %s\n", strerror (errno)); - goto out; - } - - ret = memcmp (rstr, wstr, strlen (wstr)); - if (ret != 0) { - ret = -1; - fprintf (stderr, "read returning junk\n"); - goto out; - } - - ret = ftruncate (newfd, 0); - if (ret < 0) { - fprintf (stderr, "ftruncate failed : %s\n", strerror (errno)); - goto out; - } - - ret = fstat (newfd, &stbuf); - if (ret < 0) { - fprintf (stderr, "fstat failed : %s\n", strerror (errno)); - goto out; - } - - ret = fchmod (newfd, 0640); - if (ret < 0) { - fprintf (stderr, "fchmod failed : %s\n", strerror (errno)); - goto out; - } - - ret = fchown (newfd, 10001, 10001); - if (ret < 0) { - fprintf (stderr, "fchown failed : %s\n", strerror (errno)); - goto out; - } - - ret = fsync (newfd); - if (ret < 0) { - fprintf (stderr, "fsync failed : %s\n", strerror (errno)); - goto out; - } - - ret = fsetxattr (newfd, "trusted.xattr-test", "working", 8, 0); - if (ret < 0) { - fprintf (stderr, "fsetxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fdatasync (newfd); - if (ret < 0) { - fprintf (stderr, "fdatasync failed : %s\n", strerror (errno)); - goto out; - } - - ret = flistxattr (newfd, NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "flistxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fgetxattr (newfd, "trusted.xattr-test", NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "fgetxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fremovexattr (newfd, "trusted.xattr-test"); - if (ret < 0) { - fprintf (stderr, "fremovexattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = 0; + int fd = 0; + int newfd = 0; + int ret = -1; + struct stat stbuf = { + 0, + }; + char wstr[50] = { + 0, + }; + char rstr[50] = { + 0, + }; + + fd = open(filename, O_RDWR | O_CREAT); + if (fd < 0) { + fd = 0; + fprintf(stderr, "open failed : %s\n", strerror(errno)); + goto out; + } + + newfd = dup(fd); + if (newfd < 0) { + ret = -1; + fprintf(stderr, "dup failed: %s\n", strerror(errno)); + goto out; + } + + close(fd); + + strcpy(wstr, "This is my string\n"); + ret = write(newfd, wstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "write failed: %s\n", strerror(errno)); + goto out; + } + + ret = lseek(newfd, 0, SEEK_SET); + if (ret < 0) { + fprintf(stderr, "lseek failed: %s\n", strerror(errno)); + goto out; + } + + ret = read(newfd, rstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "read failed: %s\n", strerror(errno)); + goto out; + } + + ret = memcmp(rstr, wstr, strlen(wstr)); + if (ret != 0) { + ret = -1; + fprintf(stderr, "read returning junk\n"); + goto out; + } + + ret = ftruncate(newfd, 0); + if (ret < 0) { + fprintf(stderr, "ftruncate failed : %s\n", strerror(errno)); + goto out; + } + + ret = fstat(newfd, &stbuf); + if (ret < 0) { + fprintf(stderr, "fstat failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchmod(newfd, 0640); + if (ret < 0) { + fprintf(stderr, "fchmod failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchown(newfd, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "fchown failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsync(newfd); + if (ret < 0) { + fprintf(stderr, "fsync failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsetxattr(newfd, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fdatasync(newfd); + if (ret < 0) { + fprintf(stderr, "fdatasync failed : %s\n", strerror(errno)); + goto out; + } + + ret = flistxattr(newfd, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "flistxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fgetxattr(newfd, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fremovexattr(newfd, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = 0; out: - if (newfd) - close (newfd); - ret = unlink (filename); - if (ret < 0) - fprintf (stderr, "unlink failed : %s\n", strerror (errno)); + if (newfd) + close(newfd); + ret = unlink(filename); + if (ret < 0) + fprintf(stderr, "unlink failed : %s\n", strerror(errno)); - return ret; + return ret; } int -dir_based_fops (char *dirname) +dir_based_fops(char *dirname) { - int ret = -1; - DIR *dp = NULL; - char buff[255] = {0,}; - struct dirent *dbuff = {0,}; - struct stat stbuff = {0,}; - char newdname[255] = {0,}; - char *cwd = NULL; - - ret = mkdir (dirname, 0755); - if (ret < 0) { - fprintf (stderr, "mkdir failed: %s\n", strerror (errno)); - goto out; - } - - dp = opendir (dirname); - if (dp == NULL) { - fprintf (stderr, "opendir failed: %s\n", strerror (errno)); - goto out; - } - - dbuff = readdir (dp); - if (NULL == dbuff) { - fprintf (stderr, "readdir failed: %s\n", strerror (errno)); - goto out; - } - - ret = closedir (dp); - if (ret < 0) { - fprintf (stderr, "closedir failed: %s\n", strerror (errno)); - goto out; - } - - ret = stat (dirname, &stbuff); - if (ret < 0) { - fprintf (stderr, "stat failed: %s\n", strerror (errno)); - goto out; - } - - ret = chmod (dirname, 0744); - if (ret < 0) { - fprintf (stderr, "chmod failed: %s\n", strerror (errno)); - goto out; - } - - ret = chown (dirname, 10001, 10001); - if (ret < 0) { - fprintf (stderr, "chmod failed: %s\n", strerror (errno)); - goto out; - } - - ret = setxattr (dirname, "trusted.xattr-test", "working", 8, 0); - if (ret < 0) { - fprintf (stderr, "setxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = listxattr (dirname, NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "listxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = getxattr (dirname, "trusted.xattr-test", NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "getxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = removexattr (dirname, "trusted.xattr-test"); - if (ret < 0) { - fprintf (stderr, "removexattr failed: %s\n", strerror (errno)); - goto out; - } - - strcpy (newdname, dirname); - strcat (newdname, "/../"); - ret = chdir (newdname); - if (ret < 0) { - fprintf (stderr, "chdir failed: %s\n", strerror (errno)); - goto out; - } - - cwd = getcwd (buff, 255); - if (NULL == cwd) { - fprintf (stderr, "getcwd failed: %s\n", strerror (errno)); - goto out; - } - - strcpy (newdname, dirname); - strcat (newdname, "new"); - ret = rename (dirname, newdname); - if (ret < 0) { - fprintf (stderr, "rename failed: %s\n", strerror (errno)); - goto out; - } - - ret = rmdir (newdname); - if (ret < 0) { - fprintf (stderr, "rmdir failed: %s\n", strerror (errno)); - return ret; - } + int ret = -1; + DIR *dp = NULL; + char buff[255] = { + 0, + }; + struct dirent *dbuff = { + 0, + }; + struct stat stbuff = { + 0, + }; + char newdname[255] = { + 0, + }; + char *cwd = NULL; + + ret = mkdir(dirname, 0755); + if (ret < 0) { + fprintf(stderr, "mkdir failed: %s\n", strerror(errno)); + goto out; + } + + dp = opendir(dirname); + if (dp == NULL) { + fprintf(stderr, "opendir failed: %s\n", strerror(errno)); + goto out; + } + + dbuff = readdir(dp); + if (NULL == dbuff) { + fprintf(stderr, "readdir failed: %s\n", strerror(errno)); + goto out; + } + + ret = closedir(dp); + if (ret < 0) { + fprintf(stderr, "closedir failed: %s\n", strerror(errno)); + goto out; + } + + ret = stat(dirname, &stbuff); + if (ret < 0) { + fprintf(stderr, "stat failed: %s\n", strerror(errno)); + goto out; + } + + ret = chmod(dirname, 0744); + if (ret < 0) { + fprintf(stderr, "chmod failed: %s\n", strerror(errno)); + goto out; + } + + ret = chown(dirname, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "chmod failed: %s\n", strerror(errno)); + goto out; + } + + ret = setxattr(dirname, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "setxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = listxattr(dirname, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "listxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = getxattr(dirname, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "getxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = removexattr(dirname, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "removexattr failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(newdname, dirname); + strcat(newdname, "/../"); + ret = chdir(newdname); + if (ret < 0) { + fprintf(stderr, "chdir failed: %s\n", strerror(errno)); + goto out; + } + + cwd = getcwd(buff, 255); + if (NULL == cwd) { + fprintf(stderr, "getcwd failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(newdname, dirname); + strcat(newdname, "new"); + ret = rename(dirname, newdname); + if (ret < 0) { + fprintf(stderr, "rename failed: %s\n", strerror(errno)); + goto out; + } + + ret = rmdir(newdname); + if (ret < 0) { + fprintf(stderr, "rmdir failed: %s\n", strerror(errno)); + return ret; + } out: - rmdir (dirname); - return ret; + rmdir(dirname); + return ret; } int -link_based_fops (char *filename) +link_based_fops(char *filename) { - int ret = -1; - int fd = 0; - char newname[255] = {0,}; - char linkname[255] = {0,}; - struct stat lstbuf = {0,}; - - fd = creat (filename, 0644); - if (fd < 0) { - fd = 0; - fprintf (stderr, "creat failed: %s\n", strerror (errno)); - goto out; - } - - strcpy (newname, filename); - strcat (newname, "_hlink"); - ret = link (filename, newname); - if (ret < 0) { - fprintf (stderr, "link failed: %s\n", strerror (errno)); - goto out; - } - - ret = unlink (filename); - if (ret < 0) { - fprintf (stderr, "unlink failed: %s\n", strerror (errno)); - goto out; - } - - strcpy (linkname, filename); - strcat (linkname, "_slink"); - ret = symlink (newname, linkname); - if (ret < 0) { - fprintf (stderr, "symlink failed: %s\n", strerror (errno)); - goto out; - } - - ret = lstat (linkname, &lstbuf); - if (ret < 0) { - fprintf (stderr, "lstbuf failed: %s\n", strerror (errno)); - goto out; - } - - ret = lchown (linkname, 10001, 10001); - if (ret < 0) { - fprintf (stderr, "lchown failed: %s\n", strerror (errno)); - goto out; - } - - ret = lsetxattr (linkname, "trusted.lxattr-test", "working", 8, 0); - if (ret < 0) { - fprintf (stderr, "lsetxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = llistxattr (linkname, NULL, 0); - if (ret < 0) { - ret = -1; - fprintf (stderr, "llistxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = lgetxattr (linkname, "trusted.lxattr-test", NULL, 0); - if (ret < 0) { - ret = -1; - fprintf (stderr, "lgetxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = lremovexattr (linkname, "trusted.lxattr-test"); - if (ret < 0) { - fprintf (stderr, "lremovexattr failed: %s\n", strerror (errno)); - goto out; - } - + int ret = -1; + int fd = 0; + char newname[255] = { + 0, + }; + char linkname[255] = { + 0, + }; + struct stat lstbuf = { + 0, + }; + + fd = creat(filename, 0644); + if (fd < 0) { + fd = 0; + fprintf(stderr, "creat failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(newname, filename); + strcat(newname, "_hlink"); + ret = link(filename, newname); + if (ret < 0) { + fprintf(stderr, "link failed: %s\n", strerror(errno)); + goto out; + } + + ret = unlink(filename); + if (ret < 0) { + fprintf(stderr, "unlink failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(linkname, filename); + strcat(linkname, "_slink"); + ret = symlink(newname, linkname); + if (ret < 0) { + fprintf(stderr, "symlink failed: %s\n", strerror(errno)); + goto out; + } + + ret = lstat(linkname, &lstbuf); + if (ret < 0) { + fprintf(stderr, "lstbuf failed: %s\n", strerror(errno)); + goto out; + } + + ret = lchown(linkname, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "lchown failed: %s\n", strerror(errno)); + goto out; + } + + ret = lsetxattr(linkname, "trusted.lxattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "lsetxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = llistxattr(linkname, NULL, 0); + if (ret < 0) { + ret = -1; + fprintf(stderr, "llistxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = lgetxattr(linkname, "trusted.lxattr-test", NULL, 0); + if (ret < 0) { + ret = -1; + fprintf(stderr, "lgetxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = lremovexattr(linkname, "trusted.lxattr-test"); + if (ret < 0) { + fprintf(stderr, "lremovexattr failed: %s\n", strerror(errno)); + goto out; + } out: - if (fd) - close(fd); - unlink (linkname); - unlink (newname); + if (fd) + close(fd); + unlink(linkname); + unlink(newname); } int -test_open_modes (char *filename) +test_open_modes(char *filename) { - int ret = -1; - - ret = generic_open_read_write (filename, O_CREAT|O_WRONLY); - if (3 != ret) { - fprintf (stderr, "flag O_CREAT|O_WRONLY failed: \n"); - goto out; - } - - ret = generic_open_read_write (filename, O_CREAT|O_RDWR); - if (ret != 0) { - fprintf (stderr, "flag O_CREAT|O_RDWR failed\n"); - goto out; - } - - ret = generic_open_read_write (filename, O_CREAT|O_RDONLY); - if (ret != 0) { - fprintf (stderr, "flag O_CREAT|O_RDONLY failed\n"); - goto out; - } - - ret = creat (filename, 0644); - close (ret); - ret = generic_open_read_write (filename, O_WRONLY); - if (3 != ret) { - fprintf (stderr, "flag O_WRONLY failed\n"); - goto out; - } - - ret = creat (filename, 0644); - close (ret); - ret = generic_open_read_write (filename, O_RDWR); - if (0 != ret) { - fprintf (stderr, "flag O_RDWR failed\n"); - goto out; - } - - ret = creat (filename, 0644); - close (ret); - ret = generic_open_read_write (filename, O_RDONLY); - if (0 != ret) { - fprintf (stderr, "flag O_RDONLY failed\n"); - goto out; - } - - ret = creat (filename, 0644); - close (ret); - ret = generic_open_read_write (filename, O_TRUNC|O_WRONLY); - if (3 != ret) { - fprintf (stderr, "flag O_TRUNC|O_WRONLY failed\n"); - goto out; - } + int ret = -1; + + ret = generic_open_read_write(filename, O_CREAT | O_WRONLY); + if (3 != ret) { + fprintf(stderr, "flag O_CREAT|O_WRONLY failed: \n"); + goto out; + } + + ret = generic_open_read_write(filename, O_CREAT | O_RDWR); + if (ret != 0) { + fprintf(stderr, "flag O_CREAT|O_RDWR failed\n"); + goto out; + } + + ret = generic_open_read_write(filename, O_CREAT | O_RDONLY); + if (ret != 0) { + fprintf(stderr, "flag O_CREAT|O_RDONLY failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_WRONLY); + if (3 != ret) { + fprintf(stderr, "flag O_WRONLY failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_RDWR); + if (0 != ret) { + fprintf(stderr, "flag O_RDWR failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_RDONLY); + if (0 != ret) { + fprintf(stderr, "flag O_RDONLY failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_TRUNC | O_WRONLY); + if (3 != ret) { + fprintf(stderr, "flag O_TRUNC|O_WRONLY failed\n"); + goto out; + } #if 0 /* undefined behaviour, unable to reliably test */ ret = creat (filename, 0644); @@ -785,84 +833,88 @@ test_open_modes (char *filename) } #endif - ret = generic_open_read_write (filename, O_CREAT|O_RDWR|O_SYNC); - if (0 != ret) { - fprintf (stderr, "flag O_CREAT|O_RDWR|O_SYNC failed\n"); - goto out; - } + ret = generic_open_read_write(filename, O_CREAT | O_RDWR | O_SYNC); + if (0 != ret) { + fprintf(stderr, "flag O_CREAT|O_RDWR|O_SYNC failed\n"); + goto out; + } - ret = creat (filename, 0644); - close (ret); - ret = generic_open_read_write (filename, O_CREAT|O_EXCL); - if (0 != ret) { - fprintf (stderr, "flag O_CREAT|O_EXCL failed\n"); - goto out; - } + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_CREAT | O_EXCL); + if (0 != ret) { + fprintf(stderr, "flag O_CREAT|O_EXCL failed\n"); + goto out; + } out: - return ret; + return ret; } -int generic_open_read_write (char *filename, int flag) +int +generic_open_read_write(char *filename, int flag) { - int fd = 0; - int ret = -1; - char wstring[50] = {0,}; - char rstring[50] = {0,}; - - fd = open (filename, flag); - if (fd < 0) { - if (flag == O_CREAT|O_EXCL && errno == EEXIST) { - unlink (filename); - return 0; - } - else { - fd = 0; - fprintf (stderr, "open failed: %s\n", strerror (errno)); - return 1; - } - } - - strcpy (wstring, "My string to write\n"); - ret = write (fd, wstring, strlen(wstring)); - if (ret <= 0) { - if (errno != EBADF) { - fprintf (stderr, "write failed: %s\n", strerror (errno)); - close (fd); - unlink(filename); - return 2; - } - } - - ret = lseek (fd, 0, SEEK_SET); - if (ret < 0) { - close (fd); - unlink(filename); - return 4; - } - - ret = read (fd, rstring, strlen(wstring)); - if (ret < 0) { - close (fd); - unlink (filename); - return 3; - } - - /* Compare the rstring with wstring. But we do not want to return - * error when the flag is either O_RDONLY, O_CREAT|O_RDONLY or - * O_TRUNC|O_RDONLY. Because in that case we are not writing - * anything to the file.*/ - - ret = memcmp (wstring, rstring, strlen (wstring)); - if (0 != ret && !(flag == O_CREAT|O_RDONLY || flag == O_RDONLY ||\ - flag == O_TRUNC|O_RDONLY)) { - fprintf (stderr, "read is returning junk\n"); - close (fd); - unlink (filename); - return 4; - } - - close (fd); - unlink (filename); - return 0; + int fd = 0; + int ret = -1; + char wstring[50] = { + 0, + }; + char rstring[50] = { + 0, + }; + + fd = open(filename, flag); + if (fd < 0) { + if (flag == O_CREAT | O_EXCL && errno == EEXIST) { + unlink(filename); + return 0; + } else { + fd = 0; + fprintf(stderr, "open failed: %s\n", strerror(errno)); + return 1; + } + } + + strcpy(wstring, "My string to write\n"); + ret = write(fd, wstring, strlen(wstring)); + if (ret <= 0) { + if (errno != EBADF) { + fprintf(stderr, "write failed: %s\n", strerror(errno)); + close(fd); + unlink(filename); + return 2; + } + } + + ret = lseek(fd, 0, SEEK_SET); + if (ret < 0) { + close(fd); + unlink(filename); + return 4; + } + + ret = read(fd, rstring, strlen(wstring)); + if (ret < 0) { + close(fd); + unlink(filename); + return 3; + } + + /* Compare the rstring with wstring. But we do not want to return + * error when the flag is either O_RDONLY, O_CREAT|O_RDONLY or + * O_TRUNC|O_RDONLY. Because in that case we are not writing + * anything to the file.*/ + + ret = memcmp(wstring, rstring, strlen(wstring)); + if (0 != ret && !(flag == O_CREAT | O_RDONLY || flag == O_RDONLY || + flag == O_TRUNC | O_RDONLY)) { + fprintf(stderr, "read is returning junk\n"); + close(fd); + unlink(filename); + return 4; + } + + close(fd); + unlink(filename); + return 0; } diff --git a/extras/thin-arbiter/setup-thin-arbiter.sh b/extras/thin-arbiter/setup-thin-arbiter.sh new file mode 100755 index 00000000000..0681b30ef3f --- /dev/null +++ b/extras/thin-arbiter/setup-thin-arbiter.sh @@ -0,0 +1,184 @@ +#!/bin/bash +# Copyright (c) 2018-2019 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + + +# This tool has been developed to setup thin-arbiter process on a node. +# Seting up a thin arbiter process involves following files - +# 1 - thin-arbiter.vol +# Thin-arbiter (TA) process will use the graph in this file to load the +# required translators. +# 2 - gluster-ta-volume.service (generated by gluster-ta-volume.service.in) +# TA process would be running as systemd service. +# +# TA process uses a location to save TA id files for every subvolume. +# This location can be taken as input from user. Once provided and the +# TA process is started on a node, it can not be changed using this +# script or by any other mean. The same location should be used in +# the gluster CLI when creating thin-arbiter volumes. + +MYPATH=`dirname $0` + +volloc="/var/lib/glusterd/thin-arbiter" +mkdir -p $volloc + +if [ -f /etc/glusterfs/thin-arbiter.vol ]; then + volfile=/etc/glusterfs/thin-arbiter.vol +else + volfile=$MYPATH/thin-arbiter.vol +fi + +tafile="$volloc/thin-arbiter.vol" + + +help () { + echo " " + echo ' This tool helps to setup thin-arbiter (TA) process on a node. + TA process uses a location to save TA id files for every subvolume. + This location can be taken as input from user. Once provided and the + TA process is started on a node, it can not be changed using this script + or by any other mean. The same location should be used in gluster CLI + when creating thin-arbiter volumes. + + usage: setup-thin-arbiter.sh [-s] [-h] + options: + -s - Setup thin-arbiter file path and start process + -h - Show this help message and exit +' +} + +volfile_set_brick_path () { + while read -r line + do + dir=`echo "$line" | cut -d' ' -f 2` + if [ "$dir" = "directory" ]; then + bpath=`echo "$line" | cut -d' ' -f 3` + sed -i -- 's?'$bpath'?'$1'?g' $tafile + return + fi + done < $tafile +} + +check_ta_proc () { + pro=`ps aux | grep thin-arbiter.vol | grep "volfile-id"` + if [ "${pro}" = '' ]; then + echo "" + else + curr_loc=`cat $volloc/thin-arbiter.vol | grep option | grep directory` + loc=`echo "${curr_loc##* }"` + echo "******************************************************" + echo "Error:" + echo "Thin-arbiter process is running with thin-arbiter path = $loc" + echo "Can not change TA path on this host now." + echo "$pro" + echo "******************************************************" + exit 1 + fi +} + +getpath () { + check_ta_proc + echo "******************************************************" + echo "User will be required to enter a path/folder for arbiter volume." + echo "Please note that this path will be used for ALL VOLUMES using this" + echo "node to host thin-arbiter. After setting, if a volume" + echo "has been created using this host and path then path for" + echo "thin-arbiter can not be changed " + echo "******************************************************" + echo " " + while true; + do + echo -n "Enter brick path for thin arbiter volumes: " + echo " " + read tapath + if [ "${tapath}" = '' ]; then + echo "Please enter valid path" + continue + else + echo "Entered brick path : $tapath " + echo "Please note that this brick path will be used for ALL" + echo "VOLUMES using this node to host thin-arbiter brick" + echo -n "Want to continue? (y/N): " + echo " " + read cont + + if [ "${cont}" = 'N' ] || [ "${cont}" = 'n' ]; then + exit 0 + else + break + fi + fi + done +} + +setup () { + getpath + mkdir -p $tapath/.glusterfs/indices + if [ -d $tapath/.glusterfs/indices ]; then + echo " " + else + echo "Could not create $tapath/.glusterfs/indices directory, check provided ta path." + exit 1 + fi + + cp -f --backup --suffix=_old $volfile $volloc/thin-arbiter.vol + volfile_set_brick_path "$tapath" + + echo "Directory path to be used for thin-arbiter volume is: $tapath" + echo " " + echo "========================================================" + + if [ -f /usr/lib/systemd/system/gluster-ta-volume.service ]; then + echo "Starting thin-arbiter process" + else + cp $MYPATH/../systemd/gluster-ta-volume.service /etc/systemd/system/ + echo "Starting thin-arbiter process" + chmod 0644 /etc/systemd/system/gluster-ta-volume.service + fi + + systemctl daemon-reload + systemctl enable gluster-ta-volume + systemctl stop gluster-ta-volume + systemctl start gluster-ta-volume + + if [ $? == 0 ]; then + echo "thin-arbiter process has been setup and running" + else + echo "Failed to setup thin arbiter" + exit 1 + fi + +} + +main() +{ + + if [ "$#" -ne 1 ]; then + help + exit 0 + fi + + while getopts "sh" opt; do + case $opt in + h) + help + exit 0 + ;; + s) + setup + exit 0 + ;; + *) + help + exit 0 + ;; + esac + done +} + +main "$@" diff --git a/extras/thin-arbiter/thin-arbiter.vol b/extras/thin-arbiter/thin-arbiter.vol new file mode 100644 index 00000000000..c76babc7b3c --- /dev/null +++ b/extras/thin-arbiter/thin-arbiter.vol @@ -0,0 +1,57 @@ +volume ta-posix + type storage/posix + option directory /mnt/thin-arbiter +end-volume + +volume ta-thin-arbiter + type features/thin-arbiter + subvolumes ta-posix +end-volume + +volume ta-locks + type features/locks + option notify-contention yes + subvolumes ta-thin-arbiter +end-volume + +volume ta-upcall + type features/upcall + option cache-invalidation off + subvolumes ta-locks +end-volume + +volume ta-io-threads + type performance/io-threads + subvolumes ta-upcall +end-volume + +volume ta-index + type features/index + option xattrop-pending-watchlist trusted.afr.ta- + option xattrop-dirty-watchlist trusted.afr.dirty + option index-base /mnt/thin-arbiter/.glusterfs/indices + subvolumes ta-io-threads +end-volume + +volume /mnt/thin-arbiter + type debug/io-stats + option count-fop-hits off + option latency-measurement off + option unique-id /mnt/thin-arbiter + subvolumes ta-index +end-volume + +volume ta-server + type protocol/server + option transport.listen-backlog 10 + option transport.socket.keepalive-count 9 + option transport.socket.keepalive-interval 2 + option transport.socket.keepalive-time 20 + option transport.tcp-user-timeout 0 + option transport.socket.keepalive 1 + option auth.addr./mnt/thin-arbiter.allow * + option auth-path /mnt/thin-arbiter + option transport.address-family inet + option transport-type tcp + subvolumes /mnt/thin-arbiter +end-volume diff --git a/extras/volfilter.py b/extras/volfilter.py index 0ca456a7882..5558a1beff4 100644 --- a/extras/volfilter.py +++ b/extras/volfilter.py @@ -13,6 +13,7 @@ # You should have received a copy of the GNU General Public License * along # with HekaFS. If not, see <http://www.gnu.org/licenses/>. +from __future__ import print_function import copy import string import sys @@ -35,7 +36,7 @@ good_xlators = [ "storage/posix", ] -def copy_stack (old_xl,suffix,recursive=False): +def copy_stack (old_xl, suffix, recursive=False): if recursive: new_name = old_xl.name + "-" + suffix else: @@ -45,7 +46,7 @@ def copy_stack (old_xl,suffix,recursive=False): # The results with normal assignment here are . . . amusing. new_xl.opts = copy.deepcopy(old_xl.opts) for sv in old_xl.subvols: - new_xl.subvols.append(copy_stack(sv,suffix,True)) + new_xl.subvols.append(copy_stack(sv, suffix, True)) # Patch up the path at the bottom. if new_xl.type == "storage/posix": new_xl.opts["directory"] += ("/" + suffix) @@ -63,10 +64,10 @@ def cleanup (parent, graph): parent.opts["transport-type"] = "ssl" sv = [] for child in parent.subvols: - sv.append(cleanup(child,graph)) + sv.append(cleanup(child, graph)) parent.subvols = sv else: - parent = cleanup(parent.subvols[0],graph) + parent = cleanup(parent.subvols[0], graph) return parent class Translator: @@ -82,8 +83,8 @@ class Translator: def load (path): # If it's a string, open it; otherwise, assume it's already a # file-like object (most notably from urllib*). - if type(path) in types.StringTypes: - fp = file(path,"r") + if type(path) in (str,): + fp = file(path, "r") else: fp = path all_xlators = {} @@ -98,16 +99,16 @@ def load (path): continue if text[0] == "volume": if xlator: - raise RuntimeError, "nested volume definition" + raise RuntimeError("nested volume definition") xlator = Translator(text[1]) continue if not xlator: - raise RuntimeError, "text outside volume definition" + raise RuntimeError("text outside volume definition") if text[0] == "type": xlator.type = text[1] continue if text[0] == "option": - xlator.opts[text[1]] = string.join(text[2:]) + xlator.opts[text[1]] = ''.join(text[2:]) continue if text[0] == "subvolumes": for sv in text[1:]: @@ -118,25 +119,25 @@ def load (path): last_xlator = xlator xlator = None continue - raise RuntimeError, "unrecognized keyword %s" % text[0] + raise RuntimeError("unrecognized keyword %s" % text[0]) if xlator: - raise RuntimeError, "unclosed volume definition" + raise RuntimeError("unclosed volume definition") return all_xlators, last_xlator def generate (graph, last, stream=sys.stdout): for sv in last.subvols: if not sv.dumped: - generate(graph,sv,stream) - print >> stream, "" + generate(graph, sv, stream) + print("", file=stream) sv.dumped = True - print >> stream, "volume %s" % last.name - print >> stream, " type %s" % last.type - for k, v in last.opts.iteritems(): - print >> stream, " option %s %s" % (k, v) + print("volume %s" % last.name, file=stream) + print(" type %s" % last.type, file=stream) + for k, v in last.opts.items(): + print(" option %s %s" % (k, v), file=stream) if last.subvols: - print >> stream, " subvolumes %s" % string.join( - [ sv.name for sv in last.subvols ]) - print >> stream, "end-volume" + print(" subvolumes %s" % ''.join( + [ sv.name for sv in last.subvols ]), file=stream) + print("end-volume", file=stream) def push_filter (graph, old_xl, filt_type, opts={}): suffix = "-" + old_xl.type.split("/")[1] @@ -156,7 +157,7 @@ def push_filter (graph, old_xl, filt_type, opts={}): def delete (graph, victim): if len(victim.subvols) != 1: - raise RuntimeError, "attempt to delete non-unary translator" + raise RuntimeError("attempt to delete non-unary translator") for xl in graph.itervalues(): while xl.subvols.count(victim): i = xl.subvols.index(victim) @@ -164,4 +165,4 @@ def delete (graph, victim): if __name__ == "__main__": graph, last = load(sys.argv[1]) - generate(graph,last) + generate(graph, last) diff --git a/extras/who-wrote-glusterfs/gitdm.aliases b/extras/who-wrote-glusterfs/gitdm.aliases index e19b99c79c8..901c12418e3 100644 --- a/extras/who-wrote-glusterfs/gitdm.aliases +++ b/extras/who-wrote-glusterfs/gitdm.aliases @@ -16,11 +16,13 @@ anush@gluster.com ashetty@redhat.com csaba@gluster.com csaba@redhat.com csaba@lowlife.hu csaba@redhat.com csaba@zresearch.com csaba@redhat.com +gd@samba.org gd@redhat.com harsha@gluster.com fharshav@redhat.com harsha@zresearch.com fharshav@redhat.com harsha@dev.gluster.com fharshav@redhat.com harsha@harshavardhana.net fharshav@redhat.com jclift@redhat.com jclift@gluster.org +kkeithle@linux.keithley.org kkeithle@redhat.com kkeithle@f16node1.kkeithle.usersys.redhat.com kkeithle@redhat.com kaushal@gluster.com kaushal@redhat.com kaushikbv@gluster.com kbudiger@redhat.com @@ -32,6 +34,9 @@ me@louiszuckerman.com louiszuckerman@gmail.com msvbhat@gmail.com vbhat@redhat.com nullpai@gmail.com ppai@redhat.com vishwanath@gluster.com vbhat@redhat.com +obnox@samba.org madam@redhat.com +oleksandr@natalenko.name o.natalenko@lanet.ua +patrick@puiterwijk.org puiterwijk@fedoraproject.org pavan@dev.gluster.com pavan@gluster.com zaitcev@yahoo.com zaitcev@kotori.zaitcev.us pranithk@gluster.com pkarampu@redhat.com @@ -41,6 +46,8 @@ raghavendra@zresearch.com rgowdapp@redhat.com rahulcssjce@gmail.com rahulcs@redhat.com rajesh@gluster.com rajesh@redhat.com rajesh.amaravathi@gmail.com rajesh@redhat.com +root@ravi2.(none) ravishankar@redhat.com +sabansal@localhost.localdomain sabansal@redhat.com shehjart@zresearch.com shehjart@gluster.com venky@gluster.com vshankar@redhat.com vijay@gluster.com vbellur@redhat.com @@ -48,3 +55,4 @@ vijay@dev.gluster.com vbellur@redhat.com vijaykumar.koppad@gmail.com vkoppad@redhat.com vikas@zresearch.com vikas@gluster.com shishirng@gluster.com sgowda@redhat.com +potatogim@potatogim.net potatogim@gluesys.com diff --git a/extras/who-wrote-glusterfs/gitdm.domain-map b/extras/who-wrote-glusterfs/gitdm.domain-map index 39526f0f99c..7cd2bbd605b 100644 --- a/extras/who-wrote-glusterfs/gitdm.domain-map +++ b/extras/who-wrote-glusterfs/gitdm.domain-map @@ -2,15 +2,28 @@ # Here is a set of mappings of domain names onto employer names. # active.by ActiveCloud +appeartv.com Appear TV cern.ch CERN +cmss.chinamobile.com China Mobile(Suzhou) Software Technology +datalab.es DataLab S.L. +fb.com Facebook +fedoraproject.org Fedora Project gluster.com Red Hat -gmail.com (unknown) +gmail.com (personal contributions) gooddata.com GoodData hastexo.com hastexo +horde.com (personal contributions) ibm.com IBM +io.com IO +lanet.ua Lanet Network linbit.com LINBIT +nectec.or.th NECTEC netbsd.org NetBSD netdirect.ca Net Direct +nokia.com Nokia redhat.com Red Hat stepping-stone.ch stepping stone GmbH +xtaotech.com XTAO Co. +yahoo.in (personal contributions) zresearch.com Red Hat +gluesys.com Gluesys diff --git a/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh b/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh index 7a1896e6a90..70d5964c576 100755 --- a/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh +++ b/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh @@ -28,7 +28,7 @@ check_gitdm() { if [ ! -e "${GITDM_DIR}/gitdm" ] then - git clone --quiet git://git.lwn.net/gitdm.git ${DIRNAME}/gitdm + git clone --quiet ${GITDM_REPO} ${DIRNAME}/gitdm fi } |