diff options
| -rw-r--r-- | configure.ac | 3 | ||||
| -rw-r--r-- | doc/Makefile.am | 4 | ||||
| -rw-r--r-- | doc/glusterfs-volgen.8 | 82 | ||||
| -rw-r--r-- | extras/Makefile.am | 6 | ||||
| -rw-r--r-- | extras/glusterfs-defrag.in | 123 | ||||
| -rw-r--r-- | extras/volgen/Common.py | 83 | ||||
| -rw-r--r-- | extras/volgen/CreateVolfile.py | 684 | ||||
| -rw-r--r-- | extras/volgen/Makefile.am | 8 | ||||
| -rwxr-xr-x | extras/volgen/glusterfs-volgen.in | 227 | ||||
| -rw-r--r-- | glusterfs.spec.in | 3 |
10 files changed, 4 insertions, 1219 deletions
diff --git a/configure.ac b/configure.ac index 3ce78028f..a0fcbaa4d 100644 --- a/configure.ac +++ b/configure.ac @@ -111,15 +111,12 @@ AC_CONFIG_FILES([Makefile doc/examples/Makefile doc/hacker-guide/Makefile extras/Makefile - extras/glusterfs-defrag extras/init.d/Makefile extras/init.d/glusterd.plist extras/init.d/glusterd-Debian extras/init.d/glusterd-Redhat extras/init.d/glusterd-SuSE extras/benchmarking/Makefile - extras/volgen/Makefile - extras/volgen/glusterfs-volgen contrib/fuse-util/Makefile xlators/features/access-control/Makefile xlators/features/access-control/src/Makefile diff --git a/doc/Makefile.am b/doc/Makefile.am index 0eb348c18..f26fc7503 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -1,13 +1,13 @@ EXTRA_DIST = glusterfs.vol.sample glusterfsd.vol.sample glusterfs.8 mount.glusterfs.8\ porting_guide.txt authentication.txt coding-standard.pdf get_put_api_using_xattr.txt \ translator-options.txt mac-related-xattrs.txt replicate.pdf glusterd.vol gluster.8 \ - glusterd.8 glusterfs-volgen.8 + glusterd.8 SUBDIRS = examples hacker-guide voldir = $(sysconfdir)/glusterfs vol_DATA = glusterfs.vol.sample glusterfsd.vol.sample glusterd.vol -man8_MANS = glusterfs.8 mount.glusterfs.8 gluster.8 glusterd.8 glusterfs-volgen.8 +man8_MANS = glusterfs.8 mount.glusterfs.8 gluster.8 glusterd.8 CLEANFILES = diff --git a/doc/glusterfs-volgen.8 b/doc/glusterfs-volgen.8 deleted file mode 100644 index a47873783..000000000 --- a/doc/glusterfs-volgen.8 +++ /dev/null @@ -1,82 +0,0 @@ -.\" Copyright (c) 2009-2010 Gluster, Inc. <http://www.gluster.com> -.\" This file is part of GlusterFS. -.\" -.\" GlusterFS is free software; you can redistribute it and/or modify -.\" it under the terms of the GNU Affero General Public License as published -.\" by the Free Software Foundation; either version 3 of the License, -.\" or (at your option) any later version. -.\" -.\" GlusterFS is distributed in the hope that it will be useful, but -.\" WITHOUT ANY WARRANTY; without even the implied warranty of -.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -.\" Affero General Public License for more details. -.\" -.\" You should have received a copy of the GNU Affero General Public License -.\" long with this program. If not, see -.\" <http://www.gnu.org/licenses/>. -.\" -.\" -.\" -.TH GlusterFS 8 "Cluster Filesystem" "19 March 2010" "Gluster Inc." -.SH NAME -glusterfs-volgen \- Tool to generate GlusterFS volume files -.SH SYNOPSIS -.B glusterfs-volgen --name <volumename> -.I [options] <server1:/export/directory> <server1:/export/directory> ... -.PP -.SH DESCRIPTION -glusterfs-volgen tool is used to generate glusterfs volume files as per the -options given. - -GlusterFS volume files are very important component of gluster filesystem, -using which \fBglusterfs\fR(8) will behave as a client component or a server -component. Each glusterfs modules (in technical terms, a translator) is defined -in this volume files in certain order to achieve the required behavior from -filesystem. - -glusterfs-volgen tool abstracts the complexities of volumefile logic into a -simple command line tool, which based on user's requirement (by options -provided) generates the corresponding error-free volume file. - - -.SH OPTIONS -.PP - -\fB\-n \-\-name\fR=VOLUME-NAME -This is a mandatory argument required for glusterfs-volgen. -.TP -\fB\-r \-\-raid\fR=[0|1] -Supported options are 0 (stripe), 1 (mirror). If this option is not used, -distribute is used without replicate or stripe. -.TP -\fB\-t \-\-transport\fR=[tcp|ib-verbs|tcp,ib-verbs] -Supported types are 'tcp' and 'ib-verbs'. This option can take both types -together like 'tcp,ib-verbs'. [default is 'tcp'] -.TP -\fB\-p \-\-port\fR=PORT -The port number to use in volume files. -[default is 24007 for tcp, (24007 + 1) for ib-verbs] -.TP -\fB\-a \-\-auth\fR=AUTH_PARAMS -Authentication parameters (comma seperated) to use in server volume. -[default is '*', ie allow everyone] -.TP -\fB\-c \-\-conf-dir\fR=CONF_DIR -Directory where the output volume files are generated. [default is -${prefix}/etc/glusterfs] - -.PP -.SH FILES - -${CONF_DIR}/*.vol - -.SH SEE ALSO -.nf -\fB\fBmount.glusterfs\fR(8) -\fR -.fi -.SH COPYRIGHT -.nf -Copyright(c) 2009-2010 Gluster, Inc. <http://www.gluster.com> -\fR -.fi diff --git a/extras/Makefile.am b/extras/Makefile.am index 26e1e24b6..9deb3dfa1 100644 --- a/extras/Makefile.am +++ b/extras/Makefile.am @@ -3,9 +3,7 @@ docdir = $(datadir)/doc/glusterfs/ EditorModedir = $(docdir)/ EditorMode_DATA = glusterfs-mode.el glusterfs.vim -dist_bin_SCRIPTS = glusterfs-defrag +SUBDIRS = init.d benchmarking -SUBDIRS = init.d benchmarking volgen - -EXTRA_DIST = specgen.scm MacOSX/Portfile glusterfs-mode.el glusterfs.vim migrate-unify-to-distribute.sh backend-xattr-sanitize.sh backend-cleanup.sh glusterfs-defrag disk_usage_sync.sh +EXTRA_DIST = specgen.scm MacOSX/Portfile glusterfs-mode.el glusterfs.vim migrate-unify-to-distribute.sh backend-xattr-sanitize.sh backend-cleanup.sh disk_usage_sync.sh diff --git a/extras/glusterfs-defrag.in b/extras/glusterfs-defrag.in deleted file mode 100644 index 68a393f1c..000000000 --- a/extras/glusterfs-defrag.in +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash - -# Please leave 'added_bricks' as empty if you want 100% defrag. -# If you want to move data to newly added bricks, properly give -# brick info as "<hostname>:<export-dir>" form (which was given -# in 'gluster volume create' command) -# More than one brick can be given with space inbetween. - -# -# (c) 2010 Gluster Inc <http://www.gluster.com/> -# - -set -e; - -# -#added_bricks="node1:/gfs/export1" -# -added_bricks="" - -CP="cp" -MV="mv" - -scan_dir() -{ - path=$1; - # run defrag on files first # - find "$path" -maxdepth 1 -type f -perm +01000 -exec $0 '{}' \; - - for subdir in $(find "$path" -maxdepth 1 -type d | sed 1d); do - $0 "$subdir"; - done -} - -fix_xattr() -{ - path=$1; - getfattr -n trusted.distribute.fix.layout "$path" 2>/dev/null; -} - -rsync_filename() -{ - path=$1 - dir=$(dirname "$path"); - file=$(basename "$path"); - - echo "$dir/.$file.zr$$"; -} - -relocate_file() -{ - path=$1; - - # Make sure we don't 'defrag' valid file. - stat_info=$(stat -c '%a' "$path"); - if [ $stat_info -lt 1000 ] ; then - return; - fi - - size=$(stat -c '%s' "$path"); - - # If there are some entries in added_bricks, then check - # if the link file is present on those nodes, if not, - # set flag=1, so full defrag happens - - flag=0; - for bricks in ${added_bricks}; do - linknode=$(getfattr --only-values -n trusted.distribute.linkinfo $path 2>/dev/null); - if [ -z $linknode ] ; then - return; - fi - current_brick=${linknode:0:${#bricks}}; - if [ "${bricks}" == "${current_brick}" ]; then - flag=1; - fi - done - - if [ -z ${added_bricks} ] ; then - flag=1; - fi - - if [ $flag -ne 1 ]; then - return; - fi - - tmp_path=$(rsync_filename "$path"); - - pre_mtime=$(stat -c '%Y' "$path"); - $CP -a "$path" "$tmp_path"; - post_mtime=$(stat -c '%Y' "$path"); - - if [ $pre_mtime = $post_mtime ]; then - chmod -t "$tmp_path"; - $MV "$tmp_path" "$path"; - echo "file '$path' relocated" - else - echo "file '$path' modified during defrag. skipping" - rm -f "$tmp_path"; - fi -} - -defrag_usage() -{ - echo "Usage: $0 <directory>" -} - -main() -{ - path="$1"; - - if [ -z "$path" ]; then - defrag_usage; - return; - fi - - if [ -d "$path" ]; then - fix_xattr "$path"; - scan_dir "$path"; - else - relocate_file "$@"; - fi -} - -main "$1" diff --git a/extras/volgen/Common.py b/extras/volgen/Common.py deleted file mode 100644 index 298dee989..000000000 --- a/extras/volgen/Common.py +++ /dev/null @@ -1,83 +0,0 @@ -import os, sys, re, string - -def check_duplicate_entry(args): - """Check duplicate entries in incoming arguments""" - _tmp = [] - for server in args: - if server not in _tmp: - _tmp.append (server) - else: - print "Duplicate arguments detected (%s)" % server - raise ValueError - - return - -def args2dict(args): - - keyvalue = {} - for arg in args: - if int(arg.find(':')) == -1: - continue - first = arg.split(':')[0] - keyvalue[first] = [] - - for arg in args: - if int(arg.find(':')) == -1: - continue - first = arg.split(':')[0] - if arg.split(':')[1] not in keyvalue[first]: - if arg.split(':')[1][0] != '/': - print "Absolute export path required for %s" % arg - raise ValueError - keyvalue[first].append (arg.split(':')[1]) - - return keyvalue - -def args2array(args): - - array = [] - - for arg in args: - if int(arg.find(':')) == -1: - continue - array.append(arg) - - return array - -def list_export_vols(configdir, volumename): - - list_export = [] - if os.path.isdir(configdir): - for line in os.listdir(configdir): - if re.match(r'[a-zA-Z0-9_]\S+%s-export.vol' % volumename, line): - list_export.append(line) - - return list_export - -def get_old_server_args(exports, configdir): - - list_args = [] - for export in exports: - array = gfParser("%s/%s" % (configdir, export)) - for dt in array: - if dt.has_key('option'): - if re.match("\w+tory", dt['option']): - list_args.append(export.split('-')[0] + ":" + dt['option'].split()[1]) - - return list_args - -def gfParser (volfile): - - volfile_rl = open (volfile).readlines() - volume_array = [] - for line in volfile_rl: - line = line.strip() - volfile_dict = {} - if re.match(r"[a-zA-Z0-9_]+", line): - if line.split() > 1: - volfile_dict[line.split()[0]] = string.join (line.split()[1:], ' ') - else: - volfile_dict[line.split()[0]] = " " - volume_array.append(volfile_dict) - - return volume_array diff --git a/extras/volgen/CreateVolfile.py b/extras/volgen/CreateVolfile.py deleted file mode 100644 index 2f548d7c7..000000000 --- a/extras/volgen/CreateVolfile.py +++ /dev/null @@ -1,684 +0,0 @@ -# Copyright (c) 2009-2010 Gluster, Inc. <http://www.gluster.com> -# This file is part of GlusterFS. - -# GlusterFS is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published -# by the Free Software Foundation; either version 3 of the License, -# or (at your option) any later version. - -# GlusterFS is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see -# <http://www.gnu.org/licenses/>. - -import os, sys, string - -#Cachesize calculator -cache_size = "`echo $(( $(grep 'MemTotal' /proc/meminfo | sed 's/[^0-9]//g') / 5120 ))`" - -class CreateVolfile: - - def __init__ (self, server_dict, server, transport, - transports, options, server_array): - - self.host_dict = server_dict - self.host = server - self.volume_name = options.volume_name - self.transport = transport - self.transports = transports - self.auth_parameters = options.auth_param - self.raid_type = options.raid_type - self.ib_devport = options.ib_dev - self.conf_dir = options.conf_dir - self.host_array = server_array - self.enable_safe_mode = options.enable_safe_mode - self.volume_size_server = options.size_server - self.volume_size_client = options.size_client - self.nfs = options.need_nfs - self.num_replica = options.num_replica - self.num_stripe = options.num_stripe - if options.port: - self.gfs_port = options.port - self.gfs_ib_port = options.port + 1 - else: - self.gfs_port = 24007 - self.gfs_ib_port = 24008 - - def create_pmap_mount_volfile (self): - - raid_type = self.raid_type - - mount_volfile = "%s/%s-%s.vol" % (self.conf_dir, str(self.volume_name), str(self.transport)) - mount_fd = file ("%s" % (mount_volfile), "w") - - print "Generating client volfiles.. '%s'" % mount_volfile - - cmdline = string.join (sys.argv, ' ') - - mount_fd.write ("## file auto generated by %s\n" % sys.argv[0]) - mount_fd.write ("# Cmd line:\n") - mount_fd.write ("# $ %s\n\n" % cmdline) - - if raid_type is not None: - # Used for later usage - mount_fd.write ("# RAID %d\n" % raid_type) - - mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport) - subvolumes = [] - for host in self.host_dict.keys(): - i = 1 - for export in self.host_dict[host]: - mount_fd.write ("volume %s-%s\n" % (host,export)) - mount_fd.write (" type protocol/client\n") - mount_fd.write (" option transport-type %s\n" % - self.transport) - mount_fd.write (" option remote-host %s\n" % host) - if self.transport == 'ib-verbs': - mount_fd.write (" option transport.ib-verbs.port %d\n" % - self.ib_devport) - if self.transport == 'tcp': - mount_fd.write (" option transport.socket.nodelay on\n") - - mount_fd.write (" option remote-subvolume %s\n" % export) - mount_fd.write ("end-volume\n\n") - i += 1 - subvolumes.append(str(host) + '-' + str(export)) - - # Stripe section.. if given - if raid_type is 0: - max_stripe_idx = len (subvolumes) / self.num_stripe - stripe_idx = 0 - index = 0 - while index < max_stripe_idx: - mount_fd.write ("volume stripe-%d\n" % index) - mount_fd.write (" type cluster/stripe\n") - mount_fd.write ("# option block-size 128k\n") - mount_fd.write ("# option use-xattr no\n") - mount_fd.write (" subvolumes %s" % subvolumes[stripe_idx]) - sub_idx = 1 - while sub_idx < self.num_stripe: - mount_fd.write (" %s" % subvolumes[stripe_idx+sub_idx]) - sub_idx += 1 - mount_fd.write ("\nend-volume\n\n") - stripe_idx += self.num_stripe - index +=1 - - # Replicate section - if raid_type is 1: - max_mirror_idx = len (subvolumes) / self.num_replica - mirror_idx = 0 - index = 0 - while index < max_mirror_idx: - mount_fd.write ("volume mirror-%d\n" % index) - mount_fd.write (" type cluster/replicate\n") - mount_fd.write (" subvolumes %s" % subvolumes[mirror_idx]) - sub_idx = 1 - while sub_idx < self.num_replica: - mount_fd.write (" %s" % subvolumes[mirror_idx + sub_idx]) - sub_idx += 1 - mount_fd.write ("\nend-volume\n\n") - mirror_idx += self.num_replica - index += 1 - - # Distribute section - if raid_type is 0: - subvolumes = [] - flag = 0 - while flag < index: - subvolumes.append ("stripe-%d" % flag) - flag += 1 - - if raid_type is 1: - subvolumes = [] - flag = 0 - while flag < index: - subvolumes.append ("mirror-%d" % flag) - flag += 1 - - if len (subvolumes) > 1: - mount_fd.write ("volume distribute\n") - mount_fd.write (" type cluster/distribute\n") - mount_fd.write ("# option unhashed-sticky-bit yes" - " # Used for migrating data while adding new nodes\n") - mount_fd.write ("# option min-free-disk 5%" - " # Minimum free disk available on the volume\n") - mount_fd.write (" subvolumes %s\n" % - string.join (subvolumes,' ')) - mount_fd.write ("end-volume\n\n") - subvolumes[0] = "distribute" - - if self.volume_size_client: - mount_fd.write ("volume quota\n") - mount_fd.write (" type features/quota\n") - mount_fd.write (" option disk-usage-limit %s\n" % self.volume_size_client) - mount_fd.write ("# option minimum-free-disk-limit 10GB\n" - "# minimum free disk value (default) 0\n") - mount_fd.write ("# option refresh-interval 10\n") - mount_fd.write (" subvolumes %s\n" % subvolumes[0]) - mount_fd.write ("end-volume\n\n") - subvolumes[0] = "quota" - - if self.enable_safe_mode: - return - - self.performance_mode (subvolumes[0], mount_fd) - - return - - - def performance_mode (self, cluster_subvolume, volfile_fd): - - volfile_fd.write ("volume writebehind\n") - volfile_fd.write (" type performance/write-behind\n") - volfile_fd.write (" option cache-size 4MB\n") - volfile_fd.write ("# option enable-trickling-writes yes # Flush final write calls when network is free\n") - volfile_fd.write ("# option enable-O_SYNC yes # Enable O_SYNC for write-behind\n") - volfile_fd.write ("# option disable-for-first-nbytes 1 # Disable first nbytes with very small initial writes\n") - if self.volume_size_client: - volfile_fd.write (" subvolumes quota\n") - else: - volfile_fd.write (" subvolumes %s\n" % cluster_subvolume) - volfile_fd.write ("end-volume\n\n") - - volfile_fd.write ("volume readahead\n") - volfile_fd.write (" type performance/read-ahead\n") - volfile_fd.write (" option page-count 4\n") - volfile_fd.write ("# option force-atime-update yes # force updating atimes, default off\n") - volfile_fd.write (" subvolumes writebehind\n") - volfile_fd.write ("end-volume\n\n") - - volfile_fd.write ("volume iocache\n") - volfile_fd.write (" type performance/io-cache\n") - volfile_fd.write (" option cache-size %sMB\n" % cache_size) - volfile_fd.write (" option cache-timeout 1\n") - volfile_fd.write ("# option priority *.html:1,abc*:2 # Priority list for iocaching files\n") - volfile_fd.write (" subvolumes readahead\n") - volfile_fd.write ("end-volume\n\n") - - volfile_fd.write ("volume quickread\n") - volfile_fd.write (" type performance/quick-read\n") - volfile_fd.write (" option cache-timeout 1\n") - volfile_fd.write (" option max-file-size 64kB\n") - volfile_fd.write (" subvolumes iocache\n") - volfile_fd.write ("end-volume\n\n") - - volfile_fd.write ("volume statprefetch\n") - volfile_fd.write (" type performance/stat-prefetch\n") - volfile_fd.write (" subvolumes quickread\n") - volfile_fd.write ("end-volume\n\n") - - return - - def create_pmap_nfs_volfile (self): - - raid_type = self.raid_type - - mount_volfile = "%s/%s-%s-nfs.vol" % (self.conf_dir, str(self.volume_name), str(self.transport)) - mount_fd = file ("%s" % (mount_volfile), "w") - - print "Generating client volfiles.. '%s'" % mount_volfile - - cmdline = string.join (sys.argv, ' ') - - mount_fd.write ("## file auto generated by %s\n" % sys.argv[0]) - mount_fd.write ("# Cmd line:\n") - mount_fd.write ("# $ %s\n\n" % cmdline) - - if raid_type is not None: - # Used for later usage - mount_fd.write ("# RAID %d\n" % raid_type) - - mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport) - subvolumes = [] - for host in self.host_dict.keys(): - i = 1 - for export in self.host_dict[host]: - mount_fd.write ("volume %s-%d\n" % (host,i)) - mount_fd.write (" type protocol/client\n") - mount_fd.write (" option transport-type %s\n" % - self.transport) - mount_fd.write (" option remote-host %s\n" % host) - if self.transport == 'ib-verbs': - mount_fd.write (" option transport.ib-verbs.port %d\n" % - self.ib_devport) - if self.transport == 'tcp': - mount_fd.write (" option transport.socket.nodelay on\n") - - mount_fd.write (" option remote-subvolume %s\n" % export) - mount_fd.write ("end-volume\n\n") - subvolumes.append(str(host) + '-' + str(i)) - i += 1 - - # Stripe section.. if given - if raid_type is 0: - max_stripe_idx = len (subvolumes) / self.num_stripe - stripe_idx = 0 - index = 0 - while index < max_stripe_idx: - mount_fd.write ("volume stripe-%d\n" % index) - mount_fd.write (" type cluster/stripe\n") - mount_fd.write ("# option block-size 128k\n") - mount_fd.write ("# option use-xattr no\n") - mount_fd.write (" subvolumes %s" % subvolumes[stripe_idx]) - sub_idx = 1 - while sub_idx < self.num_stripe: - mount_fd.write (" %s" % subvolumes[stripe_idx+sub_idx]) - sub_idx += 1 - mount_fd.write ("\nend-volume\n\n") - stripe_idx += self.num_stripe - index +=1 - - # Replicate section - if raid_type is 1: - max_mirror_idx = len (subvolumes) / self.num_replica - mirror_idx = 0 - index = 0 - while index < max_mirror_idx: - mount_fd.write ("volume mirror-%d\n" % index) - mount_fd.write (" type cluster/replicate\n") - mount_fd.write (" subvolumes %s" % subvolumes[mirror_idx]) - sub_idx = 1 - while sub_idx < self.num_replica: - mount_fd.write (" %s" % subvolumes[mirror_idx + sub_idx]) - sub_idx += 1 - mount_fd.write ("\nend-volume\n\n") - mirror_idx += self.num_replica - index += 1 - - # Distribute section - if raid_type is 0: - subvolumes = [] - flag = 0 - while flag < index: - subvolumes.append ("stripe-%d" % flag) - flag += 1 - - if raid_type is 1: - subvolumes = [] - flag = 0 - while flag < index: - subvolumes.append ("mirror-%d" % flag) - flag += 1 - - if len (subvolumes) > 1: - mount_fd.write ("volume distribute\n") - mount_fd.write (" type cluster/distribute\n") - mount_fd.write ("# option unhashed-sticky-bit yes" - " # Used for migrating data while adding new nodes\n") - mount_fd.write ("# option min-free-disk 5%" - " # Minimum free disk available on the volume\n") - mount_fd.write (" subvolumes %s\n" % - string.join (subvolumes,' ')) - mount_fd.write ("end-volume\n\n") - subvolumes[0] = "distribute" - - - mount_fd.write ("volume nfsxlator\n") - mount_fd.write (" type nfs/server\n") - mount_fd.write (" subvolumes %s\n" % subvolumes[0]) - mount_fd.write ("# option rpc-auth.auth-unix off #Enabled by default\n") - mount_fd.write ("# option rpc-auth.auth-null off #Enabled by default\n") - mount_fd.write ("# By default all addresses are rejected until allowed.\n") - mount_fd.write ("# option rpc-auth.addr.reject 127.*\n") - mount_fd.write ("# option rpc-auth.addr.allow localhost\n") - mount_fd.write (" option rpc-auth.addr.%s.allow %s\n" % (subvolumes[0], self.auth_parameters)) - mount_fd.write ("# By default insecure ports are not allowed.\n") - mount_fd.write ("# option rpc-auth.ports.insecure on\n") - mount_fd.write ("# option rpc-auth.ports.<volume>.insecure on\n") - mount_fd.write ("# By default all access is read-write.\n") - mount_fd.write ("# option nfs3.<volume>.volume-access read-only\n") - mount_fd.write ("# option nfs3.<volume>.volume-access read-only\n") - mount_fd.write ("# option nfs3.read-size 128Kb\n") - mount_fd.write ("# option nfs3.write-size 32Kb\n") - mount_fd.write ("# option nfs3.readdir-size 64Kb\n") - mount_fd.write ("# option nfs3.<volume>.read-size 64Kb\n") - mount_fd.write ("# option nfs3.<volume>.write-size 64Kb\n") - mount_fd.write ("# option nfs3.posix1.readdir-size 128Kb\n") - mount_fd.write ("end-volume\n\n") - - return - - def create_pmap_export_volfile (self): - - cmdline = string.join (sys.argv, ' ') - i = 0 - for export in self.host_dict[self.host]: - export_volfile = "%s/%s-%d.vol" % (self.conf_dir, - str(self.volume_name + '.' + self.host), - i) - i += 1 - exp_fd = file ("%s" % (export_volfile),"w") - - print "Generating server volfiles.. for server %s as '%s'" % (self.host, - export_volfile) - - exp_fd.write ("## file auto generated by %s\n" % sys.argv[0]) - exp_fd.write ("# Cmd line:\n") - exp_fd.write ("# $ %s\n\n" % cmdline) - exp_fd.write ("volume posix\n") - exp_fd.write (" type storage/posix\n") - exp_fd.write ("# option o-direct enable # (default: disable) boolean type only\n") - exp_fd.write ("# option export-statfs-size no # (default: yes) boolean type only\n") - exp_fd.write ("# option mandate-attribute off # (default: on) boolean type only\n") - exp_fd.write ("# option span-devices 8 # (default: 0) integer value\n") - exp_fd.write ("# option background-unlink yes # (default: no) boolean type\n") - - exp_fd.write (" option directory %s\n" % export) - exp_fd.write ("end-volume\n\n") - - if self.nfs: - exp_fd.write ("volume posix-ac\n") - exp_fd.write (" type features/access-control\n") - exp_fd.write (" subvolumes posix\n") - exp_fd.write ("end-volume\n\n") - - if self.volume_size_server: - exp_fd.write ("volume quota\n") - exp_fd.write (" type features/quota\n") - exp_fd.write (" option disk-usage-limit %s\n" % self.volume_size_server) - exp_fd.write ("# option minimum-free-disk-limit 10GB" - " # minimum free disk value (default) 0\n") - exp_fd.write ("# option refresh-interval 10\n") - if self.nfs: - exp_fd.write (" subvolumes posix-ac\n") - else: - exp_fd.write (" subvolumes posix\n") - exp_fd.write ("end-volume\n\n") - - exp_fd.write ("volume locks\n") - exp_fd.write (" type features/locks\n") - exp_fd.write ("# option mandatory on # Default off, used in specific applications\n") - if self.volume_size_server: - exp_fd.write (" subvolumes quota\n") - elif self.nfs: - exp_fd.write (" subvolumes posix-ac\n") - else: - exp_fd.write (" subvolumes posix\n") - exp_fd.write ("end-volume\n\n") - - exp_fd.write ("volume replace-brick\n") - exp_fd.write (" type protocol/client\n") - if self.transport: - exp_fd.write (" option transport-type %s\n" % self.transport) - if self.gfs_port: - exp_fd.write (" option transport.remote-port 34034\n") - exp_fd.write (" option ping-timeout 42\n") - exp_fd.write ("end-volume\n\n") - - exp_fd.write ("volume pump\n") - exp_fd.write (" type cluster/pump\n") - exp_fd.write (" subvolumes locks replace-brick\n") - exp_fd.write ("end-volume\n\n") - - exp_fd.write ("volume %s\n" % export) - exp_fd.write (" type performance/io-threads\n") - exp_fd.write (" option thread-count 8\n") - exp_fd.write ("# option autoscaling yes # Heuristic for autoscaling threads on demand\n") - exp_fd.write ("# option min-threads 2 # min count for thread pool\n") - exp_fd.write ("# option max-threads 64 # max count for thread pool\n") - exp_fd.write (" subvolumes pump\n") - exp_fd.write ("end-volume\n\n") - - for transport in self.transports: - exp_fd.write ("volume server-%s\n" % export) - exp_fd.write (" type protocol/server\n") - exp_fd.write (" option transport-type %s\n" % transport) - exp_fd.write (" option auth.addr.%s.allow %s\n" % - (export, self.auth_parameters)) - if transport == 'ib-verbs': - exp_fd.write (" option transport.ib-verbs.port %d\n" % - self.ib_devport) - if transport == 'tcp': - exp_fd.write (" option transport.socket.nodelay on\n") - exp_fd.write (" option transport.socket.listen-port %d\n" % self.gfs_port) - exp_fd.write (" subvolumes %s\n" % export) - exp_fd.write ("end-volume\n\n") - - return - - - def create_mount_volfile (self): - - raid_type = self.raid_type - - mount_volfile = "%s/%s-%s.vol" % (self.conf_dir, str(self.volume_name), str(self.transport)) - mount_fd = file ("%s" % (mount_volfile), "w") - - print "Generating client volfiles.. '%s'" % mount_volfile - - cmdline = string.join (sys.argv, ' ') - - mount_fd.write ("## file auto generated by %s\n" % sys.argv[0]) - mount_fd.write ("# Cmd line:\n") - mount_fd.write ("# $ %s\n\n" % cmdline) - - if raid_type is not None: - # Used for later usage - mount_fd.write ("# RAID %d\n" % raid_type) - - mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport) - subvolumes = [] - for host in self.host_dict.keys(): - i = 1 - for exports in self.host_dict[host]: - mount_fd.write ("volume %s-%s\n" % (host,exports)) - mount_fd.write (" type protocol/client\n") - mount_fd.write (" option transport-type %s\n" % - self.transport) - mount_fd.write (" option remote-host %s\n" % host) - if self.transport == 'ib-verbs': - mount_fd.write (" option transport.ib-verbs.port %d\n" % - self.ib_devport) - mount_fd.write (" option remote-port %d\n" % - self.gfs_ib_port) - if self.transport == 'tcp': - mount_fd.write (" option transport.socket.nodelay on\n") - mount_fd.write (" option remote-port %d\n" % - self.gfs_port) - - mount_fd.write (" option remote-subvolume %s\n" % exports) - mount_fd.write ("end-volume\n\n") - subvolumes.append(str(host) + '-' + str(exports)) - - # Stripe section.. if given - if raid_type is 0: - max_stripe_idx = len (subvolumes) / self.num_stripe - stripe_idx = 0 - index = 0 - while index < max_stripe_idx: - mount_fd.write ("volume stripe-%d\n" % index) - mount_fd.write (" type cluster/stripe\n") - mount_fd.write ("# option block-size 128k\n") - mount_fd.write ("# option use-xattr no\n") - mount_fd.write (" subvolumes %s" % subvolumes[stripe_idx]) - sub_idx = 1 - while sub_idx < self.num_stripe: - mount_fd.write (" %s" % subvolumes[stripe_idx+sub_idx]) - sub_idx += 1 - mount_fd.write ("\nend-volume\n\n") - stripe_idx += self.num_stripe - index +=1 - - # Replicate section - if raid_type is 1: - max_mirror_idx = len (subvolumes) / self.num_replica - mirror_idx = 0 - index = 0 - while index < max_mirror_idx: - mount_fd.write ("volume mirror-%d\n" % index) - mount_fd.write (" type cluster/replicate\n") - mount_fd.write (" subvolumes %s" % subvolumes[mirror_idx]) - sub_idx = 1 - while sub_idx < self.num_replica: - mount_fd.write (" %s" % subvolumes[mirror_idx + sub_idx]) - sub_idx += 1 - mount_fd.write ("\nend-volume\n\n") - mirror_idx += self.num_replica - index += 1 - - # Distribute section - if raid_type is 0: - subvolumes = [] - flag = 0 - while flag < index: - subvolumes.append ("stripe-%d" % flag) - flag += 1 - - if raid_type is 1: - subvolumes = [] - flag = 0 - while flag < index: - subvolumes.append ("mirror-%d" % flag) - flag += 1 - - if len (subvolumes) > 1: - mount_fd.write ("volume distribute\n") - mount_fd.write (" type cluster/distribute\n") - mount_fd.write ("# option unhashed-sticky-bit yes" - " # Used for migrating data while adding new nodes\n") - mount_fd.write ("# option min-free-disk 5%" - " # Minimum free disk available on the volume\n") - mount_fd.write (" subvolumes %s\n" % - string.join (subvolumes,' ')) - mount_fd.write ("end-volume\n\n") - subvolumes[0] = "distribute" - - - if self.nfs: - mount_fd.write ("volume nfsxlator\n") - mount_fd.write (" type nfs/server\n") - mount_fd.write (" subvolumes %s\n" % subvolumes[0]) - mount_fd.write ("# option rpc-auth.auth-unix off #Enabled by default\n") - mount_fd.write ("# option rpc-auth.auth-null off #Enabled by default\n") - mount_fd.write ("# By default all addresses are rejected until allowed.\n") - mount_fd.write ("# option rpc-auth.addr.reject 127.*\n") - mount_fd.write ("# option rpc-auth.addr.allow localhost\n") - mount_fd.write (" option rpc-auth.addr.%s.allow %s\n" % (subvolumes[0], self.auth_parameters)) - mount_fd.write ("# By default insecure ports are not allowed.\n") - mount_fd.write ("# option rpc-auth.ports.insecure on\n") - mount_fd.write ("# option rpc-auth.ports.<volume>.insecure on\n") - mount_fd.write ("# By default all access is read-write.\n") - mount_fd.write ("# option nfs3.<volume>.volume-access read-only\n") - mount_fd.write ("# option nfs3.<volume>.volume-access read-only\n") - mount_fd.write ("# option nfs3.read-size 128Kb\n") - mount_fd.write ("# option nfs3.write-size 32Kb\n") - mount_fd.write ("# option nfs3.readdir-size 64Kb\n") - mount_fd.write ("# option nfs3.<volume>.read-size 64Kb\n") - mount_fd.write ("# option nfs3.<volume>.write-size 64Kb\n") - mount_fd.write ("# option nfs3.posix1.readdir-size 128Kb\n") - mount_fd.write ("end-volume\n\n") - return - - if self.volume_size_client: - mount_fd.write ("volume quota\n") - mount_fd.write (" type features/quota\n") - mount_fd.write (" option disk-usage-limit %s\n" % self.volume_size_client) - mount_fd.write ("# option minimum-free-disk-limit 10GB\n" - "# minimum free disk value (default) 0\n") - mount_fd.write ("# option refresh-interval 10\n") - mount_fd.write (" subvolumes %s\n" % subvolumes[0]) - mount_fd.write ("end-volume\n\n") - subvolumes[0] = "quota" - - if self.enable_safe_mode: - return - - self.performance_mode (subvolumes[0], mount_fd) - - return - - def create_export_volfile (self): - - cmdline = string.join (sys.argv, ' ') - export_volfile = "%s/%s-export.vol" % (self.conf_dir, str(self.host + '-' + self.volume_name)) - exp_fd = file ("%s" % (export_volfile),"w") - - print "Generating server volfiles.. for server %s as '%s'" % (self.host, - export_volfile) - - exp_fd.write ("## file auto generated by %s\n" % - sys.argv[0]) - exp_fd.write ("# Cmd line:\n") - exp_fd.write ("# $ %s\n\n" % cmdline) - total_bricks = [] - i=1 - for export in self.host_dict[self.host]: - exp_fd.write ("volume posix%d\n" % i) - exp_fd.write (" type storage/posix\n") - exp_fd.write ("# option o-direct enable # (default: disable) boolean type only\n") - exp_fd.write ("# option export-statfs-size no # (default: yes) boolean type only\n") - exp_fd.write ("# option mandate-attribute off # (default: on) boolean type only\n") - exp_fd.write ("# option span-devices 8 # (default: 0) integer value\n") - exp_fd.write ("# option background-unlink yes # (default: no) boolean type\n") - - exp_fd.write (" option directory %s\n" % export) - exp_fd.write ("end-volume\n\n") - - if self.nfs: - exp_fd.write ("volume posix-ac%d\n" % i) - exp_fd.write (" type features/access-control\n") - exp_fd.write (" subvolumes posix%d\n" % i) - exp_fd.write ("end-volume\n\n") - - if self.volume_size_server: - exp_fd.write ("volume quota%d\n" % i) - exp_fd.write (" type features/quota\n") - exp_fd.write (" option disk-usage-limit %s\n" % self.volume_size_server) - exp_fd.write ("# option minimum-free-disk-limit 10GB" - " # minimum free disk value (default) 0\n") - exp_fd.write ("# option refresh-interval 10\n") - if self.nfs: - exp_fd.write (" subvolumes posix-ac%d\n" % i) - else: - exp_fd.write (" subvolumes posix%d\n" % i) - exp_fd.write ("end-volume\n\n") - - - exp_fd.write ("volume locks%d\n" % i) - exp_fd.write (" type features/locks\n") - exp_fd.write ("# option mandatory on # Default off, used in specific applications\n") - if self.volume_size_server: - exp_fd.write (" subvolumes quota%d\n" % i) - else: - exp_fd.write (" subvolumes posix%d\n" % i) - exp_fd.write ("end-volume\n\n") - - exp_fd.write ("volume %s\n" % export) - exp_fd.write (" type performance/io-threads\n") - exp_fd.write (" option thread-count 8\n") - exp_fd.write ("# option autoscaling yes # Heuristic for autoscaling threads on demand\n") - exp_fd.write ("# option min-threads 2 # min count for thread pool\n") - exp_fd.write ("# option max-threads 64 # max count for thread pool\n") - - exp_fd.write (" subvolumes locks%d\n" % i) - exp_fd.write ("end-volume\n\n") - - total_bricks.append("%s" % export) - i += 1 - - for transport in self.transports: - exp_fd.write ("volume server-%s\n" % transport) - exp_fd.write (" type protocol/server\n") - exp_fd.write (" option transport-type %s\n" % transport) - for brick in total_bricks: - exp_fd.write (" option auth.addr.%s.allow %s\n" % - (brick, self.auth_parameters)) - - if transport == 'ib-verbs': - exp_fd.write (" option transport.ib-verbs.listen-port %d\n" % self.gfs_ib_port) - exp_fd.write (" option transport.ib-verbs.port %d\n" % - self.ib_devport) - if transport == 'tcp': - exp_fd.write (" option transport.socket.listen-port %d\n" % self.gfs_port) - exp_fd.write (" option transport.socket.nodelay on\n") - - exp_fd.write (" subvolumes %s\n" % - string.join(total_bricks, ' ')) - exp_fd.write ("end-volume\n\n") - - return - diff --git a/extras/volgen/Makefile.am b/extras/volgen/Makefile.am deleted file mode 100644 index e36bd159b..000000000 --- a/extras/volgen/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ - -volgendir = $(datadir)/glusterfs -dist_volgen_DATA = CreateVolfile.py Common.py - -dist_bin_SCRIPTS = glusterfs-volgen - -CLEANFILES = - diff --git a/extras/volgen/glusterfs-volgen.in b/extras/volgen/glusterfs-volgen.in deleted file mode 100755 index 4b3742366..000000000 --- a/extras/volgen/glusterfs-volgen.in +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2009-2010 Gluster, Inc. <http://www.gluster.com> -# This file is part of GlusterFS. - -# GlusterFS is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published -# by the Free Software Foundation; either version 3 of the License, -# or (at your option) any later version. - -# GlusterFS is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see -# <http://www.gnu.org/licenses/>. - - -import getopt, sys, os, string -from optparse import OptionParser,OptionGroup,make_option - -prefix = "@prefix@" - -if not (prefix + "/share/glusterfs") in sys.path: - sys.path.append(prefix + "/share/glusterfs") - -from CreateVolfile import * -from Common import * - -def generate_volume_files (): - - usage_str = "%s%s" % ("%prog: -n <VOLUMENAME> -t <TRANSPORT> -p <NUMBER> -a <AUTH> ", - "-r <TYPE> [--num-replica N] [--num-stripe N]") - version_str = "%prog 3.0" - desc_str = "A tool to generate volume files for GlusterFS." - - parse = OptionParser(usage=usage_str, version=version_str, description=desc_str) - - # Basic option list - group = OptionGroup(parse, "Basic Options") - group.add_option("-n", "--name", dest="volume_name", - help="<volume-name>") - group.add_option("-t", "--transport", dest="transport_type", - default="tcp", help="tcp,ib-verbs default: tcp") - group.add_option("-p", "--port", type="int", - dest="port", help="<port> number") - group.add_option("--auth", dest="auth_param", default="*", - help="comma seperated ip range") - group.add_option("-r", "--raid", type="int", dest="raid_type", - help="0|1") - group.add_option("-c", "--conf-dir", dest="conf_dir", - default=os.getcwd(), - help="output directory for volume files") - group.add_option("--nfs", action="store_true", dest="need_nfs", - default=False, help="enable nfs translator") - parse.add_option_group(group) - - # Advanced option list - group = OptionGroup(parse, "Advanced Options") - group.add_option("--ibdev", type="int", dest="ib_dev", - default=1, help="Infiniband device number <N>") - group.add_option("--volume-size-server", dest="size_server", - help="volume size for each server") - group.add_option("--volume-size-client", dest="size_client", - help="volume size for each client") - group.add_option("--safe-mode", action="store_true", - dest="enable_safe_mode", default=False, - help="generate volume files in safe mode") - group.add_option("--num-replica", type="int", dest="num_replica", - help="number of file replica needed") - group.add_option("--num-stripe", type="int", dest="num_stripe", - help="number of stripe count needed") - group.add_option("--portmapper-mode", action="store_true", dest="portmapper", - default=False, help="generate volume files for glusterfs portmapper friendly way") - - parse.add_option_group(group) - - # Changes for Dynamic Volume Manager - #group = OptionGroup(parse, "Dynamic Volume Options") - #group.add_option("--add-server", dest="add_server_args", - # help="Add server to existing volume") - #group.add_option("--migrate", dest="migrate_server_args", - # help="Migrate servers from existing volumes") - - #parse.add_option_group(group) - - (options, args) = parse.parse_args() - - if options.volume_name is None: - print "Error: volume name is mandatory, please provide volume name" - raise ValueError - - if options.need_nfs and options.size_client: - print "Error: Currently quota not supported with native NFS, please use server side instead" - raise ValueError - - if options.transport_type: - transports = options.transport_type.split(',') - for transport in transports: - if (transport != "tcp" and transport != "ib-verbs"): - print "Error: --transport: option " + transport + \ - " is not valid transport type" - raise ValueError - - if options.raid_type: - if (options.raid_type != 1 and options.raid_type != 0): - print "Error: --raid: option " + str(options.raid_type) + " is not valid raid type" - raise ValueError - - if options.conf_dir: - if not os.path.isdir(options.conf_dir): - print "Specified directory %s doesn't exist" % options.conf_dir - raise ValueError - - server_dict = {} - - # Dynamic Volume Manager - #if options.add_server_args: - # add_server_list = (options.add_server_args).strip().split() - # check_duplicate_entry(add_server_list) - # old_server_list = get_old_server_args(list_export_vols(options.conf_dir, - # options.volume_name), - # options.conf_dir) - # for new_server in add_server_list: - # old_server_list.append(new_server) - - # server_dict = args2dict(old_server_list) - # server_array = args2array(old_server_list) - - # if len (server_dict.keys()) == 0: - # print "Error: no servers provided, please provide atleast one server" - # raise ValueError - - #else: - check_duplicate_entry(args) - server_dict = args2dict(args) - server_array = args2array(args) - - if len (server_dict.keys()) == 0: - print "Error: no servers provided, please provide atleast one server" - raise ValueError - - if options.num_replica is None: - options.num_replica = 2 - - if options.num_stripe is None: - options.num_stripe = 4 - - if options.num_replica < 2: - print "--num-replica option (%d) is not valid" % options.num_replica - - if options.num_stripe < 2: - print "--num-stripe option (%d) is not valid" % options.num_stripe - - if options.raid_type == 1: - if (len(server_array) % options.num_replica) != 0: - print "raid type (%d) and number of volumes (%d) invalid" % (options.raid_type, - len(server_array)) - raise ValueError - - if options.raid_type == 0: - if (len(server_array) % options.num_stripe) != 0: - print "raid type (%d) and number of volumes (%d) invalid" % (options.raid_type, - len(server_array)) - raise ValueError - - if options.portmapper is True: - for server in server_dict.keys(): - create_exp = CreateVolfile (server_dict, server, - None, transports, - options, None) - try: - create_exp.create_pmap_export_volfile () - except IOError, (errno, strerror): - print "Got %s creating server volfiles for %s" % (strerror, server) - - for transport in transports: - create_mnt = CreateVolfile (server_dict, None, - transport, transports, - options, server_array) - try: - create_mnt.create_pmap_mount_volfile () - except IOError, (errno, strerror): - print "Got %s creating client volfiles for transport '%s'" % (strerror, transport) - - if options.need_nfs: - for transport in transports: - create_mnt = CreateVolfile (server_dict, None, - transport, transports, - options, server_array) - try: - create_mnt.create_pmap_nfs_volfile () - except IOError, (errno, strerror): - print "Got %s creating nfs volfiles for transport '%s'" % (strerror, transport) - - return - - for server in server_dict.keys(): - create_exp = CreateVolfile (server_dict, server, - None, transports, - options, None) - try: - create_exp.create_export_volfile () - except IOError, (errno, strerror): - print "Got %s creating server volfiles for %s" % (strerror, server) - - for transport in transports: - create_mnt = CreateVolfile (server_dict, None, - transport, transports, - options, server_array) - try: - create_mnt.create_mount_volfile () - except IOError, (errno, strerror): - print "Got %s creating client volfiles for transport '%s'" % (strerror, transport) - - -def main (): - - try: - generate_volume_files() - except ValueError: - sys.exit(1) - - -main() diff --git a/glusterfs.spec.in b/glusterfs.spec.in index 85ae0d6aa..2b7eb4d9c 100644 --- a/glusterfs.spec.in +++ b/glusterfs.spec.in @@ -151,17 +151,14 @@ fi %{_bindir}/fusermount-glusterfs %endif %{_datadir}/glusterfs -%{_bindir}/glusterfs-volgen %{_libdir}/glusterfs %{_libdir}/*.so.* %{_sbindir}/glusterfs* -%{_bindir}/glusterfs-defrag %{_sbindir}/gluster %{_sbindir}/glusterd %{_mandir}/man8/*glusterfs.8* %{_mandir}/man8/*gluster.8* %{_mandir}/man8/*glusterd.8* -%{_mandir}/man8/*glusterfs-volgen.8* %dir /var/log/glusterfs %if 0%{!?_without_rdma:1} %exclude %{_libdir}/glusterfs/%{version}/rpc-transport/rdma* |