Volgen rewritten using option parser and added proper support for booster.

Signed-off-by: Harshavardhana <harsha@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 411 (Rewrite volgen using option parser and extend cifs/nfs support)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=411

3 files changed, 274 insertions, 211 deletions

diff --git a/extras/volgen/CreateBooster.py b/extras/volgen/CreateBooster.py
index 9051f44017f..3f56db34d3e 100644
--- a/extras/volgen/CreateBooster.py
+++ b/extras/volgen/CreateBooster.py
@@ -7,50 +7,71 @@ fstype = "glusterfs"
 
 class CreateBooster:
 
-    def __init__ (self, main_name, export_dir):
+    def __init__ (self, options):
 
-        self.volume_name = main_name
-        self.export = export_dir
+        self.volume_name = options.volume_name
+        self.need_nfs = options.need_nfs
+        self.need_cifs = options.need_cifs
+        self.username = options.cifs_username
+        self.enable_guest = options.enable_guest
 
     def configure_booster_fstab (self):
-        if self.volume_name is None or self.export is None:
-            sys.exit(1)
 
         booster_fstab_fd = file (GLUSTERFS_BOOSTER_FSTAB, "a")
-        _fstab = "%s/%s.vol  %s" % (CONFDIR, self.volume_name, self.export)
-        _options = "%s subvolume=io-cache" % fstype
-        _options_log = "logfile=%s/%s.log" % (LOGDIR, self.volume_name)
-        _options_ext = "loglevel=ERROR,attr_timeout=0"
-        booster_fstab_fd.write ("%s %s,%s,%s\n" %
-                                (_fstab,
-                                 _options,
-                                 _options_log,
-                                 _options_ext))
+        if self.need_nfs:
+            _fstab = "%s/%s.vol  %s" % (CONFDIR, self.volume_name, str("/nfs/" + self.volume_name))
+            _options = "%s" % fstype
+            _options_log = "logfile=%s/%s-nfs.log" % (LOGDIR, self.volume_name)
+            _options_ext = "loglevel=ERROR,attr_timeout=0"
+            booster_fstab_fd.write ("%s %s %s,%s\n" %
+                                    (_fstab,
+                                     _options,
+                                     _options_log,
+                                     _options_ext))
+
+        if self.need_cifs:
+            _fstab = "%s/%s.vol  %s" % (CONFDIR, self.volume_name, str("/cifs/" + self.volume_name))
+            _options = "%s" % fstype
+            _options_log = "logfile=%s/%s-cifs.log" % (LOGDIR, self.volume_name)
+            _options_ext = "loglevel=ERROR,attr_timeout=0"
+            booster_fstab_fd.write ("%s %s %s,%s\n" %
+                                    (_fstab,
+                                     _options,
+                                     _options_log,
+                                     _options_ext))
 
         return
 
     def configure_nfs_booster (self):
-        if self.volume_name is None or self.export is None:
-            sys.exit(1)
 
         nfs_exports_fd = file (GLUSTERFS_UNFS3_EXPORTS, "a")
         nfs_exports_fd.write ("%s  0.0.0.0/0(rw,no_root_squash)\n" %
-                              self.export)
+                              str("/nfs/" + self.volume_name))
         return
 
     def configure_cifs_booster (self):
 
-        if self.volume_name is None or self.export is None:
-            sys.exit(1)
-
         cifs_config_fd = file (GLUSTERFS_CIFS_CONFIG, "a")
         cifs_config_fd.write ("[%s]\n" % self.volume_name)
         cifs_config_fd.write ("comment = %s volume served by Gluster\n" %
                               self.volume_name)
-        cifs_config_fd.write ("path = %s\n" % self.export)
-        cifs_config_fd.write ("guest ok = yes\n")
+        cifs_config_fd.write ("path = %s\n" % str("/cifs/" + self.volume_name))
+        if self.enable_guest:
+            cifs_config_fd.write ("guest ok = yes\n")
         cifs_config_fd.write ("public = yes\n")
         cifs_config_fd.write ("writable = yes\n")
-        cifs_config_fd.write ("admin users = gluster\n")
+        cifs_config_fd.write ("users = %s\n" % self.username)
         cifs_config_fd.close()
         return
+
+    def configure_booster (self):
+
+        self.configure_booster_fstab()
+        if self.need_nfs:
+            self.configure_nfs_booster()
+            print "Generating booster configuration for NFS reexport"
+        if self.need_cifs:
+            self.configure_cifs_booster()
+            print "Generating booster configuration for CIFS reexport"
+
+        return
diff --git a/extras/volgen/CreateVolfile.py b/extras/volgen/CreateVolfile.py
index 1d22e8fb164..b51be766aae 100644
--- a/extras/volgen/CreateVolfile.py
+++ b/extras/volgen/CreateVolfile.py
@@ -7,24 +7,28 @@ cache_size = "1GB"
 
 class CreateVolfile:
 
-    def __init__ (self, servers, server, volume_name,
-                  transport, transports, port, auth_param,
-                  ib_port, confdir, args):
-        self.hosts = servers
+    def __init__ (self, server_dict, server, transport,
+                  transports, options, server_array):
+
+        self.host_dict = server_dict
         self.host = server
-        self.volume_name = volume_name
+        self.volume_name = options.volume_name
         self.transport = transport
         self.transports = transports
-        self.gfs_port = port
-        self.gfs_ib_port = port + 1
-        self.auth_parameters = auth_param
-        self.ib_devport = ib_port
-        self.num_servers = len (self.hosts.keys())
-        self.conf_dir = confdir
-        self.arguments = args
+        self.gfs_port = options.port
+        self.gfs_ib_port = options.port + 1
+        self.auth_parameters = options.auth_param
+        self.raid_type = options.raid_type
+        self.ib_devport = options.ib_dev
+        self.num_servers = len (self.host_dict.keys())
+        self.conf_dir = options.conf_dir
+        self.host_array = server_array
+        self.unused = options.unused
+        self.debug = options.debug
 
-    def create_mount_volfile (self, raid_type):
+    def create_mount_volfile (self):
 
+        raid_type = self.raid_type
 
         if self.conf_dir:
             mount_fd = file ("%s/%s-%s.vol" % (self.conf_dir,
@@ -36,9 +40,6 @@ class CreateVolfile:
 
         print "Generating client volfiles.. for transport '%s'" % (self.transport)
 
-        num_stripe = 4
-        num_replica = 2
-
 
         cmdline = string.join (sys.argv, ' ')
         mount_fd.write ("## file auto generated by %s (mount.vol)\n" %
@@ -52,9 +53,9 @@ class CreateVolfile:
 
         mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport)
         subvolumes = []
-        for host in self.hosts.keys():
+        for host in self.host_dict.keys():
             i = 1
-            for exports in self.hosts[host]:
+            for exports in self.host_dict[host]:
                 mount_fd.write ("volume %s-%s\n" % (host,i))
                 mount_fd.write ("    type protocol/client\n")
                 mount_fd.write ("    option transport-type %s\n" %
@@ -88,7 +89,7 @@ class CreateVolfile:
                 i += 1
 
         exportlist = {}
-        for entry in self.arguments:
+        for entry in self.host_array:
             node = entry.split(':')[0]
             if not exportlist.has_key(node):
                 exportlist[node] = 1
@@ -96,17 +97,6 @@ class CreateVolfile:
                 exportlist[node] += 1
             subvolumes.append(str(node) + '-' + str(exportlist[node]))
 
-
-        if raid_type == 1:
-            if (len(subvolumes) % num_replica) != 0:
-                print "raid type (%d) and number of volumes (%d) invalid" % (raid_type, len(subvolumes))
-                sys.exit (1)
-
-        if raid_type == 0:
-            if (len(subvolumes) % num_stripe) != 0:
-                print "raid type (%d) and number of volumes (%d) invalid" % (raid_type, len(subvolumes))
-                sys.exit (1)
-
         # Stripe section.. if given
         if raid_type is 0:
             max_stripe_idx = len (subvolumes) / num_stripe
@@ -115,6 +105,10 @@ class CreateVolfile:
             while index < max_stripe_idx:
                 mount_fd.write ("volume stripe-%d\n" % index)
                 mount_fd.write ("    type cluster/stripe\n")
+                if self.unused:
+                    mount_fd.write ("#    option block-size 128k\n")
+                    mount_fd.write ("#    option use-xattr no\n")
+
                 mount_fd.write ("    subvolumes %s %s %s %s\n" %
                                 (subvolumes[stripe_idx],
                                  subvolumes[stripe_idx+1],
@@ -156,6 +150,11 @@ class CreateVolfile:
         if len (subvolumes) > 1:
             mount_fd.write ("volume distribute\n")
             mount_fd.write ("    type cluster/distribute\n")
+            if self.unused:
+                mount_fd.write("# option unhashed-sticky-bit yes # Used for migrating data while adding new nodes\n")
+                mount_fd.write("# option min-free-disk 5% # Minimum free disk available on the volume\n")
+
+
             mount_fd.write ("    subvolumes %s\n" %
                                  string.join (subvolumes,' '))
             mount_fd.write ("end-volume\n\n")
@@ -164,13 +163,42 @@ class CreateVolfile:
         mount_fd.write ("volume writebehind\n")
         mount_fd.write ("    type performance/write-behind\n")
         mount_fd.write ("    option cache-size 4MB\n")
+        if self.unused:
+            mount_fd.write ("#   option enable-trickling-writes yes # Flush final write calls when network is free\n")
+            mount_fd.write ("#   option enable-O_SYNC yes # Enable O_SYNC for write-behind\n")
+            mount_fd.write ("#   option disable-for-first-nbytes 1 # Disable first nbytes with very small initial writes\n")
+
         mount_fd.write ("    subvolumes %s\n" % subvolumes[0])
         mount_fd.write ("end-volume\n\n")
 
-        mount_fd.write ("volume io-cache\n")
+        mount_fd.write ("volume readahead\n")
+        mount_fd.write ("    type performance/read-ahead\n")
+        mount_fd.write ("    option page-count 4\n")
+
+        if self.unused:
+            mount_fd.write ("#   option force-atime-update yes # force updating atimes, default off\n")
+        mount_fd.write ("    subvolumes writebehind\n")
+        mount_fd.write ("end-volume\n\n")
+
+        mount_fd.write ("volume iocache\n")
         mount_fd.write ("    type performance/io-cache\n")
         mount_fd.write ("    option cache-size %s\n" % cache_size)
-        mount_fd.write ("    subvolumes writebehind\n")
+        mount_fd.write ("    option cache-timeout 1\n")
+        if self.unused:
+            mount_fd.write ("#   option priority *.html:1,abc*:2 # Priority list for iocaching files\n")
+        mount_fd.write ("    subvolumes readahead\n")
+        mount_fd.write ("end-volume\n\n")
+
+        mount_fd.write ("volume quickread\n")
+        mount_fd.write ("    type performance/quick-read\n")
+        mount_fd.write ("    option cache-timeout 1\n")
+        mount_fd.write ("    option max-file-size 64kB\n")
+        mount_fd.write ("    subvolumes iocache\n")
+        mount_fd.write ("end-volume\n\n")
+
+        mount_fd.write ("volume statprefetch\n")
+        mount_fd.write ("    type performance/stat-prefetch\n")
+        mount_fd.write ("    subvolumes quickread\n")
         mount_fd.write ("end-volume\n\n")
 
         return
@@ -195,20 +223,35 @@ class CreateVolfile:
         exp_fd.write ("# $ %s\n\n" % cmdline)
         total_bricks = []
         i=1
-        for export in self.hosts[self.host]:
+        for export in self.host_dict[self.host]:
             exp_fd.write ("volume posix%d\n" % i)
             exp_fd.write ("  type storage/posix\n")
+            if self.unused:
+                exp_fd.write("# option o-direct enable # (default: disable) boolean type only\n")
+                exp_fd.write("# option export-statfs-size no # (default: yes) boolean type only\n")
+                exp_fd.write("# option mandate-attribute off # (default: on) boolean type only\n")
+                exp_fd.write("# option span-devices 8 # (default: 0) integer value\n")
+                exp_fd.write("# option background-unlink yes # (default: no) boolean type\n")
+
             exp_fd.write ("  option directory %s\n" % export)
             exp_fd.write ("end-volume\n\n")
 
             exp_fd.write ("volume locks%d\n" % i)
             exp_fd.write ("    type features/locks\n")
+            if self.unused:
+                exp_fd.write ("#   option mandatory on # Default off, used in specific applications\n")
+
             exp_fd.write ("    subvolumes posix%d\n" % i)
             exp_fd.write ("end-volume\n\n")
 
             exp_fd.write ("volume brick%d\n" % i)
             exp_fd.write ("    type performance/io-threads\n")
             exp_fd.write ("    option thread-count 8\n")
+            if self.unused:
+                exp_fd.write ("#    option autoscaling yes # Heuristic for autoscaling threads on demand\n")
+                exp_fd.write ("#    option min-threads 2 # min count for thread pool\n")
+                exp_fd.write ("#    option max-threads 64 # max count for thread pool\n")
+
             exp_fd.write ("    subvolumes locks%d\n" % i)
             exp_fd.write ("end-volume\n\n")
 
diff --git a/extras/volgen/glusterfs-volgen b/extras/volgen/glusterfs-volgen
index 1151053acf6..3781f6d0bfb 100755
--- a/extras/volgen/glusterfs-volgen
+++ b/extras/volgen/glusterfs-volgen
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 
 import getopt, sys, os, string
-import subprocess
+from optparse import OptionParser,OptionGroup,make_option
 
 if not "/usr/share/glusterfs" in sys.path:
     sys.path.append("/usr/share/glusterfs")
@@ -9,176 +9,175 @@ if not "/usr/share/glusterfs" in sys.path:
 from CreateVolfile import *
 from CreateBooster import *
 
-def print_usage (name):
-    spaces = ' ' * (len(name) + 1)
-    print name, "--name <volume-name>"
-    print "%s[--raid 0|1]" % spaces
-    print "%s[--transport tcp,ib-verbs]" % spaces
-    print "%s[--port <port>]" % spaces
-    print "%s[--ibdev <lid>]" % spaces
-    print "%s[--auth <ip-range>]" % spaces
-    print "%s[--conf-dir <confdir>]" % spaces
-    print "%s[--usage]" % spaces
-    print "%s[--nfs]" % spaces
-    print "%s[--cifs]" % spaces
-    print "%s[--version]" % spaces
-    print "%shost1:<export> host2:<export> ... hostN:<exportNN>" % spaces
-    return
-
-def print_version (version):
-    print "glusterfs-volgen %s: A tool to generate volume files for GlusterFS." % version
-    print "Copyright (C) 2009 Gluster, Inc. <http://www.gluster.com>"
-    print """License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
-This is free software: you are free to change and redistribute it.
-There is NO WARRANTY, to the extent permitted by law."""
-
-
-def main ():
-
-    needs_upgrade = None
-    needs_nfs = None
-    needs_cifs = None
-    version_num ="3.0"
-    _volume_name = None
-    _server_chain = {}
-    _server = None
-    _transports = ['tcp']
-    _transport = None
-    _port = 6996
-    _auth_param = "*"
-    _ib_dev = 1
-    _raid_type = None
-    node = None
-    _conf_dir = None
-
-    try:
-        (opt, args) = getopt.getopt (sys.argv[1:], "r:t:p:n:a:i:ch",
-                                     ["raid=",
-                                      "transport=",
-                                      "port=",
-                                      "name=",
-                                      "auth=",
-                                      "ibdev=",
-                                      "conf-dir=",
-                                      "nfs",
-                                      "cifs",
-                                      "usage",
-                                      "version",
-                                      "help"])
-
-    except getopt.GetoptError, (msg, opt):
-        print msg
-        sys.exit (1)
-
-    for (o, val) in opt:
-        if o == '--usage' or o == '--help':
-            print_usage (sys.argv[0])
-            sys.exit (0)
-
-        if o == '-n' or o == '--name':
-            _volume_name = val
-
-        if o == '--nfs':
-            needs_nfs = 1
-
-        if o == '--cifs':
-            needs_cifs = 1
-
-        if o == '-t' or o == '--transport':
-            if not val:
-                print "--transport: option " + val + \
-                    " is not valid transport type"
-                sys.exit (1)
-            _transports = val.split(',')
-
-        if o == '-a' or o == '--auth':
-            _auth_param = val
-
-        if o == '-p' or o == '--port':
-            _port = int(val)
-
-        if o == '-r' or o == '--raid':
-            if (val != "1" and val != "0"):
-                print "--raid: option " + val + " is not valid raid type"
-                sys.exit (1)
-            _raid_type = int (val)
-
-        if o == '-i' or o == '--ibdev':
-            _ib_dev = int(val)
-
-        if o == '-v' or o == '--version':
-            print_version (version_num)
-            sys.exit (0)
-
-        if o == '-c' or o == '--conf-dir':
-            _conf_dir = val
-
-    if _volume_name is None:
-        print "Volume name is mandatory, please provide volume name.. exiting"
-        print_usage(sys.argv[0])
-        sys.exit(1)
-
+def check_duplicate_entry(args):
+    """Check duplicate entries in incoming arguments"""
     _tmp = []
     for server in args:
         if server not in _tmp:
             _tmp.append (server)
         else:
             print "Duplicate arguments detected (%s)" % server
-            sys.exit(1)
+            raise ValueError
 
-        node = server.split(':')[0]
-        _server_chain[node] = []
+    return
 
-    for server in args:
-        node = server.split(':')[0]
-        if server.split(':')[1] not in _server_chain [node]:
-            if server.split(':')[1][0] != '/':
-                print "Absolute export path required for %s" % server
-                sys.exit(1)
-            _server_chain [node].append (server.split(':')[1])
+def args2dict(args):
+
+    keyvalue = {}
+    for arg in args:
+        first = arg.split(':')[0]
+        keyvalue[first] = []
+
+    for arg in args:
+        first = arg.split(':')[0]
+        if arg.split(':')[1] not in keyvalue[first]:
+            if arg.split(':')[1][0] != '/':
+                print "Absolute export path required for %s" % arg
+                raise ValueError
+            keyvalue[first].append (arg.split(':')[1])
+
+    return keyvalue
+
+def args2array(args):
+
+    array = []
+    array = args
+
+    return array
+
+def generate_volume_files ():
+
+    num_stripe = 4
+    num_replica = 2
+
+    usage_str = "%prog: -n <VOLUMENAME> -t <TRANSPORT> -p <NUMBER> -a <AUTH> -r <TYPE>"
+    version_str = "%prog 3.0"
+    desc_str = "A tool to generate volume files for GlusterFS."
+
+    parse = OptionParser(usage=usage_str, version=version_str, description=desc_str)
+
+    # Basic option list
+    group = OptionGroup(parse, "Basic Options")
+    group.add_option("-n", "--name", dest="volume_name",
+                help="<volume-name>")
+    group.add_option("-t", "--transport", dest="transport_type",
+                default="tcp", help="tcp,ib-verbs default: tcp")
+    group.add_option("-p", "--port", type="int",
+                dest="port", default=6996,
+                help="<port> number")
+    group.add_option("-a", "--auth", dest="auth_param", default="*",
+                help="comma seperated ip range")
+    group.add_option("-r", "--raid", type="int", dest="raid_type",
+                help="0|1")
+    group.add_option("--nfs", action="store_true", dest="need_nfs",
+                default=False, help="booster nfs reexport")
+    group.add_option("--cifs", action="store_true", dest="need_cifs",
+                default=False, help="booster cifs reexport"),
+    parse.add_option_group(group)
+
+    # CIFS option list
+    group = OptionGroup(parse, "CIFS Options")
+    group.add_option("--username", dest="cifs_username",
+                     default="gluster", help="<cifs_username>")
+    group.add_option("--guest", action="store_true",
+                     dest="enable_guest", default=False,
+                     help="enable guest access")
+    parse.add_option_group(group)
+    # NFS option list
+
+    # Advanced option list
+    group = OptionGroup(parse, "Advanced Options")
+    group.add_option("--ibdev", type="int", dest="ib_dev",
+                     default=1, help="Infiniband device number <N>")
+    group.add_option("-c", "--conf-dir", dest="conf_dir",
+                     help="output directory for volume files"),
+    parse.add_option_group(group)
+
+    group = OptionGroup(parse, "Extra Options")
+    group.add_option("--unused", action="store_true",
+                     dest="unused", default=False,
+                     help="enable unused options")
+    group.add_option("--debug", action="store_true",
+                     dest="debug", default=False,
+                     help="add all debug modules to volumes")
+
+    parse.add_option_group(group)
+
+    (options, args) = parse.parse_args()
+
+    if options.volume_name is None:
+        print "Error: volume name is mandatory, please provide volume name"
+        raise ValueError
+
+    if options.transport_type:
+        transports = options.transport_type.split(',')
+        for transport in transports:
+            if (transport != "tcp" and transport != "ib-verbs"):
+                print "Error: --transport: option " + transport + \
+                    " is not valid transport type"
+                raise ValueError
+
+    if options.raid_type:
+        if (options.raid_type != 1 and options.raid_type != 0):
+            print "Error: --raid: option " + str(options.raid_type) + " is not valid raid type"
+            raise ValueError
+
+
+    check_duplicate_entry(args)
 
-    num_servers = len (_server_chain.keys())
+    server_dict = {}
 
-    if num_servers is 0:
-        print "no servers provided, exiting"
-        print_usage(sys.argv[0])
-        sys.exit (1)
+    server_dict = args2dict(args)
 
-    print num_servers
+    server_array = args2array(args)
 
+    if len (server_dict.keys()) == 0:
+        print "Error: no servers provided, please provide atleast one server"
+        raise ValueError
 
-    for _server in _server_chain.keys():
-        create_exp = CreateVolfile (_server_chain, _server,
-                                    _volume_name, None,
-                                    _transports, _port,
-                                    _auth_param, _ib_dev,
-                                    _conf_dir, None)
+    if options.raid_type == 1:
+        if (len(server_array) % num_replica) != 0:
+            print "raid type (%d) and number of volumes (%d) invalid" % (options.raid_type,
+                                                                         len(server_array))
+            raise ValueError
+
+    if options.raid_type == 0:
+        if (len(server_array) % num_stripe) != 0:
+            print "raid type (%d) and number of volumes (%d) invalid" % (options.raid_type,
+                                                                         len(server_array))
+            raise ValueError
+
+    for server in server_dict.keys():
+        create_exp = CreateVolfile (server_dict, server,
+                                    None, transports,
+                                    options, None)
         try:
             create_exp.create_export_volfile ()
         except IOError, (errno, strerror):
-            print "Got %s creating server volfiles for %s" % (strerror, _server)
-
-    for _transport in _transports:
-        create_mnt = CreateVolfile (_server_chain, None,
-                                    _volume_name, _transport,
-                                    _transports, _port,
-                                    _auth_param, _ib_dev,
-                                    _conf_dir, args)
+            print "Got %s creating server volfiles for %s" % (strerror, server)
+
+    for transport in transports:
+        create_mnt = CreateVolfile (server_dict, None,
+                                    transport, transports,
+                                    options, server_array)
         try:
-            create_mnt.create_mount_volfile (_raid_type)
+            create_mnt.create_mount_volfile ()
         except IOError, (errno, strerror):
-            print "Got %s creating client volfiles for transport '%s'" % (strerror, _transport)
-
-
-    for _server in _server_chain.keys():
-        for export_dir in _server_chain[_server]:
-            commonobj = CreateBooster (_volume_name, export_dir)
-            if needs_nfs is 1 or needs_cifs is 1:
-                commonobj.configure_booster_fstab ()
-            if needs_nfs is 1:
-                commonobj.configure_nfs_booster ()
-            if needs_cifs is 1:
-                commonobj.configure_cifs_booster ()
-    return
+            print "Got %s creating client volfiles for transport '%s'" % (strerror, transport)
+
+
+    create_booster = CreateBooster (options)
+    try:
+        create_booster.configure_booster ()
+    except IOError, (errno, strerror):
+        print "Got %s creating booster configuration '%s'" % (strerror, export_dir)
+
+def main ():
+
+    try:
+        generate_volume_files()
+    except ValueError:
+        sys.exit(1)
+
 
-main ()
+main()