new option to volgen '--portmapper-mode'

* without which, it generates volume files in old fashioned way.
* with portmapper mode, no port option is taken, and there will be
  one process per one export.
* corresponding changes in 'mgmt/glusterd' is made

Signed-off-by: Amar Tumballi <amar@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 1341 (glusterfs-volgen should behave differently for flag '--internal')
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1341

4 files changed, 282 insertions, 39 deletions

diff --git a/extras/volgen/CreateVolfile.py b/extras/volgen/CreateVolfile.py
index df38a9946..fdcf60566 100644
--- a/extras/volgen/CreateVolfile.py
+++ b/extras/volgen/CreateVolfile.py
@@ -45,10 +45,10 @@ class CreateVolfile:
             self.gfs_port    = options.port
             self.gfs_ib_port = options.port + 1
         else:
-            self.gfs_port    = None
-            self.gfs_ib_port = None
+            self.gfs_port    = 6969
+            self.gfs_ib_port = 6970
 
-    def create_mount_volfile (self):
+    def create_pmap_mount_volfile (self):
 
         raid_type = self.raid_type
 
@@ -72,7 +72,7 @@ class CreateVolfile:
         for host in self.host_dict.keys():
             i = 1
             for export in self.host_dict[host]:
-                mount_fd.write ("volume %s-%s\n" % (host,i))
+                mount_fd.write ("volume %s-%s\n" % (host,export))
                 mount_fd.write ("    type protocol/client\n")
                 mount_fd.write ("    option transport-type %s\n" %
                                 self.transport)
@@ -80,26 +80,13 @@ class CreateVolfile:
                 if self.transport == 'ib-verbs':
                     mount_fd.write ("    option transport.ib-verbs.port %d\n" %
                                     self.ib_devport)
-                    if self.gfs_ib_port:
-                        mount_fd.write ("    option remote-port %d\n" %
-                                        self.gfs_ib_port)
                 if self.transport == 'tcp':
                     mount_fd.write ("    option transport.socket.nodelay on\n")
-                    if self.gfs_port:
-                        mount_fd.write ("    option remote-port %d\n" % self.gfs_port)
 
                 mount_fd.write ("    option remote-subvolume %s\n" % export)
                 mount_fd.write ("end-volume\n\n")
                 i += 1
-
-        exportlist = {}
-        for entry in self.host_array:
-            node = entry.split(':')[0]
-            if not exportlist.has_key(node):
-                exportlist[node] = 1
-            else:
-                exportlist[node] += 1
-            subvolumes.append(str(node) + '-' + str(exportlist[node]))
+                subvolumes.append(str(host) + '-' + str(export))
 
         # Stripe section.. if given
         if raid_type is 0:
@@ -250,7 +237,7 @@ class CreateVolfile:
 
         return
 
-    def create_export_volfile (self):
+    def create_pmap_export_volfile (self):
 
         cmdline = string.join (sys.argv, ' ')
         i = 0
@@ -297,7 +284,6 @@ class CreateVolfile:
                     exp_fd.write ("    subvolumes posix\n")
                 exp_fd.write ("end-volume\n\n")
 
-
             exp_fd.write ("volume locks\n")
             exp_fd.write ("    type features/locks\n")
             exp_fd.write ("#   option mandatory on # Default off, used in specific applications\n")
@@ -313,9 +299,6 @@ class CreateVolfile:
             exp_fd.write ("    type protocol/client\n")
             if self.transport:
                 exp_fd.write ("    option transport-type %s\n" % self.transport)
-
-            if self.gfs_port:
-                exp_fd.write ("    option transport.remote-port %d\n" % self.gfs_port)
             exp_fd.write ("    option ping-timeout 42\n")
             exp_fd.write ("end-volume\n\n")
 
@@ -330,28 +313,263 @@ class CreateVolfile:
             exp_fd.write ("#   option autoscaling yes # Heuristic for autoscaling threads on demand\n")
             exp_fd.write ("#   option min-threads 2 # min count for thread pool\n")
             exp_fd.write ("#   option max-threads 64 # max count for thread pool\n")
-
             exp_fd.write ("    subvolumes pump\n")
             exp_fd.write ("end-volume\n\n")
 
             for transport in self.transports:
-                exp_fd.write ("volume server-%s\n" % transport)
+                exp_fd.write ("volume server-%s\n" % export)
                 exp_fd.write ("    type protocol/server\n")
                 exp_fd.write ("    option transport-type %s\n" % transport)
                 exp_fd.write ("    option auth.addr.%s.allow %s\n" %
                                   (export, self.auth_parameters))
-
                 if transport == 'ib-verbs':
-                    if self.gfs_ib_port:
-                        exp_fd.write ("    option listen-port %d\n" % self.gfs_ib_port)
                     exp_fd.write ("    option transport.ib-verbs.port %d\n" %
                                   self.ib_devport)
                 if transport == 'tcp':
-                    if self.gfs_port:
-                        exp_fd.write ("    option listen-port %d\n" % self.gfs_port)
                     exp_fd.write ("    option transport.socket.nodelay on\n")
-
                 exp_fd.write ("    subvolumes %s\n" % export)
                 exp_fd.write ("end-volume\n\n")
 
         return
+
+
+    def create_mount_volfile (self):
+
+        raid_type = self.raid_type
+
+        mount_volfile = "%s/%s-%s.vol" % (self.conf_dir, str(self.volume_name), str(self.transport))
+        mount_fd = file ("%s" % (mount_volfile), "w")
+
+        print "Generating client volfiles.. '%s'" % mount_volfile
+
+        cmdline = string.join (sys.argv, ' ')
+
+        mount_fd.write ("## file auto generated by %s\n" % sys.argv[0])
+        mount_fd.write ("# Cmd line:\n")
+        mount_fd.write ("# $ %s\n\n" % cmdline)
+
+        if raid_type is not None:
+            # Used for later usage
+            mount_fd.write ("# RAID %d\n" % raid_type)
+
+        mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport)
+        subvolumes = []
+        for host in self.host_dict.keys():
+            i = 1
+            for exports in self.host_dict[host]:
+                mount_fd.write ("volume %s-%s\n" % (host,exports))
+                mount_fd.write ("    type protocol/client\n")
+                mount_fd.write ("    option transport-type %s\n" %
+                                self.transport)
+                mount_fd.write ("    option remote-host %s\n" % host)
+                if self.transport == 'ib-verbs':
+                    mount_fd.write ("    option transport.ib-verbs.port %d\n" %
+                                    self.ib_devport)
+                    mount_fd.write ("    option remote-port %d\n" %
+                                    self.gfs_ib_port)
+                if self.transport == 'tcp':
+                    mount_fd.write ("    option transport.socket.nodelay on\n")
+                    mount_fd.write ("    option remote-port %d\n" %
+                                    self.gfs_port)
+
+                mount_fd.write ("    option remote-subvolume %s\n" % exports)
+                mount_fd.write ("end-volume\n\n")
+                subvolumes.append(str(host) + '-' + str(exports))
+
+        # Stripe section.. if given
+        if raid_type is 0:
+            max_stripe_idx = len (subvolumes) / self.num_stripe
+            stripe_idx = 0
+            index = 0
+            while index < max_stripe_idx:
+                mount_fd.write ("volume stripe-%d\n" % index)
+                mount_fd.write ("    type cluster/stripe\n")
+                mount_fd.write ("#    option block-size 128k\n")
+                mount_fd.write ("#    option use-xattr no\n")
+                mount_fd.write ("    subvolumes %s" % subvolumes[stripe_idx])
+                sub_idx = 1
+                while sub_idx < self.num_stripe:
+                    mount_fd.write (" %s" % subvolumes[stripe_idx+sub_idx])
+                    sub_idx += 1
+                mount_fd.write ("\nend-volume\n\n")
+                stripe_idx += self.num_stripe
+                index +=1
+
+        # Replicate section
+        if raid_type is 1:
+            max_mirror_idx = len (subvolumes) / self.num_replica
+            mirror_idx = 0
+            index = 0
+            while index < max_mirror_idx:
+                mount_fd.write ("volume mirror-%d\n" % index)
+                mount_fd.write ("    type cluster/replicate\n")
+                mount_fd.write ("    subvolumes %s" % subvolumes[mirror_idx])
+                sub_idx = 1
+                while sub_idx < self.num_replica:
+                    mount_fd.write (" %s" % subvolumes[mirror_idx + sub_idx])
+                    sub_idx += 1
+                mount_fd.write ("\nend-volume\n\n")
+                mirror_idx += self.num_replica
+                index += 1
+
+        # Distribute section
+        if raid_type is 0:
+            subvolumes = []
+            flag = 0
+            while flag < index:
+                subvolumes.append ("stripe-%d" % flag)
+                flag += 1
+
+        if raid_type is 1:
+            subvolumes = []
+            flag = 0
+            while flag < index:
+                subvolumes.append ("mirror-%d" % flag)
+                flag += 1
+
+        if len (subvolumes) > 1:
+            mount_fd.write ("volume distribute\n")
+            mount_fd.write ("    type cluster/distribute\n")
+            mount_fd.write ("#   option unhashed-sticky-bit yes"
+                           "  # Used for migrating data while adding new nodes\n")
+            mount_fd.write ("#   option min-free-disk 5%"
+                           "  # Minimum free disk available on the volume\n")
+            mount_fd.write ("    subvolumes %s\n" %
+                                 string.join (subvolumes,' '))
+            mount_fd.write ("end-volume\n\n")
+            subvolumes[0] = "distribute"
+
+
+        if self.nfs:
+            mount_fd.write ("volume nfsxlator\n")
+            mount_fd.write ("    type nfs/server\n")
+            mount_fd.write ("    subvolumes %s\n" % subvolumes[0])
+            mount_fd.write ("#   option rpc-auth.auth-unix         off  #Enabled by default\n")
+            mount_fd.write ("#   option rpc-auth.auth-null         off  #Enabled by default\n")
+            mount_fd.write ("#   By default all addresses are rejected until allowed.\n")
+            mount_fd.write ("#   option rpc-auth.addr.reject       127.*\n")
+            mount_fd.write ("#   option rpc-auth.addr.allow        localhost\n")
+            mount_fd.write ("    option rpc-auth.addr.%s.allow %s\n" % (subvolumes[0], self.auth_parameters))
+            mount_fd.write ("#   By default insecure ports are not allowed.\n")
+            mount_fd.write ("#   option rpc-auth.ports.insecure    on\n")
+            mount_fd.write ("#   option rpc-auth.ports.<volume>.insecure  on\n")
+            mount_fd.write ("#   By default all access is read-write.\n")
+            mount_fd.write ("#   option nfs3.<volume>.volume-access       read-only\n")
+            mount_fd.write ("#   option nfs3.<volume>.volume-access       read-only\n")
+            mount_fd.write ("#   option nfs3.read-size             128Kb\n")
+            mount_fd.write ("#   option nfs3.write-size            32Kb\n")
+            mount_fd.write ("#   option nfs3.readdir-size          64Kb\n")
+            mount_fd.write ("#   option nfs3.<volume>.read-size    64Kb\n")
+            mount_fd.write ("#   option nfs3.<volume>.write-size   64Kb\n")
+            mount_fd.write ("#   option nfs3.posix1.readdir-size   128Kb\n")
+            mount_fd.write ("end-volume\n\n")
+            return
+
+        if self.volume_size_client:
+            mount_fd.write ("volume quota\n")
+            mount_fd.write ("    type features/quota\n")
+            mount_fd.write ("    option disk-usage-limit %s\n" % self.volume_size_client)
+            mount_fd.write ("#   option minimum-free-disk-limit 10GB\n"
+                            "#   minimum free disk value (default) 0\n")
+            mount_fd.write ("#   option refresh-interval 10\n")
+            mount_fd.write ("    subvolumes %s\n" % subvolumes[0])
+            mount_fd.write ("end-volume\n\n")
+
+        if self.enable_safe_mode:
+            return
+
+        self.performance_mode (subvolumes[0], mount_fd)
+
+        return
+
+    def create_export_volfile (self):
+
+        cmdline = string.join (sys.argv, ' ')
+        export_volfile = "%s/%s-export.vol" % (self.conf_dir, str(self.host + '-' + self.volume_name))
+        exp_fd = file ("%s" % (export_volfile),"w")
+
+        print "Generating server volfiles.. for server %s as '%s'" % (self.host,
+                                                                      export_volfile)
+
+        exp_fd.write ("## file auto generated by %s\n" %
+                      sys.argv[0])
+        exp_fd.write ("# Cmd line:\n")
+        exp_fd.write ("# $ %s\n\n" % cmdline)
+        total_bricks = []
+        i=1
+        for export in self.host_dict[self.host]:
+            exp_fd.write ("volume posix%d\n" % i)
+            exp_fd.write ("    type storage/posix\n")
+            exp_fd.write ("#   option o-direct enable # (default: disable) boolean type only\n")
+            exp_fd.write ("#   option export-statfs-size no # (default: yes) boolean type only\n")
+            exp_fd.write ("#   option mandate-attribute off # (default: on) boolean type only\n")
+            exp_fd.write ("#   option span-devices 8 # (default: 0) integer value\n")
+            exp_fd.write ("#   option background-unlink yes # (default: no) boolean type\n")
+
+            exp_fd.write ("    option directory %s\n" % export)
+            exp_fd.write ("end-volume\n\n")
+
+            if self.nfs:
+                exp_fd.write ("volume posix-ac%d\n" % i)
+                exp_fd.write ("    type features/access-control\n")
+                exp_fd.write ("    subvolumes posix%d\n" % i)
+                exp_fd.write ("end-volume\n\n")
+
+            if self.volume_size_server:
+                exp_fd.write ("volume quota%d\n" % i)
+                exp_fd.write ("    type features/quota\n")
+                exp_fd.write ("    option disk-usage-limit %s\n" % self.volume_size_server)
+                exp_fd.write ("#   option minimum-free-disk-limit 10GB"
+                              "  # minimum free disk value (default) 0\n")
+                exp_fd.write ("#   option refresh-interval 10\n")
+                if self.nfs:
+                    exp_fd.write ("    subvolumes posix-ac%d\n" % i)
+                else:
+                    exp_fd.write ("    subvolumes posix%d\n" % i)
+                exp_fd.write ("end-volume\n\n")
+
+
+            exp_fd.write ("volume locks%d\n" % i)
+            exp_fd.write ("    type features/locks\n")
+            exp_fd.write ("#   option mandatory on # Default off, used in specific applications\n")
+            if self.volume_size_server:
+                exp_fd.write ("    subvolumes quota%d\n" % i)
+            else:
+                exp_fd.write ("    subvolumes posix%d\n" % i)
+            exp_fd.write ("end-volume\n\n")
+
+            exp_fd.write ("volume %s\n" % export)
+            exp_fd.write ("    type performance/io-threads\n")
+            exp_fd.write ("    option thread-count 8\n")
+            exp_fd.write ("#   option autoscaling yes # Heuristic for autoscaling threads on demand\n")
+            exp_fd.write ("#   option min-threads 2 # min count for thread pool\n")
+            exp_fd.write ("#   option max-threads 64 # max count for thread pool\n")
+
+            exp_fd.write ("    subvolumes locks%d\n" % i)
+            exp_fd.write ("end-volume\n\n")
+
+            total_bricks.append("%s" % export)
+            i += 1
+
+        for transport in self.transports:
+            exp_fd.write ("volume server-%s\n" % transport)
+            exp_fd.write ("    type protocol/server\n")
+            exp_fd.write ("    option transport-type %s\n" % transport)
+            for brick in total_bricks:
+                exp_fd.write ("    option auth.addr.%s.allow %s\n" %
+                              (brick, self.auth_parameters))
+
+            if transport == 'ib-verbs':
+                exp_fd.write ("    option transport.ib-verbs.listen-port %d\n" % self.gfs_ib_port)
+                exp_fd.write ("    option transport.ib-verbs.port %d\n" %
+                              self.ib_devport)
+            if transport == 'tcp':
+                exp_fd.write ("    option transport.socket.listen-port %d\n" % self.gfs_port)
+                exp_fd.write ("    option transport.socket.nodelay on\n")
+
+            exp_fd.write ("    subvolumes %s\n" %
+                          string.join(total_bricks, ' '))
+            exp_fd.write ("end-volume\n\n")
+
+        return
+
diff --git a/extras/volgen/glusterfs-volgen.in b/extras/volgen/glusterfs-volgen.in
index 3ab5aa1cd..31a811228 100755
--- a/extras/volgen/glusterfs-volgen.in
+++ b/extras/volgen/glusterfs-volgen.in
@@ -72,6 +72,8 @@ def generate_volume_files ():
                      help="number of file replica needed")
     group.add_option("--num-stripe", type="int", dest="num_stripe",
                      help="number of stripe count needed")
+    group.add_option("--portmapper-mode", action="store_true", dest="portmapper",
+                     default=False, help="generate volume files for glusterfs portmapper friendly way")
 
     parse.add_option_group(group)
 
@@ -164,6 +166,27 @@ def generate_volume_files ():
                                                                          len(server_array))
             raise ValueError
 
+    if options.portmapper is True:
+        for server in server_dict.keys():
+            create_exp = CreateVolfile (server_dict, server,
+                                        None, transports,
+                                        options, None)
+            try:
+                create_exp.create_pmap_export_volfile ()
+            except IOError, (errno, strerror):
+                print "Got %s creating server volfiles for %s" % (strerror, server)
+
+        for transport in transports:
+            create_mnt = CreateVolfile (server_dict, None,
+                                        transport, transports,
+                                        options, server_array)
+            try:
+                create_mnt.create_pmap_mount_volfile ()
+            except IOError, (errno, strerror):
+                print "Got %s creating client volfiles for transport '%s'" % (strerror, transport)
+
+        return
+
     for server in server_dict.keys():
         create_exp = CreateVolfile (server_dict, server,
                                     None, transports,
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 57ec659f9..b3d27efa0 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -287,10 +287,10 @@ glusterd_volume_create_generate_volfiles (glusterd_volinfo_t *volinfo)
                 case GF_CLUSTER_TYPE_REPLICATE:
                 {
                         snprintf (cmd_str, 8192,
-                                  "glusterfs-volgen -n %s -c %s -r 1 %s -p %d "
-                                  "--num-replica %d",
-                                   volinfo->volname, path, bricks,
-                                   volinfo->port, volinfo->sub_count);
+                                  "glusterfs-volgen --portmapper-mode -n %s "
+                                  " -c %s -r 1 %s -p %d --num-replica %d",
+                                  volinfo->volname, path, bricks,
+                                  volinfo->port, volinfo->sub_count);
                         ret = system (cmd_str);
                         break;
                 }
@@ -298,8 +298,8 @@ glusterd_volume_create_generate_volfiles (glusterd_volinfo_t *volinfo)
                 case GF_CLUSTER_TYPE_STRIPE:
                 {
                         snprintf (cmd_str, 8192,
-                                  "glusterfs-volgen -n %s -c %s -r 0 %s -p %d "
-                                  "--num-stripe %d",
+                                  "glusterfs-volgen --portmapper-mode -n %s "
+                                  " -c %s -r 0 %s -p %d --num-stripe %d",
                                   volinfo->volname, path, bricks,
                                   volinfo->port, volinfo->sub_count);
                         ret = system (cmd_str);
@@ -309,9 +309,11 @@ glusterd_volume_create_generate_volfiles (glusterd_volinfo_t *volinfo)
                 case GF_CLUSTER_TYPE_NONE:
                 {
                         snprintf (cmd_str, 8192,
-                                  "glusterfs-volgen -n %s -c %s %s -p %d",
+                                  "glusterfs-volgen --portmapper-mode -n %s "
+                                  " -c %s %s -p %d",
                                   volinfo->volname, path, bricks,
                                   volinfo->port);
+                        gf_log ("", 1, "%s", cmd_str);
                         ret = system (cmd_str);
                         break;
                 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index c135e7526..81c6cb2f4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -754,7 +754,7 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
                   brickinfo->hostname, volinfo->volname, count);
 
         snprintf (cmd_str, 8192,
-                  "glusterfs --xlator-option server-tcp.listen-port=%d -f %s -p %s",
+                  "glusterfs --xlator-option server-*.listen-port=%d -f %s -p %s",
                   port, volfile, pidfile);
         ret = system (cmd_str);