summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAmar Tumballi <amar@gluster.com>2010-08-14 04:36:47 +0000
committerAnand V. Avati <avati@dev.gluster.com>2010-08-15 11:28:40 -0700
commit2b8545f1a21efe5e9944bccdebdff49e22fb6e6b (patch)
tree2195f781bfaaf95cda486792645cf9675d5f2b2c
parentdd6c4b0e0db5fbe05a0c7927c780c96d5a97a966 (diff)
new option to volgen '--portmapper-mode'
* without which, it generates volume files in old fashioned way. * with portmapper mode, no port option is taken, and there will be one process per one export. * corresponding changes in 'mgmt/glusterd' is made Signed-off-by: Amar Tumballi <amar@gluster.com> Signed-off-by: Anand V. Avati <avati@dev.gluster.com> BUG: 1341 (glusterfs-volgen should behave differently for flag '--internal') URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1341
-rw-r--r--extras/volgen/CreateVolfile.py280
-rwxr-xr-xextras/volgen/glusterfs-volgen.in23
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-op-sm.c16
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-utils.c2
4 files changed, 282 insertions, 39 deletions
diff --git a/extras/volgen/CreateVolfile.py b/extras/volgen/CreateVolfile.py
index df38a9946a5..fdcf6056639 100644
--- a/extras/volgen/CreateVolfile.py
+++ b/extras/volgen/CreateVolfile.py
@@ -45,10 +45,10 @@ class CreateVolfile:
self.gfs_port = options.port
self.gfs_ib_port = options.port + 1
else:
- self.gfs_port = None
- self.gfs_ib_port = None
+ self.gfs_port = 6969
+ self.gfs_ib_port = 6970
- def create_mount_volfile (self):
+ def create_pmap_mount_volfile (self):
raid_type = self.raid_type
@@ -72,7 +72,7 @@ class CreateVolfile:
for host in self.host_dict.keys():
i = 1
for export in self.host_dict[host]:
- mount_fd.write ("volume %s-%s\n" % (host,i))
+ mount_fd.write ("volume %s-%s\n" % (host,export))
mount_fd.write (" type protocol/client\n")
mount_fd.write (" option transport-type %s\n" %
self.transport)
@@ -80,26 +80,13 @@ class CreateVolfile:
if self.transport == 'ib-verbs':
mount_fd.write (" option transport.ib-verbs.port %d\n" %
self.ib_devport)
- if self.gfs_ib_port:
- mount_fd.write (" option remote-port %d\n" %
- self.gfs_ib_port)
if self.transport == 'tcp':
mount_fd.write (" option transport.socket.nodelay on\n")
- if self.gfs_port:
- mount_fd.write (" option remote-port %d\n" % self.gfs_port)
mount_fd.write (" option remote-subvolume %s\n" % export)
mount_fd.write ("end-volume\n\n")
i += 1
-
- exportlist = {}
- for entry in self.host_array:
- node = entry.split(':')[0]
- if not exportlist.has_key(node):
- exportlist[node] = 1
- else:
- exportlist[node] += 1
- subvolumes.append(str(node) + '-' + str(exportlist[node]))
+ subvolumes.append(str(host) + '-' + str(export))
# Stripe section.. if given
if raid_type is 0:
@@ -250,7 +237,7 @@ class CreateVolfile:
return
- def create_export_volfile (self):
+ def create_pmap_export_volfile (self):
cmdline = string.join (sys.argv, ' ')
i = 0
@@ -297,7 +284,6 @@ class CreateVolfile:
exp_fd.write (" subvolumes posix\n")
exp_fd.write ("end-volume\n\n")
-
exp_fd.write ("volume locks\n")
exp_fd.write (" type features/locks\n")
exp_fd.write ("# option mandatory on # Default off, used in specific applications\n")
@@ -313,9 +299,6 @@ class CreateVolfile:
exp_fd.write (" type protocol/client\n")
if self.transport:
exp_fd.write (" option transport-type %s\n" % self.transport)
-
- if self.gfs_port:
- exp_fd.write (" option transport.remote-port %d\n" % self.gfs_port)
exp_fd.write (" option ping-timeout 42\n")
exp_fd.write ("end-volume\n\n")
@@ -330,28 +313,263 @@ class CreateVolfile:
exp_fd.write ("# option autoscaling yes # Heuristic for autoscaling threads on demand\n")
exp_fd.write ("# option min-threads 2 # min count for thread pool\n")
exp_fd.write ("# option max-threads 64 # max count for thread pool\n")
-
exp_fd.write (" subvolumes pump\n")
exp_fd.write ("end-volume\n\n")
for transport in self.transports:
- exp_fd.write ("volume server-%s\n" % transport)
+ exp_fd.write ("volume server-%s\n" % export)
exp_fd.write (" type protocol/server\n")
exp_fd.write (" option transport-type %s\n" % transport)
exp_fd.write (" option auth.addr.%s.allow %s\n" %
(export, self.auth_parameters))
-
if transport == 'ib-verbs':
- if self.gfs_ib_port:
- exp_fd.write (" option listen-port %d\n" % self.gfs_ib_port)
exp_fd.write (" option transport.ib-verbs.port %d\n" %
self.ib_devport)
if transport == 'tcp':
- if self.gfs_port:
- exp_fd.write (" option listen-port %d\n" % self.gfs_port)
exp_fd.write (" option transport.socket.nodelay on\n")
-
exp_fd.write (" subvolumes %s\n" % export)
exp_fd.write ("end-volume\n\n")
return
+
+
+ def create_mount_volfile (self):
+
+ raid_type = self.raid_type
+
+ mount_volfile = "%s/%s-%s.vol" % (self.conf_dir, str(self.volume_name), str(self.transport))
+ mount_fd = file ("%s" % (mount_volfile), "w")
+
+ print "Generating client volfiles.. '%s'" % mount_volfile
+
+ cmdline = string.join (sys.argv, ' ')
+
+ mount_fd.write ("## file auto generated by %s\n" % sys.argv[0])
+ mount_fd.write ("# Cmd line:\n")
+ mount_fd.write ("# $ %s\n\n" % cmdline)
+
+ if raid_type is not None:
+ # Used for later usage
+ mount_fd.write ("# RAID %d\n" % raid_type)
+
+ mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport)
+ subvolumes = []
+ for host in self.host_dict.keys():
+ i = 1
+ for exports in self.host_dict[host]:
+ mount_fd.write ("volume %s-%s\n" % (host,exports))
+ mount_fd.write (" type protocol/client\n")
+ mount_fd.write (" option transport-type %s\n" %
+ self.transport)
+ mount_fd.write (" option remote-host %s\n" % host)
+ if self.transport == 'ib-verbs':
+ mount_fd.write (" option transport.ib-verbs.port %d\n" %
+ self.ib_devport)
+ mount_fd.write (" option remote-port %d\n" %
+ self.gfs_ib_port)
+ if self.transport == 'tcp':
+ mount_fd.write (" option transport.socket.nodelay on\n")
+ mount_fd.write (" option remote-port %d\n" %
+ self.gfs_port)
+
+ mount_fd.write (" option remote-subvolume %s\n" % exports)
+ mount_fd.write ("end-volume\n\n")
+ subvolumes.append(str(host) + '-' + str(exports))
+
+ # Stripe section.. if given
+ if raid_type is 0:
+ max_stripe_idx = len (subvolumes) / self.num_stripe
+ stripe_idx = 0
+ index = 0
+ while index < max_stripe_idx:
+ mount_fd.write ("volume stripe-%d\n" % index)
+ mount_fd.write (" type cluster/stripe\n")
+ mount_fd.write ("# option block-size 128k\n")
+ mount_fd.write ("# option use-xattr no\n")
+ mount_fd.write (" subvolumes %s" % subvolumes[stripe_idx])
+ sub_idx = 1
+ while sub_idx < self.num_stripe:
+ mount_fd.write (" %s" % subvolumes[stripe_idx+sub_idx])
+ sub_idx += 1
+ mount_fd.write ("\nend-volume\n\n")
+ stripe_idx += self.num_stripe
+ index +=1
+
+ # Replicate section
+ if raid_type is 1:
+ max_mirror_idx = len (subvolumes) / self.num_replica
+ mirror_idx = 0
+ index = 0
+ while index < max_mirror_idx:
+ mount_fd.write ("volume mirror-%d\n" % index)
+ mount_fd.write (" type cluster/replicate\n")
+ mount_fd.write (" subvolumes %s" % subvolumes[mirror_idx])
+ sub_idx = 1
+ while sub_idx < self.num_replica:
+ mount_fd.write (" %s" % subvolumes[mirror_idx + sub_idx])
+ sub_idx += 1
+ mount_fd.write ("\nend-volume\n\n")
+ mirror_idx += self.num_replica
+ index += 1
+
+ # Distribute section
+ if raid_type is 0:
+ subvolumes = []
+ flag = 0
+ while flag < index:
+ subvolumes.append ("stripe-%d" % flag)
+ flag += 1
+
+ if raid_type is 1:
+ subvolumes = []
+ flag = 0
+ while flag < index:
+ subvolumes.append ("mirror-%d" % flag)
+ flag += 1
+
+ if len (subvolumes) > 1:
+ mount_fd.write ("volume distribute\n")
+ mount_fd.write (" type cluster/distribute\n")
+ mount_fd.write ("# option unhashed-sticky-bit yes"
+ " # Used for migrating data while adding new nodes\n")
+ mount_fd.write ("# option min-free-disk 5%"
+ " # Minimum free disk available on the volume\n")
+ mount_fd.write (" subvolumes %s\n" %
+ string.join (subvolumes,' '))
+ mount_fd.write ("end-volume\n\n")
+ subvolumes[0] = "distribute"
+
+
+ if self.nfs:
+ mount_fd.write ("volume nfsxlator\n")
+ mount_fd.write (" type nfs/server\n")
+ mount_fd.write (" subvolumes %s\n" % subvolumes[0])
+ mount_fd.write ("# option rpc-auth.auth-unix off #Enabled by default\n")
+ mount_fd.write ("# option rpc-auth.auth-null off #Enabled by default\n")
+ mount_fd.write ("# By default all addresses are rejected until allowed.\n")
+ mount_fd.write ("# option rpc-auth.addr.reject 127.*\n")
+ mount_fd.write ("# option rpc-auth.addr.allow localhost\n")
+ mount_fd.write (" option rpc-auth.addr.%s.allow %s\n" % (subvolumes[0], self.auth_parameters))
+ mount_fd.write ("# By default insecure ports are not allowed.\n")
+ mount_fd.write ("# option rpc-auth.ports.insecure on\n")
+ mount_fd.write ("# option rpc-auth.ports.<volume>.insecure on\n")
+ mount_fd.write ("# By default all access is read-write.\n")
+ mount_fd.write ("# option nfs3.<volume>.volume-access read-only\n")
+ mount_fd.write ("# option nfs3.<volume>.volume-access read-only\n")
+ mount_fd.write ("# option nfs3.read-size 128Kb\n")
+ mount_fd.write ("# option nfs3.write-size 32Kb\n")
+ mount_fd.write ("# option nfs3.readdir-size 64Kb\n")
+ mount_fd.write ("# option nfs3.<volume>.read-size 64Kb\n")
+ mount_fd.write ("# option nfs3.<volume>.write-size 64Kb\n")
+ mount_fd.write ("# option nfs3.posix1.readdir-size 128Kb\n")
+ mount_fd.write ("end-volume\n\n")
+ return
+
+ if self.volume_size_client:
+ mount_fd.write ("volume quota\n")
+ mount_fd.write (" type features/quota\n")
+ mount_fd.write (" option disk-usage-limit %s\n" % self.volume_size_client)
+ mount_fd.write ("# option minimum-free-disk-limit 10GB\n"
+ "# minimum free disk value (default) 0\n")
+ mount_fd.write ("# option refresh-interval 10\n")
+ mount_fd.write (" subvolumes %s\n" % subvolumes[0])
+ mount_fd.write ("end-volume\n\n")
+
+ if self.enable_safe_mode:
+ return
+
+ self.performance_mode (subvolumes[0], mount_fd)
+
+ return
+
+ def create_export_volfile (self):
+
+ cmdline = string.join (sys.argv, ' ')
+ export_volfile = "%s/%s-export.vol" % (self.conf_dir, str(self.host + '-' + self.volume_name))
+ exp_fd = file ("%s" % (export_volfile),"w")
+
+ print "Generating server volfiles.. for server %s as '%s'" % (self.host,
+ export_volfile)
+
+ exp_fd.write ("## file auto generated by %s\n" %
+ sys.argv[0])
+ exp_fd.write ("# Cmd line:\n")
+ exp_fd.write ("# $ %s\n\n" % cmdline)
+ total_bricks = []
+ i=1
+ for export in self.host_dict[self.host]:
+ exp_fd.write ("volume posix%d\n" % i)
+ exp_fd.write (" type storage/posix\n")
+ exp_fd.write ("# option o-direct enable # (default: disable) boolean type only\n")
+ exp_fd.write ("# option export-statfs-size no # (default: yes) boolean type only\n")
+ exp_fd.write ("# option mandate-attribute off # (default: on) boolean type only\n")
+ exp_fd.write ("# option span-devices 8 # (default: 0) integer value\n")
+ exp_fd.write ("# option background-unlink yes # (default: no) boolean type\n")
+
+ exp_fd.write (" option directory %s\n" % export)
+ exp_fd.write ("end-volume\n\n")
+
+ if self.nfs:
+ exp_fd.write ("volume posix-ac%d\n" % i)
+ exp_fd.write (" type features/access-control\n")
+ exp_fd.write (" subvolumes posix%d\n" % i)
+ exp_fd.write ("end-volume\n\n")
+
+ if self.volume_size_server:
+ exp_fd.write ("volume quota%d\n" % i)
+ exp_fd.write (" type features/quota\n")
+ exp_fd.write (" option disk-usage-limit %s\n" % self.volume_size_server)
+ exp_fd.write ("# option minimum-free-disk-limit 10GB"
+ " # minimum free disk value (default) 0\n")
+ exp_fd.write ("# option refresh-interval 10\n")
+ if self.nfs:
+ exp_fd.write (" subvolumes posix-ac%d\n" % i)
+ else:
+ exp_fd.write (" subvolumes posix%d\n" % i)
+ exp_fd.write ("end-volume\n\n")
+
+
+ exp_fd.write ("volume locks%d\n" % i)
+ exp_fd.write (" type features/locks\n")
+ exp_fd.write ("# option mandatory on # Default off, used in specific applications\n")
+ if self.volume_size_server:
+ exp_fd.write (" subvolumes quota%d\n" % i)
+ else:
+ exp_fd.write (" subvolumes posix%d\n" % i)
+ exp_fd.write ("end-volume\n\n")
+
+ exp_fd.write ("volume %s\n" % export)
+ exp_fd.write (" type performance/io-threads\n")
+ exp_fd.write (" option thread-count 8\n")
+ exp_fd.write ("# option autoscaling yes # Heuristic for autoscaling threads on demand\n")
+ exp_fd.write ("# option min-threads 2 # min count for thread pool\n")
+ exp_fd.write ("# option max-threads 64 # max count for thread pool\n")
+
+ exp_fd.write (" subvolumes locks%d\n" % i)
+ exp_fd.write ("end-volume\n\n")
+
+ total_bricks.append("%s" % export)
+ i += 1
+
+ for transport in self.transports:
+ exp_fd.write ("volume server-%s\n" % transport)
+ exp_fd.write (" type protocol/server\n")
+ exp_fd.write (" option transport-type %s\n" % transport)
+ for brick in total_bricks:
+ exp_fd.write (" option auth.addr.%s.allow %s\n" %
+ (brick, self.auth_parameters))
+
+ if transport == 'ib-verbs':
+ exp_fd.write (" option transport.ib-verbs.listen-port %d\n" % self.gfs_ib_port)
+ exp_fd.write (" option transport.ib-verbs.port %d\n" %
+ self.ib_devport)
+ if transport == 'tcp':
+ exp_fd.write (" option transport.socket.listen-port %d\n" % self.gfs_port)
+ exp_fd.write (" option transport.socket.nodelay on\n")
+
+ exp_fd.write (" subvolumes %s\n" %
+ string.join(total_bricks, ' '))
+ exp_fd.write ("end-volume\n\n")
+
+ return
+
diff --git a/extras/volgen/glusterfs-volgen.in b/extras/volgen/glusterfs-volgen.in
index 3ab5aa1cdc0..31a81122810 100755
--- a/extras/volgen/glusterfs-volgen.in
+++ b/extras/volgen/glusterfs-volgen.in
@@ -72,6 +72,8 @@ def generate_volume_files ():
help="number of file replica needed")
group.add_option("--num-stripe", type="int", dest="num_stripe",
help="number of stripe count needed")
+ group.add_option("--portmapper-mode", action="store_true", dest="portmapper",
+ default=False, help="generate volume files for glusterfs portmapper friendly way")
parse.add_option_group(group)
@@ -164,6 +166,27 @@ def generate_volume_files ():
len(server_array))
raise ValueError
+ if options.portmapper is True:
+ for server in server_dict.keys():
+ create_exp = CreateVolfile (server_dict, server,
+ None, transports,
+ options, None)
+ try:
+ create_exp.create_pmap_export_volfile ()
+ except IOError, (errno, strerror):
+ print "Got %s creating server volfiles for %s" % (strerror, server)
+
+ for transport in transports:
+ create_mnt = CreateVolfile (server_dict, None,
+ transport, transports,
+ options, server_array)
+ try:
+ create_mnt.create_pmap_mount_volfile ()
+ except IOError, (errno, strerror):
+ print "Got %s creating client volfiles for transport '%s'" % (strerror, transport)
+
+ return
+
for server in server_dict.keys():
create_exp = CreateVolfile (server_dict, server,
None, transports,
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 57ec659f999..b3d27efa0fb 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -287,10 +287,10 @@ glusterd_volume_create_generate_volfiles (glusterd_volinfo_t *volinfo)
case GF_CLUSTER_TYPE_REPLICATE:
{
snprintf (cmd_str, 8192,
- "glusterfs-volgen -n %s -c %s -r 1 %s -p %d "
- "--num-replica %d",
- volinfo->volname, path, bricks,
- volinfo->port, volinfo->sub_count);
+ "glusterfs-volgen --portmapper-mode -n %s "
+ " -c %s -r 1 %s -p %d --num-replica %d",
+ volinfo->volname, path, bricks,
+ volinfo->port, volinfo->sub_count);
ret = system (cmd_str);
break;
}
@@ -298,8 +298,8 @@ glusterd_volume_create_generate_volfiles (glusterd_volinfo_t *volinfo)
case GF_CLUSTER_TYPE_STRIPE:
{
snprintf (cmd_str, 8192,
- "glusterfs-volgen -n %s -c %s -r 0 %s -p %d "
- "--num-stripe %d",
+ "glusterfs-volgen --portmapper-mode -n %s "
+ " -c %s -r 0 %s -p %d --num-stripe %d",
volinfo->volname, path, bricks,
volinfo->port, volinfo->sub_count);
ret = system (cmd_str);
@@ -309,9 +309,11 @@ glusterd_volume_create_generate_volfiles (glusterd_volinfo_t *volinfo)
case GF_CLUSTER_TYPE_NONE:
{
snprintf (cmd_str, 8192,
- "glusterfs-volgen -n %s -c %s %s -p %d",
+ "glusterfs-volgen --portmapper-mode -n %s "
+ " -c %s %s -p %d",
volinfo->volname, path, bricks,
volinfo->port);
+ gf_log ("", 1, "%s", cmd_str);
ret = system (cmd_str);
break;
}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index c135e75264f..81c6cb2f4e4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -754,7 +754,7 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo,
brickinfo->hostname, volinfo->volname, count);
snprintf (cmd_str, 8192,
- "glusterfs --xlator-option server-tcp.listen-port=%d -f %s -p %s",
+ "glusterfs --xlator-option server-*.listen-port=%d -f %s -p %s",
port, volfile, pidfile);
ret = system (cmd_str);