From e2093fb1500f55f58236d37a996609a2a1e1af8e Mon Sep 17 00:00:00 2001 From: Santosh Kumar Pradhan Date: Wed, 18 Sep 2013 14:43:40 +0530 Subject: gNFS: NFS daemon is limiting IOs to 64KB Problem: Gluster NFS server is hard-coding the max rsize/wsize to 64KB which is very less for NFS running over 10GE NIC. The existing options nfs.read-size, nfs.write-size are not working as expected. FIX: Make the options nfs.read-size (for rsize) and nfs.write-size (for wsize) work to tune the NFS I/O size. Value range would be 4KB(Min)-64KB(Default)-1MB(max). NB: Credit to "Richard Wareing" for catching it. Change-Id: I2754ecb0975692304308be8bcf496c713355f1c8 BUG: 1009223 Signed-off-by: Santosh Kumar Pradhan Reviewed-on: http://review.gluster.org/5964 Tested-by: Gluster Build System Reviewed-by: Kaleb KEITHLEY Reviewed-by: Anand Avati --- xlators/mgmt/glusterd/src/glusterd-volume-set.c | 18 +++++++++++ xlators/nfs/server/src/nfs.c | 16 +++++++--- xlators/nfs/server/src/nfs3-helpers.c | 2 +- xlators/nfs/server/src/nfs3.h | 40 ++++++++++++++++++------- 4 files changed, 61 insertions(+), 15 deletions(-) diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c index 8c6c5b954..76d2333d7 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c @@ -1248,6 +1248,24 @@ struct volopt_map_entry glusterd_volopt_map[] = { .type = GLOBAL_DOC, .op_version = 3 }, + { .key = "nfs.read-size", + .voltype = "nfs/server", + .option = "nfs3.read-size", + .type = GLOBAL_DOC, + .op_version = 3 + }, + { .key = "nfs.write-size", + .voltype = "nfs/server", + .option = "nfs3.write-size", + .type = GLOBAL_DOC, + .op_version = 3 + }, + { .key = "nfs.readdir-size", + .voltype = "nfs/server", + .option = "nfs3.readdir-size", + .type = GLOBAL_DOC, + .op_version = 3 + }, /* Other options which don't fit any place above */ { .key = "features.read-only", diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c index 831e82a8c..dfb92068e 100644 --- a/xlators/nfs/server/src/nfs.c +++ b/xlators/nfs/server/src/nfs.c @@ -1121,23 +1121,31 @@ struct xlator_dumpops dumpops = { struct volume_options options[] = { { .key = {"nfs3.read-size"}, .type = GF_OPTION_TYPE_SIZET, + .min = GF_NFS3_RTMIN, + .max = GF_NFS3_RTMAX, .default_value = TOSTRING(GF_NFS3_RTPREF), .description = "Size in which the client should issue read requests" " to the Gluster NFSv3 server. Must be a multiple of" - " 4KB." + " 4KB. Max supported value is 1MB(1048576)." }, { .key = {"nfs3.write-size"}, .type = GF_OPTION_TYPE_SIZET, + .min = GF_NFS3_WTMIN, + .max = GF_NFS3_WTMAX, .default_value = TOSTRING(GF_NFS3_WTPREF), .description = "Size in which the client should issue write requests" " to the Gluster NFSv3 server. Must be a multiple of" - " 4KB." + " 4KB. Max supported value is 1MB(1048576)." }, { .key = {"nfs3.readdir-size"}, .type = GF_OPTION_TYPE_SIZET, + .min = GF_NFS3_DTMIN, + .max = GF_NFS3_DTMAX, .default_value = TOSTRING(GF_NFS3_DTPREF), - .description = "Size in which the client should issue directory " - " reading requests." + .description = "Size in which the client should issue directory" + " reading requests to Gluster NFSv3 server. Must" + " be a multiple of 4KB. Max supported value is" + " 1MB(1048576)." }, { .key = {"nfs3.*.volume-access"}, .type = GF_OPTION_TYPE_STR, diff --git a/xlators/nfs/server/src/nfs3-helpers.c b/xlators/nfs/server/src/nfs3-helpers.c index 4a5854d2b..9069a90d7 100644 --- a/xlators/nfs/server/src/nfs3-helpers.c +++ b/xlators/nfs/server/src/nfs3-helpers.c @@ -496,7 +496,7 @@ nfs3_fill_fsinfo3res (struct nfs3_state *nfs3, fsinfo3res *res, resok.wtpref = nfs3->writesize; resok.wtmult = GF_NFS3_WTMULT; resok.dtpref = nfs3->readdirsize; - resok.maxfilesize = GF_NFS3_MAXFILE; + resok.maxfilesize = GF_NFS3_MAXFILESIZE; resok.time_delta = tdelta; resok.properties = GF_NFS3_FS_PROP; diff --git a/xlators/nfs/server/src/nfs3.h b/xlators/nfs/server/src/nfs3.h index 6b37cd458..19353abea 100644 --- a/xlators/nfs/server/src/nfs3.h +++ b/xlators/nfs/server/src/nfs3.h @@ -48,16 +48,36 @@ /* Static values used for FSINFO -FIXME: This should be configurable */ -#define GF_NFS3_RTMAX (64 * GF_UNIT_KB) -#define GF_NFS3_RTPREF (64 * GF_UNIT_KB) -#define GF_NFS3_RTMULT (4 * GF_UNIT_KB) -#define GF_NFS3_WTMAX (64 * GF_UNIT_KB) -#define GF_NFS3_WTPREF (64 * GF_UNIT_KB) -#define GF_NFS3_WTMULT (4 * GF_UNIT_KB) -#define GF_NFS3_DTMIN (4 * GF_UNIT_KB) -#define GF_NFS3_DTPREF (64 * GF_UNIT_KB) -#define GF_NFS3_MAXFILE (1 * GF_UNIT_PB) + * To change the maximum rsize and wsize supported by the NFS client, + * adjust GF_NFS3_FILE_IO_SIZE_MAX. The Gluster NFS server defaults to + * 64KB(65536) which is reasonable for 1GE and 10GE. But for better + * performance on 10GE, rsize/wsize can be raised to 1MB(1048576). + * rsize and wsize can be tuned through nfs.read-size and + * nfs.write-size respectively. + * + * NB: For Kernel-NFS, NFS_MAX_FILE_IO_SIZE is 1048576U (1MB). + */ +#define GF_NFS3_FILE_IO_SIZE_MAX (1 * GF_UNIT_MB) /* 1048576 */ +#define GF_NFS3_FILE_IO_SIZE_DEF (64 * GF_UNIT_KB) /* 65536 */ +#define GF_NFS3_FILE_IO_SIZE_MIN (4 * GF_UNIT_KB) /* 4096 */ + +#define GF_NFS3_RTMAX GF_NFS3_FILE_IO_SIZE_MAX +#define GF_NFS3_RTMIN GF_NFS3_FILE_IO_SIZE_MIN +#define GF_NFS3_RTPREF GF_NFS3_FILE_IO_SIZE_DEF +#define GF_NFS3_RTMULT GF_NFS3_FILE_IO_SIZE_MIN + +#define GF_NFS3_WTMAX GF_NFS3_FILE_IO_SIZE_MAX +#define GF_NFS3_WTMIN GF_NFS3_FILE_IO_SIZE_MIN +#define GF_NFS3_WTPREF GF_NFS3_FILE_IO_SIZE_DEF +#define GF_NFS3_WTMULT GF_NFS3_FILE_IO_SIZE_MIN + +/* This can be tuned through nfs.readdir-size */ +#define GF_NFS3_DTMAX GF_NFS3_FILE_IO_SIZE_MAX +#define GF_NFS3_DTMIN GF_NFS3_FILE_IO_SIZE_MIN +#define GF_NFS3_DTPREF GF_NFS3_FILE_IO_SIZE_DEF + +#define GF_NFS3_MAXFILESIZE (1 * GF_UNIT_PB) + /* FIXME: Handle time resolutions */ #define GF_NFS3_TIMEDELTA_SECS {1,0} #define GF_NFS3_TIMEDELTA_NSECS {0,1} -- cgit