From e479660d9dd8bf7017c7dc78ccfa6edd9c51ec7a Mon Sep 17 00:00:00 2001 From: Santosh Kumar Pradhan Date: Mon, 28 Oct 2013 12:46:37 +0530 Subject: gNFS: RFE for NFS connection behavior Implement reconfigure() for NFS xlator so that volume set/reset wont restart the NFS server process. But few options can not be reconfigured dynamically e.g. nfs.mem-factor, nfs.port etc which needs NFS to be restarted. Change-Id: Ic586fd55b7933c0a3175708d8c41ed0475d74a1c BUG: 1027409 Signed-off-by: Santosh Kumar Pradhan Reviewed-on: http://review.gluster.org/6236 Tested-by: Gluster Build System Reviewed-by: Rajesh Joseph Reviewed-by: Anand Avati --- libglusterfs/src/gidcache.c | 15 + libglusterfs/src/gidcache.h | 1 + rpc/rpc-lib/src/rpc-drc.c | 63 +++- rpc/rpc-lib/src/rpc-drc.h | 4 + rpc/rpc-lib/src/rpc-transport.c | 2 +- rpc/rpc-lib/src/rpcsvc-auth.c | 43 +++ rpc/rpc-lib/src/rpcsvc-common.h | 15 +- rpc/rpc-lib/src/rpcsvc.c | 162 ++++++++-- rpc/rpc-lib/src/rpcsvc.h | 19 +- xlators/mgmt/glusterd/src/glusterd-utils.c | 92 ++++++ xlators/mgmt/glusterd/src/glusterd-utils.h | 6 + xlators/mgmt/glusterd/src/glusterd-volgen.c | 48 +++ xlators/mgmt/glusterd/src/glusterd-volgen.h | 2 + xlators/nfs/server/src/acl3.c | 15 +- xlators/nfs/server/src/mount3.c | 71 ++++- xlators/nfs/server/src/mount3.h | 7 +- xlators/nfs/server/src/nfs.c | 466 +++++++++++++++++++++++++--- xlators/nfs/server/src/nfs.h | 3 +- xlators/nfs/server/src/nfs3-helpers.c | 14 + xlators/nfs/server/src/nfs3-helpers.h | 3 + xlators/nfs/server/src/nfs3.c | 125 +++++--- xlators/nfs/server/src/nfs3.h | 11 +- xlators/nfs/server/src/nlm4.c | 6 + xlators/protocol/server/src/server.c | 7 +- 24 files changed, 1041 insertions(+), 159 deletions(-) diff --git a/libglusterfs/src/gidcache.c b/libglusterfs/src/gidcache.c index c55ed2581..c5bdda925 100644 --- a/libglusterfs/src/gidcache.c +++ b/libglusterfs/src/gidcache.c @@ -34,6 +34,21 @@ int gid_cache_init(gid_cache_t *cache, uint32_t timeout) return 0; } +/* + * Reconfigure the cache timeout. + */ +int gid_cache_reconf(gid_cache_t *cache, uint32_t timeout) +{ + if (!cache) + return -1; + + LOCK(&cache->gc_lock); + cache->gc_max_age = timeout; + UNLOCK(&cache->gc_lock); + + return 0; +} + /* * Look up an ID in the cache. If found, return the actual cache entry to avoid * an additional allocation and memory copy. The caller should copy the data and diff --git a/libglusterfs/src/gidcache.h b/libglusterfs/src/gidcache.h index f904f26eb..9379f8e8b 100644 --- a/libglusterfs/src/gidcache.h +++ b/libglusterfs/src/gidcache.h @@ -45,6 +45,7 @@ typedef struct { } gid_cache_t; int gid_cache_init(gid_cache_t *, uint32_t); +int gid_cache_reconf(gid_cache_t *, uint32_t); const gid_list_t *gid_cache_lookup(gid_cache_t *, uint64_t); void gid_cache_release(gid_cache_t *, const gid_list_t *); int gid_cache_add(gid_cache_t *, gid_list_t *); diff --git a/rpc/rpc-lib/src/rpc-drc.c b/rpc/rpc-lib/src/rpc-drc.c index 66d07cfe6..8181e6aee 100644 --- a/rpc/rpc-lib/src/rpc-drc.c +++ b/rpc/rpc-lib/src/rpc-drc.c @@ -708,10 +708,15 @@ rpcsvc_drc_init (rpcsvc_t *svc, dict_t *options) uint32_t drc_size = 0; uint32_t drc_factor = 0; rpcsvc_drc_globals_t *drc = NULL; + static gf_boolean_t drc_inited = _gf_false; GF_ASSERT (svc); GF_ASSERT (options); + /* Already inited */ + if (drc_inited) + return 0; + if (!svc->drc) { drc = GF_CALLOC (1, sizeof (rpcsvc_drc_globals_t), gf_common_mt_drc_globals_t); @@ -732,11 +737,12 @@ rpcsvc_drc_init (rpcsvc_t *svc, dict_t *options) /* Toggle DRC on/off, when more drc types(persistent/cluster) are added, we shouldn't treat this as boolean */ - ret = dict_get_str_boolean (options, "nfs.drc", _gf_false); + ret = dict_get_str_boolean (options, "nfs.drc", _gf_true); if (ret == -1) { gf_log (GF_RPCSVC, GF_LOG_INFO, "drc user options need second look"); ret = _gf_true; } + drc->enable_drc = ret; if (ret == _gf_false) { /* drc off */ @@ -796,6 +802,7 @@ rpcsvc_drc_init (rpcsvc_t *svc, dict_t *options) gf_log (GF_RPCSVC, GF_LOG_DEBUG, "drc init successful"); drc->status = DRC_INITIATED; + drc_inited = _gf_true; out: UNLOCK (&drc->lock); @@ -809,3 +816,57 @@ rpcsvc_drc_init (rpcsvc_t *svc, dict_t *options) } return ret; } + +int +rpcsvc_drc_reconfigure (rpcsvc_t *svc, dict_t *options) +{ + int ret = -1; + gf_boolean_t enable_drc = _gf_false; + rpcsvc_drc_globals_t *drc = NULL; + uint32_t drc_size = 0; + + if ((!svc) || (!options)) + return (-1); + + drc = svc->drc; + /* reconfig for drc-size */ + if (dict_get_uint32 (options, "nfs.drc-size", &drc_size)) + drc_size = DRC_DEFAULT_CACHE_SIZE; + + if (drc->global_cache_size != drc_size) { + gf_log (GF_RPCSVC, GF_LOG_DEBUG, "nfs.drc-size size can not " + "be reconfigured without NFS server restart."); + return (-1); + } + + /* reconfig for nfs.drc */ + ret = dict_get_str_boolean (options, "nfs.drc", _gf_true); + if (ret < 0) { + ret = _gf_true; + } + enable_drc = ret; + + if (drc->enable_drc == enable_drc) + return 0; + + drc->enable_drc = enable_drc; + if (enable_drc) { + if (drc == NULL) + return rpcsvc_drc_init(svc, options); + } else { + if (drc == NULL) + return (0); + + LOCK (&drc->lock); + (void) rpcsvc_unregister_notify (svc, rpcsvc_drc_notify, THIS); + if (drc->mempool) { + mem_pool_destroy (drc->mempool); + drc->mempool = NULL; + } + UNLOCK (&drc->lock); + GF_FREE (drc); + svc->drc = NULL; + } + + return (0); +} diff --git a/rpc/rpc-lib/src/rpc-drc.h b/rpc/rpc-lib/src/rpc-drc.h index 0a1688992..7dfaef978 100644 --- a/rpc/rpc-lib/src/rpc-drc.h +++ b/rpc/rpc-lib/src/rpc-drc.h @@ -71,6 +71,7 @@ struct drc_globals { struct list_head cache_head; uint32_t client_count; struct list_head clients_head; + gf_boolean_t enable_drc; }; int @@ -97,4 +98,7 @@ rpcsvc_drc_priv (rpcsvc_drc_globals_t *drc); int rpcsvc_drc_init (rpcsvc_t *svc, dict_t *options); +int +rpcsvc_drc_reconfigure (rpcsvc_t *svc, dict_t *options); + #endif /* RPC_DRC_H */ diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c index 5f2e91c70..c24d41084 100644 --- a/rpc/rpc-lib/src/rpc-transport.c +++ b/rpc/rpc-lib/src/rpc-transport.c @@ -295,7 +295,7 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name) } *VOID(&(trans->reconfigure)) = dlsym (handle, "reconfigure"); - if (trans->fini == NULL) { + if (trans->reconfigure == NULL) { gf_log ("rpc-transport", GF_LOG_DEBUG, "dlsym (gf_rpc_transport_reconfigure) on %s", dlerror()); } diff --git a/rpc/rpc-lib/src/rpcsvc-auth.c b/rpc/rpc-lib/src/rpcsvc-auth.c index 7ee96d85b..4cb86a758 100644 --- a/rpc/rpc-lib/src/rpcsvc-auth.c +++ b/rpc/rpc-lib/src/rpcsvc-auth.c @@ -177,6 +177,29 @@ err: } +int +rpcsvc_set_addr_namelookup (rpcsvc_t *svc, dict_t *options) +{ + int ret; + static char *addrlookup_key = "rpc-auth.addr.namelookup"; + + if (!svc || !options) + return (-1); + + /* By default it's disabled */ + ret = dict_get_str_boolean (options, addrlookup_key, _gf_false); + if (ret < 0) { + svc->addr_namelookup = _gf_false; + } else { + svc->addr_namelookup = ret; + } + + if (svc->addr_namelookup) + gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Addr-Name lookup enabled"); + + return (0); +} + int rpcsvc_set_allow_insecure (rpcsvc_t *svc, dict_t *options) { @@ -233,6 +256,7 @@ rpcsvc_auth_init (rpcsvc_t *svc, dict_t *options) (void) rpcsvc_set_allow_insecure (svc, options); (void) rpcsvc_set_root_squash (svc, options); + (void) rpcsvc_set_addr_namelookup (svc, options); ret = rpcsvc_auth_add_initers (svc); if (ret == -1) { gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to add initers"); @@ -249,6 +273,25 @@ out: return ret; } +int +rpcsvc_auth_reconf (rpcsvc_t *svc, dict_t *options) +{ + int ret = 0; + + if ((!svc) || (!options)) + return (-1); + + ret = rpcsvc_set_allow_insecure (svc, options); + if (ret) + return (-1); + + ret = rpcsvc_set_root_squash (svc, options); + if (ret) + return (-1); + + return rpcsvc_set_addr_namelookup (svc, options); +} + rpcsvc_auth_t * __rpcsvc_auth_get_handler (rpcsvc_request_t *req) diff --git a/rpc/rpc-lib/src/rpcsvc-common.h b/rpc/rpc-lib/src/rpcsvc-common.h index 53c1a8fe3..aed55e039 100644 --- a/rpc/rpc-lib/src/rpcsvc-common.h +++ b/rpc/rpc-lib/src/rpcsvc-common.h @@ -52,28 +52,29 @@ typedef struct rpcsvc_state { dict_t *options; /* Allow insecure ports. */ - int allow_insecure; + gf_boolean_t allow_insecure; gf_boolean_t register_portmap; gf_boolean_t root_squash; glusterfs_ctx_t *ctx; /* list of connections which will listen for incoming connections */ - struct list_head listeners; + struct list_head listeners; /* list of programs registered with rpcsvc */ - struct list_head programs; + struct list_head programs; /* list of notification callbacks */ - struct list_head notify; - int notify_count; + struct list_head notify; + int notify_count; void *mydata; /* This is xlator */ - rpcsvc_notify_t notifyfn; + rpcsvc_notify_t notifyfn; struct mem_pool *rxpool; rpcsvc_drc_globals_t *drc; /* per-client limit of outstanding rpc requests */ - int outstanding_rpc_limit; + int outstanding_rpc_limit; + gf_boolean_t addr_namelookup; } rpcsvc_t; /* DRC START */ diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c index 8fe2e52bc..037c157f2 100644 --- a/rpc/rpc-lib/src/rpcsvc.c +++ b/rpc/rpc-lib/src/rpcsvc.c @@ -1249,12 +1249,13 @@ rpcsvc_error_reply (rpcsvc_request_t *req) inline int rpcsvc_program_register_portmap (rpcsvc_program_t *newprog, uint32_t port) { - int ret = 0; + int ret = -1; /* FAIL */ if (!newprog) { goto out; } + /* pmap_set() returns 0 for FAIL and 1 for SUCCESS */ if (!(pmap_set (newprog->prognum, newprog->progver, IPPROTO_TCP, port))) { gf_log (GF_RPCSVC, GF_LOG_ERROR, "Could not register with" @@ -1262,16 +1263,16 @@ rpcsvc_program_register_portmap (rpcsvc_program_t *newprog, uint32_t port) goto out; } - ret = 0; + ret = 0; /* SUCCESS */ out: return ret; } -static inline int +inline int rpcsvc_program_unregister_portmap (rpcsvc_program_t *prog) { - int ret = 0; + int ret = -1; if (!prog) goto out; @@ -1889,27 +1890,89 @@ rpcsvc_init_options (rpcsvc_t *svc, dict_t *options) gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Portmap registration " "disabled"); - svc->outstanding_rpc_limit = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT; + ret = rpcsvc_set_outstanding_rpc_limit (svc, options); +out: + return ret; +} - if (dict_get (options, "rpc.outstanding-rpc-limit")) { - ret = dict_get_str (options, "rpc.oustanding-rpc-limit", - &optstr); - if (ret < 0) { - gf_log (GF_RPCSVC, GF_LOG_ERROR, "Value went missing"); - goto out; +int +rpcsvc_reconfigure_options (rpcsvc_t *svc, dict_t *options) +{ + xlator_t *xlator = NULL; + xlator_list_t *volentry = NULL; + char *srchkey = NULL; + char *keyval = NULL; + int ret = -1; + + if ((!svc) || (!svc->options) || (!options)) + return (-1); + + /* Fetch the xlator from svc */ + xlator = (xlator_t *) svc->mydata; + if (!xlator) + return (-1); + + /* Reconfigure the volume specific rpc-auth.addr allow part */ + volentry = xlator->children; + while (volentry) { + ret = gf_asprintf (&srchkey, "rpc-auth.addr.%s.allow", + volentry->xlator->name); + if (ret == -1) { + gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed"); + return (-1); } - ret = gf_string2int32 (optstr, &svc->outstanding_rpc_limit); - if (ret < 0) { - gf_log (GF_RPCSVC, GF_LOG_ERROR, "Invalid RPC limit %s", - optstr); - goto out; + /* If found the srchkey, delete old key/val pair + * and set the key with new value. + */ + if (!dict_get_str (options, srchkey, &keyval)) { + dict_del (svc->options, srchkey); + ret = dict_set_str (svc->options, srchkey, keyval); + if (ret < 0) { + gf_log (GF_RPCSVC, GF_LOG_ERROR, + "dict_set_str error"); + GF_FREE (srchkey); + return (-1); + } } + + GF_FREE (srchkey); + volentry = volentry->next; } - ret = 0; -out: - return ret; + /* Reconfigure the volume specific rpc-auth.addr reject part */ + volentry = xlator->children; + while (volentry) { + ret = gf_asprintf (&srchkey, "rpc-auth.addr.%s.reject", + volentry->xlator->name); + if (ret == -1) { + gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed"); + return (-1); + } + + /* If found the srchkey, delete old key/val pair + * and set the key with new value. + */ + if (!dict_get_str (options, srchkey, &keyval)) { + dict_del (svc->options, srchkey); + ret = dict_set_str (svc->options, srchkey, keyval); + if (ret < 0) { + gf_log (GF_RPCSVC, GF_LOG_ERROR, + "dict_set_str error"); + GF_FREE (srchkey); + return (-1); + } + } + + GF_FREE (srchkey); + volentry = volentry->next; + } + + ret = rpcsvc_init_options (svc, options); + if (ret) + return (-1); + + return rpcsvc_auth_reconf (svc, options); } int @@ -1958,6 +2021,48 @@ out: return ret; } +/* + * Reconfigure() the rpc.outstanding-rpc-limit param. + */ +int +rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options) +{ + int ret = -1; /* FAILURE */ + int rpclim = 0; + static char *rpclimkey = "rpc.outstanding-rpc-limit"; + + if ((!svc) || (!options)) + return (-1); + + /* Reconfigure() the rpc.outstanding-rpc-limit param */ + ret = dict_get_int32 (options, rpclimkey, &rpclim); + if (ret < 0) { + /* Fall back to default for FAILURE */ + rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT; + } else { + /* SUCCESS: round off to multiple of 8. + * If the input value fails Boundary check, fall back to + * default i.e. RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT. + * NB: value 0 is special, means its unset i.e. unlimited. + */ + rpclim = ((rpclim + 8 - 1) >> 3) * 8; + if (rpclim < RPCSVC_MIN_OUTSTANDING_RPC_LIMIT) { + rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT; + } else if (rpclim > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT) { + rpclim = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT; + } + } + + if (svc->outstanding_rpc_limit != rpclim) { + svc->outstanding_rpc_limit = rpclim; + gf_log (GF_RPCSVC, GF_LOG_INFO, + "Configured %s with value %d", + rpclimkey, rpclim); + } + + return (0); +} + /* The global RPC service initializer. */ rpcsvc_t * @@ -2089,7 +2194,7 @@ err: } -int +static int rpcsvc_transport_peer_check_allow (dict_t *options, char *volname, char *ip, char *hostname) { @@ -2118,7 +2223,7 @@ out: return ret; } -int +static int rpcsvc_transport_peer_check_reject (dict_t *options, char *volname, char *ip, char *hostname) { @@ -2170,7 +2275,7 @@ rpcsvc_combine_allow_reject_volume_check (int allow, int reject) } int -rpcsvc_auth_check (dict_t *options, char *volname, +rpcsvc_auth_check (rpcsvc_t *svc, char *volname, rpc_transport_t *trans) { int ret = RPCSVC_AUTH_REJECT; @@ -2182,8 +2287,14 @@ rpcsvc_auth_check (dict_t *options, char *volname, char *allow_str = NULL; char *reject_str = NULL; char *srchstr = NULL; + dict_t *options = NULL; + + if (!svc || !volname || !trans) + return ret; - if (!options || !volname || !trans) + /* Fetch the options from svc struct and validate */ + options = svc->options; + if (!options) return ret; ret = rpcsvc_transport_peername (trans, client_ip, RPCSVC_PEER_STRLEN); @@ -2225,9 +2336,8 @@ rpcsvc_auth_check (dict_t *options, char *volname, if (!get_host_name (client_ip, &ip)) ip = client_ip; - /* addr-namelookup disabled by default */ - ret = dict_get_str_boolean (options, "rpc-auth.addr.namelookup", 0); - if (ret == _gf_true) { + /* addr-namelookup check */ + if (svc->addr_namelookup == _gf_true) { ret = gf_get_hostname_from_ip (ip, &hostname); if (ret) { if (hostname) diff --git a/rpc/rpc-lib/src/rpcsvc.h b/rpc/rpc-lib/src/rpcsvc.h index ac2f09bea..cbc1f4226 100644 --- a/rpc/rpc-lib/src/rpcsvc.h +++ b/rpc/rpc-lib/src/rpcsvc.h @@ -40,6 +40,7 @@ #define RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT 64 #define RPCSVC_MAX_OUTSTANDING_RPC_LIMIT 65536 +#define RPCSVC_MIN_OUTSTANDING_RPC_LIMIT 0 /* No limit i.e. Unlimited */ #define GF_RPCSVC "rpc-service" #define RPCSVC_THREAD_STACK_SIZE ((size_t)(1024 * GF_UNIT_KB)) @@ -439,6 +440,9 @@ rpcsvc_listener_destroy (rpcsvc_listener_t *listener); extern int rpcsvc_program_register_portmap (rpcsvc_program_t *newprog, uint32_t port); +extern int +rpcsvc_program_unregister_portmap (rpcsvc_program_t *newprog); + extern int rpcsvc_register_portmap_enabled (rpcsvc_t *svc); @@ -449,6 +453,9 @@ extern rpcsvc_t * rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options, uint32_t poolcount); +extern int +rpcsvc_reconfigure_options (rpcsvc_t *svc, dict_t *options); + int rpcsvc_register_notify (rpcsvc_t *svc, rpcsvc_notify_t notify, void *mydata); @@ -491,8 +498,7 @@ rpcsvc_transport_peeraddr (rpc_transport_t *trans, char *addrstr, int addrlen, struct sockaddr_storage *returnsa, socklen_t sasize); extern int -rpcsvc_auth_check (dict_t *options, char *volname, - rpc_transport_t *trans); +rpcsvc_auth_check (rpcsvc_t *svc, char *volname, rpc_transport_t *trans); extern int rpcsvc_transport_privport_check (rpcsvc_t *svc, char *volname, @@ -552,6 +558,9 @@ rpcsvc_auth_request_init (rpcsvc_request_t *req); extern int rpcsvc_auth_init (rpcsvc_t *svc, dict_t *options); +extern int +rpcsvc_auth_reconf (rpcsvc_t *svc, dict_t *options); + extern int rpcsvc_auth_transport_init (rpc_transport_t *xprt); @@ -584,11 +593,13 @@ rpcsvc_transport_unix_options_build (dict_t **options, char *filepath); int rpcsvc_set_allow_insecure (rpcsvc_t *svc, dict_t *options); int +rpcsvc_set_addr_namelookup (rpcsvc_t *svc, dict_t *options); +int rpcsvc_set_root_squash (rpcsvc_t *svc, dict_t *options); int +rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options); +int rpcsvc_auth_array (rpcsvc_t *svc, char *volname, int *autharr, int arrlen); -char * -rpcsvc_volume_allowed (dict_t *options, char *volname); rpcsvc_vector_sizer rpcsvc_get_program_vector_sizer (rpcsvc_t *svc, uint32_t prognum, uint32_t progver, uint32_t procnum); diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index f0445cf0b..515c23a8c 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -3834,6 +3834,11 @@ glusterd_reconfigure_nfs () int ret = -1; gf_boolean_t identical = _gf_false; + /* + * Check both OLD and NEW volfiles, if they are SAME by size + * and cksum i.e. "character-by-character". If YES, then + * NOTHING has been changed, just return. + */ ret = glusterd_check_nfs_volfile_identical (&identical); if (ret) goto out; @@ -3843,6 +3848,31 @@ glusterd_reconfigure_nfs () goto out; } + /* + * They are not identical. Find out if the topology is changed + * OR just the volume options. If just the options which got + * changed, then inform the xlator to reconfigure the options. + */ + identical = _gf_false; /* RESET the FLAG */ + ret = glusterd_check_nfs_topology_identical (&identical); + if (ret) + goto out; + + /* Topology is not changed, but just the options. But write the + * options to NFS volfile, so that NFS will be reconfigured. + */ + if (identical) { + ret = glusterd_create_nfs_volfile(); + if (ret == 0) {/* Only if above PASSES */ + ret = glusterd_fetchspec_notify (THIS); + } + goto out; + } + + /* + * NFS volfile's topology has been changed. NFS server needs + * to be RESTARTED to ACT on the changed volfile. + */ ret = glusterd_check_generate_start_nfs (); out: @@ -6364,6 +6394,68 @@ glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo, return ret; } +int +glusterd_check_topology_identical (const char *filename1, + const char *filename2, + gf_boolean_t *identical) +{ + int ret = -1; /* FAILURE */ + xlator_t *this = NULL; + FILE *fp1 = NULL; + FILE *fp2 = NULL; + glusterfs_graph_t *grph1 = NULL; + glusterfs_graph_t *grph2 = NULL; + + if ((!filename1) || (!filename2) || (!identical)) + goto out; + + this = THIS; + + errno = 0; /* RESET the errno */ + + /* fopen() the volfile1 to create the graph */ + fp1 = fopen (filename1, "r"); + if (fp1 == NULL) { + gf_log (this->name, GF_LOG_ERROR, "fopen() on file: %s failed " + "(%s)", filename1, strerror (errno)); + goto out; + } + + /* fopen() the volfile2 to create the graph */ + fp2 = fopen (filename2, "r"); + if (fp2 == NULL) { + gf_log (this->name, GF_LOG_ERROR, "fopen() on file: %s failed " + "(%s)", filename2, strerror (errno)); + goto out; + } + + /* create the graph for filename1 */ + grph1 = glusterfs_graph_construct(fp1); + if (grph1 == NULL) + goto out; + + /* create the graph for filename2 */ + grph2 = glusterfs_graph_construct(fp2); + if (grph2 == NULL) + goto out; + + /* compare the graph topology */ + *identical = is_graph_topology_equal(grph1, grph2); + ret = 0; /* SUCCESS */ +out: + if (fp1) + fclose(fp1); + if (fp2) + fclose(fp2); + if (grph1) + glusterfs_graph_destroy(grph1); + if (grph2) + glusterfs_graph_destroy(grph2); + + gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret); + return ret; +} + int glusterd_check_files_identical (char *filename1, char *filename2, gf_boolean_t *identical) diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h index 024675c7f..57120fe39 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.h +++ b/xlators/mgmt/glusterd/src/glusterd-utils.h @@ -456,6 +456,12 @@ glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo, int glusterd_check_files_identical (char *filename1, char *filename2, gf_boolean_t *identical); + +int +glusterd_check_topology_identical (const char *filename1, + const char *filename2, + gf_boolean_t *identical); + void glusterd_volinfo_reset_defrag_stats (glusterd_volinfo_t *volinfo); int diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c index 6af52abe2..0de71a49a 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.c +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c @@ -3439,6 +3439,54 @@ out: return ret; } +int +glusterd_check_nfs_topology_identical (gf_boolean_t *identical) +{ + char nfsvol[PATH_MAX] = {0,}; + char tmpnfsvol[PATH_MAX] = {0,}; + glusterd_conf_t *conf = NULL; + xlator_t *this = THIS; + int ret = -1; + int tmpclean = 0; + int tmpfd = -1; + + if ((!identical) || (!this) || (!this->private)) + goto out; + + conf = (glusterd_conf_t *) this->private; + + /* Fetch the original NFS volfile */ + glusterd_get_nodesvc_volfile ("nfs", conf->workdir, + nfsvol, sizeof (nfsvol)); + + /* Create the temporary NFS volfile */ + snprintf (tmpnfsvol, sizeof (tmpnfsvol), "/tmp/gnfs-XXXXXX"); + tmpfd = mkstemp (tmpnfsvol); + if (tmpfd < 0) { + gf_log (this->name, GF_LOG_WARNING, + "Unable to create temp file %s: (%s)", + tmpnfsvol, strerror (errno)); + goto out; + } + + tmpclean = 1; /* SET the flag to unlink() tmpfile */ + + ret = glusterd_create_global_volfile (build_nfs_graph, + tmpnfsvol, NULL); + if (ret) + goto out; + + /* Compare the topology of volfiles */ + ret = glusterd_check_topology_identical (nfsvol, tmpnfsvol, + identical); +out: + if (tmpfd >= 0) + close (tmpfd); + if (tmpclean) + unlink (tmpnfsvol); + return ret; +} + int glusterd_check_nfs_volfile_identical (gf_boolean_t *identical) { diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h index 31bfe980d..2d31c4040 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.h +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h @@ -144,6 +144,8 @@ char* glusterd_get_trans_type_rb (gf_transport_type ttype); int glusterd_check_nfs_volfile_identical (gf_boolean_t *identical); +int +glusterd_check_nfs_topology_identical (gf_boolean_t *identical); uint32_t glusterd_get_op_version_for_key (char *key); diff --git a/xlators/nfs/server/src/acl3.c b/xlators/nfs/server/src/acl3.c index e10123e69..08b099b4e 100644 --- a/xlators/nfs/server/src/acl3.c +++ b/xlators/nfs/server/src/acl3.c @@ -248,7 +248,7 @@ acl3_getacl_cbk (call_frame_t *frame, void *cookie, xlator_t *this, cs = frame->local; getaclreply = &cs->args.getaclreply; if (op_ret == -1) { - stat = nfs3_errno_to_nfsstat3 (op_errno); + stat = nfs3_cbk_errno_status (op_ret, op_errno); goto err; } @@ -316,7 +316,7 @@ acl3_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this, getaclreply = &cs->args.getaclreply; if (op_ret == -1) { - stat = nfs3_errno_to_nfsstat3 (op_errno); + stat = nfs3_cbk_errno_status (op_ret, op_errno); goto err; } @@ -327,7 +327,7 @@ acl3_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this, ret = nfs_getxattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, NULL, NULL, acl3_getacl_cbk, cs); if (ret == -1) { - stat = nfs3_errno_to_nfsstat3 (op_errno); + stat = nfs3_cbk_errno_status (op_ret, op_errno); goto err; } return 0; @@ -429,7 +429,8 @@ acl3_setacl_cbk (call_frame_t *frame, void *cookie, nfs3_call_state_t *cs = NULL; cs = frame->local; if (op_ret < 0) { - cs->args.setaclreply.status = nfs3_errno_to_nfsstat3 (op_errno); + nfsstat3 status = nfs3_cbk_errno_status (op_ret, op_errno); + cs->args.setaclreply.status = status; } acl3svc_submit_reply (cs->req, (void *)&cs->args.setaclreply, @@ -643,6 +644,11 @@ acl3svc_init(xlator_t *nfsx) dict_t *options = NULL; int ret = -1; char *portstr = NULL; + static gf_boolean_t acl3_inited = _gf_false; + + /* Already inited */ + if (acl3_inited) + return &acl3prog; nfs = (struct nfs_state*)nfsx->private; @@ -695,6 +701,7 @@ acl3svc_init(xlator_t *nfsx) goto err; } + acl3_inited = _gf_true; return &acl3prog; err: return NULL; diff --git a/xlators/nfs/server/src/mount3.c b/xlators/nfs/server/src/mount3.c index 449bbc149..b0824bf10 100644 --- a/xlators/nfs/server/src/mount3.c +++ b/xlators/nfs/server/src/mount3.c @@ -74,6 +74,17 @@ typedef ssize_t (*mnt3_serializer) (struct iovec outmsg, void *args); extern void * mount3udp_thread (void *argv); +static inline void +mnt3_export_free (struct mnt3_export *exp) +{ + if (!exp) + return; + + if (exp->exptype == MNT3_EXPTYPE_DIR) + FREE_HOSTSPEC (exp); + GF_FREE (exp->expname); + GF_FREE (exp); +} /* Generic reply function for MOUNTv3 specific replies. */ int @@ -580,6 +591,7 @@ __mnt3_get_volume_id (struct mount3_state *ms, xlator_t *mntxl, if ((!ms) || (!mntxl)) return ret; + LOCK (&ms->mountlock); list_for_each_entry (exp, &ms->exportlist, explist) { if (exp->vol == mntxl) { uuid_copy (volumeid, exp->volumeid); @@ -589,6 +601,7 @@ __mnt3_get_volume_id (struct mount3_state *ms, xlator_t *mntxl, } out: + UNLOCK (&ms->mountlock); return ret; } @@ -1235,6 +1248,7 @@ mnt3_mntpath_to_export (struct mount3_state *ms, char *dirpath) if ((!ms) || (!dirpath)) return NULL; + LOCK (&ms->mountlock); list_for_each_entry (exp, &ms->exportlist, explist) { /* Search for the an exact match with the volume */ @@ -1248,6 +1262,7 @@ mnt3_mntpath_to_export (struct mount3_state *ms, char *dirpath) gf_log (GF_MNT, GF_LOG_DEBUG, "Export not found"); foundexp: + UNLOCK (&ms->mountlock); return found; } @@ -1276,7 +1291,7 @@ mnt3_check_client_net (struct mount3_state *ms, rpcsvc_request_t *req, gai_strerror (ret)); } - ret = rpcsvc_auth_check (svc->options, targetxl->name, trans); + ret = rpcsvc_auth_check (svc, targetxl->name, trans); if (ret == RPCSVC_AUTH_REJECT) { gf_log (GF_MNT, GF_LOG_INFO, "Peer %s not allowed", peer); goto err; @@ -1804,6 +1819,10 @@ mnt3_xlchildren_to_exports (rpcsvc_t *svc, struct mount3_state *ms) return NULL; nfs = (struct nfs_state *)ms->nfsx->private; + if (!nfs) + return NULL; + + LOCK (&ms->mountlock); list_for_each_entry(ent, &ms->exportlist, explist) { /* If volume is not started yet, do not list it for tools like @@ -1861,6 +1880,7 @@ mnt3_xlchildren_to_exports (rpcsvc_t *svc, struct mount3_state *ms) ret = 0; free_list: + UNLOCK (&ms->mountlock); if (ret == -1) { xdr_free_exports_list (first); first = NULL; @@ -2193,8 +2213,9 @@ mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath, ret = snprintf (exp->expname, alloclen, "/%s", xl->name); } if (ret < 0) { - gf_log (xl->name, GF_LOG_WARNING, - "failed to get the export name"); + gf_log (xl->name, GF_LOG_ERROR, + "Failed to set the export name"); + goto err; } /* Just copy without discrimination, we'll determine whether to * actually use it when a mount request comes in and a file handle @@ -2208,8 +2229,7 @@ mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath, err: /* On failure free exp and it's members.*/ if (NULL != exp) { - FREE_HOSTSPEC (exp); - GF_FREE (exp); + mnt3_export_free (exp); exp = NULL; } @@ -2696,3 +2716,44 @@ mnt1svc_init (xlator_t *nfsx) err: return NULL; } + +int +mount_reconfigure_state (xlator_t *nfsx, dict_t *options) +{ + int ret = -1; + struct nfs_state *nfs = NULL; + struct mount3_state *ms = NULL; + struct mnt3_export *exp = NULL; + struct mnt3_export *texp = NULL; + + if ((!nfsx) || (!options)) + return (-1); + + nfs = (struct nfs_state *)nfs_state (nfsx); + if (!nfs) + return (-1); + + ms = nfs->mstate; + if (!ms) + return (-1); + + /* + * Free() up the old export list. mnt3_init_options() will + * rebuild the export list from scratch. Do it with locking + * to avoid unnecessary race conditions. + */ + LOCK (&ms->mountlock); + list_for_each_entry_safe (exp, texp, &ms->exportlist, explist) { + list_del (&exp->explist); + mnt3_export_free (exp); + } + ret = mnt3_init_options (ms, options); + UNLOCK (&ms->mountlock); + + if (ret < 0) { + gf_log (GF_MNT, GF_LOG_ERROR, "Options reconfigure failed"); + return (-1); + } + + return (0); +} diff --git a/xlators/nfs/server/src/mount3.h b/xlators/nfs/server/src/mount3.h index 870a7570f..7fc16ed57 100644 --- a/xlators/nfs/server/src/mount3.h +++ b/xlators/nfs/server/src/mount3.h @@ -44,6 +44,9 @@ mnt1svc_init (xlator_t *nfsx); extern int mount_init_state (xlator_t *nfsx); +extern int +mount_reconfigure_state (xlator_t *nfsx, dict_t *options); + void mount_rewrite_rmtab (struct mount3_state *ms, char *new_rmtab); @@ -107,8 +110,8 @@ struct mount3_state { gf_lock_t mountlock; /* Set to 0 if exporting full volumes is disabled. On by default. */ - int export_volumes; - int export_dirs; + gf_boolean_t export_volumes; + gf_boolean_t export_dirs; }; #define gf_mnt3_export_dirs(mst) ((mst)->export_dirs) diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c index 33bd36e74..75c8fe44e 100644 --- a/xlators/nfs/server/src/nfs.c +++ b/xlators/nfs/server/src/nfs.c @@ -46,6 +46,194 @@ #define DATADIR "/var/lib/glusterd" #define NFS_DATADIR DATADIR "/nfs" +/* Forward declaration */ +int nfs_add_initer (struct list_head *list, nfs_version_initer_t init); + +static int +nfs_init_version (xlator_t *this, nfs_version_initer_t init) +{ + int ret = -1; + struct nfs_initer_list *version = NULL; + struct nfs_initer_list *tmp = NULL; + rpcsvc_program_t *prog = NULL; + struct list_head *versions = NULL; + struct nfs_state *nfs = NULL; + gf_boolean_t found = _gf_false; + + if ((!this) || (!this->private) || (!init)) + return (-1); + + nfs = (struct nfs_state *)this->private; + + ret = nfs_add_initer (&nfs->versions, init); + if (ret == -1) { + gf_log (GF_NFS, GF_LOG_ERROR, + "Failed to add protocol initializer"); + goto err; + } + + versions = &nfs->versions; + list_for_each_entry_safe (version, tmp, versions, list) { + prog = version->program; + if (version->init == init) { + prog = init(this); + if (!prog) { + ret = -1; + goto err; + } + version->program = prog; + found = _gf_true; + break; + } + } + + /* program not added */ + if (!found) { + gf_log (GF_NFS, GF_LOG_ERROR, + "Program: %s NOT found", prog->progname); + goto err; + } + + /* Check if nfs.port is configured */ + if (nfs->override_portnum) + prog->progport = nfs->override_portnum; + + gf_log (GF_NFS, GF_LOG_DEBUG, "Starting program: %s", prog->progname); + + ret = rpcsvc_program_register (nfs->rpcsvc, prog); + if (ret == -1) { + gf_log (GF_NFS, GF_LOG_ERROR, "Program: %s init failed", + prog->progname); + goto err; + } + + /* Registration with portmapper is disabled, Nothing to do */ + if (!nfs->register_portmap) + goto err; + + ret = rpcsvc_program_register_portmap (prog, prog->progport); + if (ret == -1) { + gf_log (GF_NFS, GF_LOG_ERROR, + "Program %s registration failed", + prog->progname); + goto err; + } + ret = 0; /* All well */ +err: + return ret; +} + +static int +nfs_deinit_version (struct nfs_state *nfs, nfs_version_initer_t init) +{ + int ret = -1; + struct nfs_initer_list *version = NULL; + struct nfs_initer_list *tmp = NULL; + rpcsvc_program_t *prog = NULL; + struct list_head *versions = NULL; + + if ((!nfs) || (!init)) + return (-1); + + versions = &nfs->versions; + list_for_each_entry_safe (version, tmp, versions, list) { + prog = version->program; + if (version->init == init) { + prog = version->program; + ret = rpcsvc_program_unregister (nfs->rpcsvc, prog); + if (ret != 0) + return (-1); + list_del (&version->list); + GF_FREE (version); + return (0); + } + } + + return (-1); +} + +static int +nfs_reconfigure_acl3 (xlator_t *this) +{ + struct nfs_state *nfs = NULL; + + if ((!this) || (!this->private)) + return (-1); + + nfs = (struct nfs_state *)this->private; + + /* ACL is enabled */ + if (nfs->enable_acl) + return nfs_init_version (this, acl3svc_init); + + /* ACL is disabled */ + return nfs_deinit_version (nfs, acl3svc_init); +} + +static int +nfs_reconfigure_nlm4 (xlator_t *this) +{ + struct nfs_state *nfs = NULL; + + if ((!this) || (!this->private)) + return (-1); + + nfs = (struct nfs_state *)this->private; + + /* NLM is enabled */ + if (nfs->enable_nlm) + return nfs_init_version (this, nlm4svc_init); + + /* NLM is disabled */ + return nfs_deinit_version (nfs, nlm4svc_init); +} + +static int +nfs_program_register_portmap_all (struct nfs_state *nfs) +{ + struct list_head *versions = NULL; + struct nfs_initer_list *version = NULL; + struct nfs_initer_list *tmp = NULL; + rpcsvc_program_t *prog = NULL; + + if (nfs == NULL) + return (-1); + + versions = &nfs->versions; + list_for_each_entry_safe (version, tmp, versions, list) { + prog = version->program; + if (prog == NULL) + continue; + if (nfs->override_portnum) + prog->progport = nfs->override_portnum; + (void) rpcsvc_program_register_portmap (prog, prog->progport); + } + + return (0); +} + +static int +nfs_program_unregister_portmap_all (struct nfs_state *nfs) +{ + struct list_head *versions = NULL; + struct nfs_initer_list *version = NULL; + struct nfs_initer_list *tmp = NULL; + rpcsvc_program_t *prog = NULL; + + if (nfs == NULL) + return (-1); + + versions = &nfs->versions; + list_for_each_entry_safe (version, tmp, versions, list) { + prog = version->program; + if (prog == NULL) + continue; + (void) rpcsvc_program_unregister_portmap (prog); + } + + return (0); +} + /* Every NFS version must call this function with the init function * for its particular version. */ @@ -122,7 +310,7 @@ nfs_init_versions (struct nfs_state *nfs, xlator_t *this) ret = -1; goto err; } -// prog->actorxl = this; + version->program = prog; if (nfs->override_portnum) prog->progport = nfs->override_portnum; @@ -131,17 +319,21 @@ nfs_init_versions (struct nfs_state *nfs, xlator_t *this) ret = rpcsvc_program_register (nfs->rpcsvc, prog); if (ret == -1) { - gf_log (GF_NFS, GF_LOG_ERROR, "Program init failed"); + gf_log (GF_NFS, GF_LOG_ERROR, "Program: %s init failed", + prog->progname); goto err; } - if (rpcsvc_register_portmap_enabled(nfs->rpcsvc)) { + if (nfs->register_portmap) { ret = rpcsvc_program_register_portmap (prog, prog->progport); if (ret == -1) { - gf_log (GF_NFS, GF_LOG_ERROR, "Program registration failed"); + gf_log (GF_NFS, GF_LOG_ERROR, + "Program %s registration failed", + prog->progname); goto err; } } + } ret = 0; @@ -158,22 +350,22 @@ nfs_add_all_initiators (struct nfs_state *nfs) /* Add the initializers for all versions. */ ret = nfs_add_initer (&nfs->versions, mnt3svc_init); if (ret == -1) { - gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol" - " initializer"); + gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add " + "MOUNT3 protocol initializer"); goto ret; } ret = nfs_add_initer (&nfs->versions, mnt1svc_init); if (ret == -1) { - gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol" - " initializer"); + gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add " + "MOUNT1 protocol initializer"); goto ret; } ret = nfs_add_initer (&nfs->versions, nfs3svc_init); if (ret == -1) { - gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol" - " initializer"); + gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add " + "NFS3 protocol initializer"); goto ret; } @@ -588,19 +780,10 @@ nfs_init_state (xlator_t *this) } nfs->enable_nlm = _gf_true; - if (!dict_get_str (this->options, "nfs.nlm", &optstr)) { - - ret = gf_string2boolean (optstr, &boolt); - if (ret < 0) { - gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse" - " bool string"); - goto free_foppool; - } - - if (boolt == _gf_false) { - gf_log (GF_NFS, GF_LOG_INFO, "NLM is manually disabled"); - nfs->enable_nlm = _gf_false; - } + ret = dict_get_str_boolean (this->options, "nfs.nlm", _gf_true); + if (ret == _gf_false) { + gf_log (GF_NFS, GF_LOG_INFO, "NLM is manually disabled"); + nfs->enable_nlm = _gf_false; } nfs->enable_acl = _gf_true; @@ -691,7 +874,7 @@ nfs_init_state (xlator_t *this) nfs->mount_udp = 1; } - nfs->rmtab = NFS_DATADIR "/rmtab"; + nfs->rmtab = gf_strdup (NFS_DATADIR "/rmtab"); if (dict_get(this->options, "nfs.mount-rmtab")) { ret = dict_get_str (this->options, "nfs.mount-rmtab", &nfs->rmtab); if (ret == -1) { @@ -783,6 +966,8 @@ nfs_init_state (xlator_t *this) goto free_foppool; } + nfs->register_portmap = rpcsvc_register_portmap_enabled (nfs->rpcsvc); + this->private = (void *)nfs; INIT_LIST_HEAD (&nfs->versions); nfs->generation = 1965; @@ -820,48 +1005,227 @@ nfs_drc_init (xlator_t *this) return ret; } +int +nfs_reconfigure_state (xlator_t *this, dict_t *options) +{ + int ret = 0; + int keyindx = 0; + char *optstr = NULL; + gf_boolean_t optbool; + uint32_t optuint32; + struct nfs_state *nfs = NULL; + char *blacklist_keys[] = { + "nfs.port", + "nfs.transport-type", + "nfs.mem-factor", + NULL}; + + GF_VALIDATE_OR_GOTO (GF_NFS, this, out); + GF_VALIDATE_OR_GOTO (GF_NFS, this->private, out); + GF_VALIDATE_OR_GOTO (GF_NFS, options, out); + + nfs = (struct nfs_state *)this->private; + + /* Black listed options can't be reconfigured, they need + * NFS to be restarted. There are two cases 1. SET 2. UNSET. + * 1. SET */ + while (blacklist_keys[keyindx]) { + if (dict_get (options, blacklist_keys[keyindx])) { + gf_log (GF_NFS, GF_LOG_ERROR, + "Reconfiguring %s needs NFS restart", + blacklist_keys[keyindx]); + goto out; + } + keyindx ++; + } + + /* UNSET for nfs.mem-factor */ + if ((!dict_get (options, "nfs.mem-factor")) && + (nfs->memfactor != GF_NFS_DEFAULT_MEMFACTOR)) { + gf_log (GF_NFS, GF_LOG_INFO, + "Reconfiguring nfs.mem-factor needs NFS restart"); + goto out; + } + + /* UNSET for nfs.port */ + if ((!dict_get (options, "nfs.port")) && + (nfs->override_portnum)) { + gf_log (GF_NFS, GF_LOG_ERROR, + "Reconfiguring nfs.port needs NFS restart"); + goto out; + } + + /* reconfig nfs.mount-rmtab */ + optstr = NFS_DATADIR "/rmtab"; + if (dict_get (options, "nfs.mount-rmtab")) { + ret = dict_get_str (options, "nfs.mount-rmtab", &optstr); + if (ret < 0) { + gf_log (GF_NFS, GF_LOG_ERROR, "Failed to read " + "reconfigured option: nfs.mount-rmtab"); + goto out; + } + gf_path_strip_trailing_slashes (optstr); + } + if (strcmp (nfs->rmtab, optstr) != 0) { + mount_rewrite_rmtab (nfs->mstate, optstr); + gf_log (GF_NFS, GF_LOG_INFO, + "Reconfigured nfs.mount-rmtab path: %s", + nfs->rmtab); + } + + GF_OPTION_RECONF (OPT_SERVER_AUX_GIDS, optbool, + options, bool, out); + if (nfs->server_aux_gids != optbool) { + nfs->server_aux_gids = optbool; + gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured %s with value %d", + OPT_SERVER_AUX_GIDS, optbool); + } + + GF_OPTION_RECONF (OPT_SERVER_GID_CACHE_TIMEOUT, optuint32, + options, uint32, out); + if (nfs->server_aux_gids_max_age != optuint32) { + nfs->server_aux_gids_max_age = optuint32; + gid_cache_reconf (&nfs->gid_cache, optuint32); + gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured %s with value %d", + OPT_SERVER_GID_CACHE_TIMEOUT, optuint32); + } + + /* reconfig nfs.dynamic-volumes */ + ret = dict_get_str_boolean (options, "nfs.dynamic-volumes", + GF_NFS_DVM_OFF); + switch (ret) { + case GF_NFS_DVM_ON: + case GF_NFS_DVM_OFF: + optbool = ret; + break; + default: + optbool = GF_NFS_DVM_OFF; + break; + } + if (nfs->dynamicvolumes != optbool) { + nfs->dynamicvolumes = optbool; + gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured nfs.dynamic-volumes" + " with value %d", optbool); + } + + optbool = _gf_false; + if (dict_get (options, "nfs.enable-ino32")) { + ret = dict_get_str_boolean (options, "nfs.enable-ino32", + _gf_false); + if (ret < 0) { + gf_log (GF_NFS, GF_LOG_ERROR, + "Failed to read reconfigured option: " + "nfs.enable-ino32"); + goto out; + } + optbool = ret; + } + if (nfs->enable_ino32 != optbool) { + nfs->enable_ino32 = optbool; + gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured nfs.enable-ino32" + " with value %d", optbool); + } + + /* nfs.nlm is enabled by default */ + ret = dict_get_str_boolean (options, "nfs.nlm", _gf_true); + if (ret < 0) { + optbool = _gf_true; + } else { + optbool = ret; + } + if (nfs->enable_nlm != optbool) { + gf_log (GF_NFS, GF_LOG_INFO, "NLM is manually %s", + (optbool ? "enabled":"disabled")); + nfs->enable_nlm = optbool; + nfs_reconfigure_nlm4 (this); + } + + /* nfs.acl is enabled by default */ + ret = dict_get_str_boolean (options, "nfs.acl", _gf_true); + if (ret < 0) { + optbool = _gf_true; + } else { + optbool = ret; + } + if (nfs->enable_acl != optbool) { + gf_log (GF_NFS, GF_LOG_INFO, "ACL is manually %s", + (optbool ? "enabled":"disabled")); + nfs->enable_acl = optbool; + nfs_reconfigure_acl3 (this); + } + + ret = 0; +out: + return ret; +} + -#if 0 -/* reconfigure() is currently not used for the NFS-server. Upon setting an - * option for the NFS-xlator, the glusterfs process it restarted. - * - * This current implementation makes sure to read the currently used rmtab and - * merge it with the new rmtab. - * - * As this function is never called, it is provided for the future, for when - * the NFS-server supports reloading. +/* + * reconfigure() for NFS server xlator. */ int reconfigure (xlator_t *this, dict_t *options) { - int ret = 0; - char *rmtab = NULL; + int ret = 0; struct nfs_state *nfs = NULL; + gf_boolean_t regpmap = _gf_true; + + if ((!this) || (!this->private) || (!options)) + return (-1); nfs = (struct nfs_state *)this->private; - if (!nfs) { - gf_log_callingfn (this->name, GF_LOG_DEBUG, "conf == null!!!"); - goto out; + /* Reconfigure nfs options */ + ret = nfs_reconfigure_state(this, options); + if (ret) { + gf_log (GF_NFS, GF_LOG_ERROR, + "nfs reconfigure state failed"); + return (-1); } - ret = dict_get_str (options, "nfs.mount-rmtab", &rmtab); + /* Reconfigure nfs3 options */ + ret = nfs3_reconfigure_state(this, options); if (ret) { - goto out; + gf_log (GF_NFS, GF_LOG_ERROR, + "nfs3 reconfigure state failed"); + return (-1); } - gf_path_strip_trailing_slashes (rmtab); - if (strcmp (nfs->rmtab, rmtab) != 0) { - mount_rewrite_rmtab (nfs->mstate, rmtab); + /* Reconfigure mount options */ + ret = mount_reconfigure_state(this, options); + if (ret) { + gf_log (GF_NFS, GF_LOG_ERROR, + "mount reconfigure state failed"); + return (-1); + } - gf_log (this->name, GF_LOG_INFO, - "Reconfigured nfs.mount-rmtab path: %s", nfs->rmtab); + /* Reconfigure rpc layer */ + ret = rpcsvc_reconfigure_options (nfs->rpcsvc, options); + if (ret) { + gf_log (GF_NFS, GF_LOG_ERROR, + "rpcsvc reconfigure options failed"); + return (-1); + } + regpmap = rpcsvc_register_portmap_enabled(nfs->rpcsvc); + if (nfs->register_portmap != regpmap) { + nfs->register_portmap = regpmap; + if (regpmap) { + (void) nfs_program_register_portmap_all (nfs); + } else { + (void) nfs_program_unregister_portmap_all (nfs); + } } -out: - return ret; + /* Reconfigure drc */ + ret = rpcsvc_drc_reconfigure (nfs->rpcsvc, options); + if (ret) { + gf_log (GF_NFS, GF_LOG_ERROR, + "rpcsvc DRC reconfigure failed"); + return (-1); + } + + return (0); } -#endif /* glusterfs/nfs is restarted and reconfigure() is never called */ int init (xlator_t *this) { @@ -1361,7 +1725,7 @@ struct volume_options options[] = { }, { .key = {"rpc.outstanding-rpc-limit"}, .type = GF_OPTION_TYPE_INT, - .min = 0, + .min = RPCSVC_MIN_OUTSTANDING_RPC_LIMIT, .max = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT, .default_value = TOSTRING(RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT), .description = "Parameter to throttle the number of incoming RPC " @@ -1446,8 +1810,8 @@ struct volume_options options[] = { }, { .key = {"nfs.drc"}, .type = GF_OPTION_TYPE_STR, - .default_value = "off", - .description = "Enable Duplicate Request Cache in gNfs server to " + .default_value = "on", + .description = "Enable Duplicate Request Cache in gNFS server to " "improve correctness of non-idempotent operations like " "write, delete, link, et al" }, diff --git a/xlators/nfs/server/src/nfs.h b/xlators/nfs/server/src/nfs.h index 39342f63a..00c7f8046 100644 --- a/xlators/nfs/server/src/nfs.h +++ b/xlators/nfs/server/src/nfs.h @@ -36,7 +36,7 @@ #define GF_NFS_MAX_MEMFACTOR 30 #define GF_NFS_DVM_ON 1 -#define GF_NFS_DVM_OFF 2 +#define GF_NFS_DVM_OFF 0 /* This corresponds to the max 16 number of group IDs that are sent through an * RPC request. Since NFS is the only one going to set this, we can be safe @@ -85,6 +85,7 @@ struct nfs_state { uint32_t server_aux_gids_max_age; gid_cache_t gid_cache; uint32_t generation; + gf_boolean_t register_portmap; }; struct nfs_inode_ctx { diff --git a/xlators/nfs/server/src/nfs3-helpers.c b/xlators/nfs/server/src/nfs3-helpers.c index e299bad31..9059fc341 100644 --- a/xlators/nfs/server/src/nfs3-helpers.c +++ b/xlators/nfs/server/src/nfs3-helpers.c @@ -256,6 +256,20 @@ nfs3_errno_to_nfsstat3 (int errnum) return stat; } +/* + * Special case: If op_ret is -1, it's very unusual op_errno being + * 0 which means something came wrong from upper layer(s). If it + * happens by any means, then set NFS3 status to NFS3ERR_SERVERFAULT. + */ +inline nfsstat3 +nfs3_cbk_errno_status (int32_t op_ret, int32_t op_errno) +{ + if ((op_ret == -1) && (op_errno == 0)) { + return NFS3ERR_SERVERFAULT; + } + + return nfs3_errno_to_nfsstat3 (op_errno); +} void nfs3_fill_lookup3res_error (lookup3res *res, nfsstat3 stat, diff --git a/xlators/nfs/server/src/nfs3-helpers.h b/xlators/nfs/server/src/nfs3-helpers.h index 162c1e740..4de1d5623 100644 --- a/xlators/nfs/server/src/nfs3-helpers.h +++ b/xlators/nfs/server/src/nfs3-helpers.h @@ -35,6 +35,9 @@ nfs3_extract_lookup_name (lookup3args *args); extern nfsstat3 nfs3_errno_to_nfsstat3 (int errnum); +extern nfsstat3 +nfs3_cbk_errno_status (int32_t, int32_t); + extern void nfs3_fill_lookup3res (lookup3res *res, nfsstat3 stat, struct nfs3_fh *newfh, struct iatt *stbuf, struct iatt *postparent, diff --git a/xlators/nfs/server/src/nfs3.c b/xlators/nfs/server/src/nfs3.c index 4faec3559..f914c3193 100644 --- a/xlators/nfs/server/src/nfs3.c +++ b/xlators/nfs/server/src/nfs3.c @@ -64,19 +64,6 @@ } while (0); \ -/* - * Special case: If op_ret is -1, it's very unusual op_errno being - * 0 which means something came wrong from upper layer(s). If it - * happens by any means, then set NFS3 status to NFS3ERR_SERVERFAULT. - */ -static inline nfsstat3 nfs3_cbk_errno_status (int32_t op_ret, int32_t op_errno) -{ - if ((op_ret == -1) && (op_errno == 0)){ - return NFS3ERR_SERVERFAULT; - } - return nfs3_errno_to_nfsstat3 (op_errno); -} - struct nfs3_export * __nfs3_get_export_by_index (struct nfs3_state *nfs3, uuid_t exportid) { @@ -5151,22 +5138,22 @@ rpcsvc_program_t nfs3prog = { }; /* - * This function rounds up the input value to multiple of 4096. If the - * value is same as default, then its a NO-OP. Default is already a - * multiple of 4096. Min and Max supported I/O size limits are - * 4KB (GF_NFS3_FILE_IO_SIZE_MIN) and 1MB (GF_NFS3_FILE_IO_SIZE_MAX). + * This function rounds up the input value to multiple of 4096. Min and Max + * supported I/O size limits are 4KB (GF_NFS3_FILE_IO_SIZE_MIN) and + * 1MB (GF_NFS3_FILE_IO_SIZE_MAX). */ -static void -nfs3_iosize_roundup_4KB (size_t *iosz, size_t iodef) +void +nfs3_iosize_roundup_4KB (uint64_t *ioszptr) { - size_t iosize = *iosz; - size_t iopages; + uint64_t iosize; + uint64_t iopages; - if (iosize == iodef) + if (!ioszptr) return; + iosize = *ioszptr; iopages = (iosize + GF_NFS3_IO_SIZE -1) >> GF_NFS3_IO_SHIFT; - iosize = iopages * GF_NFS3_IO_SIZE; + iosize = (iopages * GF_NFS3_IO_SIZE); /* Double check - boundary conditions */ if (iosize < GF_NFS3_FILE_IO_SIZE_MIN) { @@ -5175,23 +5162,23 @@ nfs3_iosize_roundup_4KB (size_t *iosz, size_t iodef) iosize = GF_NFS3_FILE_IO_SIZE_MAX; } - *iosz = iosize; + *ioszptr = iosize; } int -nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx) +nfs3_init_options (struct nfs3_state *nfs3, dict_t *options) { int ret = -1; char *optstr = NULL; uint64_t size64 = 0; - if ((!nfs3) || (!nfsx)) + if ((!nfs3) || (!options)) return -1; /* nfs3.read-size */ nfs3->readsize = GF_NFS3_RTPREF; - if (dict_get (nfsx->options, "nfs3.read-size")) { - ret = dict_get_str (nfsx->options, "nfs3.read-size", &optstr); + if (dict_get (options, "nfs3.read-size")) { + ret = dict_get_str (options, "nfs3.read-size", &optstr); if (ret < 0) { gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read " " option: nfs3.read-size"); @@ -5200,20 +5187,21 @@ nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx) } ret = gf_string2bytesize (optstr, &size64); - nfs3->readsize = size64; if (ret == -1) { gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to format" " option: nfs3.read-size"); ret = -1; goto err; } + + nfs3_iosize_roundup_4KB (&size64); + nfs3->readsize = size64; } - nfs3_iosize_roundup_4KB (&nfs3->readsize, GF_NFS3_RTPREF); /* nfs3.write-size */ nfs3->writesize = GF_NFS3_WTPREF; - if (dict_get (nfsx->options, "nfs3.write-size")) { - ret = dict_get_str (nfsx->options, "nfs3.write-size", &optstr); + if (dict_get (options, "nfs3.write-size")) { + ret = dict_get_str (options, "nfs3.write-size", &optstr); if (ret < 0) { gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read " " option: nfs3.write-size"); @@ -5222,20 +5210,21 @@ nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx) } ret = gf_string2bytesize (optstr, &size64); - nfs3->writesize = size64; if (ret == -1) { gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to format" " option: nfs3.write-size"); ret = -1; goto err; } + + nfs3_iosize_roundup_4KB (&size64); + nfs3->writesize = size64; } - nfs3_iosize_roundup_4KB (&nfs3->writesize, GF_NFS3_WTPREF); /* nfs3.readdir.size */ nfs3->readdirsize = GF_NFS3_DTPREF; - if (dict_get (nfsx->options, "nfs3.readdir-size")) { - ret = dict_get_str (nfsx->options,"nfs3.readdir-size", &optstr); + if (dict_get (options, "nfs3.readdir-size")) { + ret = dict_get_str (options,"nfs3.readdir-size", &optstr); if (ret < 0) { gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read" " option: nfs3.readdir-size"); @@ -5244,16 +5233,16 @@ nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx) } ret = gf_string2bytesize (optstr, &size64); - nfs3->readdirsize = size64; if (ret == -1) { gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to format" " option: nfs3.readdir-size"); ret = -1; goto err; } - } - nfs3_iosize_roundup_4KB (&nfs3->readdirsize, GF_NFS3_DTPREF); + nfs3_iosize_roundup_4KB (&size64); + nfs3->readdirsize = size64; + } /* We want to use the size of the biggest param for the io buffer size. */ @@ -5274,9 +5263,10 @@ err: return ret; } - int -nfs3_init_subvolume_options (struct nfs3_state *nfs3, struct nfs3_export *exp) +nfs3_init_subvolume_options (xlator_t *nfsx, + struct nfs3_export *exp, + dict_t *options) { int ret = -1; char *optstr = NULL; @@ -5284,14 +5274,20 @@ nfs3_init_subvolume_options (struct nfs3_state *nfs3, struct nfs3_export *exp) char *name = NULL; gf_boolean_t boolt = _gf_false; uuid_t volumeid = {0, }; - dict_t *options = NULL; - if ((!exp) || (!nfs3)) + if ((!nfsx) || (!exp)) return -1; - options = nfs3->nfsx->options; + /* For init, fetch options from xlator but for + * reconfigure, take the parameter */ + if (!options) + options = nfsx->options; + + if (!options) + return (-1); + uuid_clear (volumeid); - if (gf_nfs_dvm_off (nfs_state (nfs3->nfsx))) + if (gf_nfs_dvm_off (nfs_state (nfsx))) goto no_dvm; ret = snprintf (searchkey, 1024, "nfs3.%s.volume-id",exp->subvol->name); @@ -5457,7 +5453,7 @@ nfs3_init_subvolume (struct nfs3_state *nfs3, xlator_t *subvol) INIT_LIST_HEAD (&exp->explist); gf_log (GF_NFS3, GF_LOG_TRACE, "Initing state: %s", exp->subvol->name); - ret = nfs3_init_subvolume_options (nfs3, exp); + ret = nfs3_init_subvolume_options (nfs3->nfsx, exp, NULL); if (ret == -1) { gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to init subvol"); goto exp_free; @@ -5522,7 +5518,7 @@ nfs3_init_state (xlator_t *nfsx) } nfs = nfsx->private; - ret = nfs3_init_options (nfs3, nfsx); + ret = nfs3_init_options (nfs3, nfsx->options); if (ret == -1) { gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to init options"); goto ret; @@ -5596,4 +5592,39 @@ nfs3svc_init (xlator_t *nfsx) return &nfs3prog; } +int +nfs3_reconfigure_state (xlator_t *nfsx, dict_t *options) +{ + int ret = -1; + struct nfs3_export *exp = NULL; + struct nfs_state *nfs = NULL; + struct nfs3_state *nfs3 = NULL; + + if ((!nfsx) || (!nfsx->private) || (!options)) + goto out; + + nfs = (struct nfs_state *)nfsx->private; + nfs3 = nfs->nfs3state; + if (!nfs3) + goto out; + + ret = nfs3_init_options (nfs3, options); + if (ret) { + gf_log (GF_NFS3, GF_LOG_ERROR, + "Failed to reconfigure options"); + goto out; + } + + list_for_each_entry (exp, &nfs3->exports, explist) { + ret = nfs3_init_subvolume_options (nfsx, exp, options); + if (ret) { + gf_log (GF_NFS3, GF_LOG_ERROR, + "Failed to reconfigure subvol options"); + goto out; + } + } + ret = 0; +out: + return ret; +} diff --git a/xlators/nfs/server/src/nfs3.h b/xlators/nfs/server/src/nfs3.h index 054429c82..0c35445a4 100644 --- a/xlators/nfs/server/src/nfs3.h +++ b/xlators/nfs/server/src/nfs3.h @@ -134,14 +134,14 @@ typedef struct nfs3_state { uint64_t serverstart; /* NFSv3 Protocol configurables */ - size_t readsize; - size_t writesize; - size_t readdirsize; + uint64_t readsize; + uint64_t writesize; + uint64_t readdirsize; /* Size of the iobufs used, depends on the sizes of the three params * above. */ - size_t iobsize; + uint64_t iobsize; unsigned int memfactor; @@ -279,4 +279,7 @@ struct inode_op_queue { extern rpcsvc_program_t * nfs3svc_init (xlator_t *nfsx); + +extern int +nfs3_reconfigure_state (xlator_t *nfsx, dict_t *options); #endif diff --git a/xlators/nfs/server/src/nlm4.c b/xlators/nfs/server/src/nlm4.c index f78e28687..5c5d87412 100644 --- a/xlators/nfs/server/src/nlm4.c +++ b/xlators/nfs/server/src/nlm4.c @@ -2359,6 +2359,11 @@ nlm4svc_init(xlator_t *nfsx) struct timespec timeout = {0,}; FILE *pidfile = NULL; pid_t pid = -1; + static gf_boolean_t nlm4_inited = _gf_false; + + /* Already inited */ + if (nlm4_inited) + return &nlm4prog; nfs = (struct nfs_state*)nfsx->private; @@ -2463,6 +2468,7 @@ nlm4svc_init(xlator_t *nfsx) timeout.tv_nsec = 0; gf_timer_call_after (nfsx->ctx, timeout, nlm_grace_period_over, NULL); + nlm4_inited = _gf_true; return &nlm4prog; err: return NULL; diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c index 65ea0bb55..702deaa45 100644 --- a/xlators/protocol/server/src/server.c +++ b/xlators/protocol/server/src/server.c @@ -708,12 +708,6 @@ reconfigure (xlator_t *this, dict_t *options) } - /*ret = dict_get_str (options, "statedump-path", &statedump_path); - if (!ret) { - gf_path_strip_trailing_slashes (statedump_path); - GF_FREE (this->ctx->statedump_path); - this->ctx->statedump_path = gf_strdup (statedump_path); - }*/ GF_OPTION_RECONF ("statedump-path", statedump_path, options, path, out); if (!statedump_path) { @@ -752,6 +746,7 @@ reconfigure (xlator_t *this, dict_t *options) (void) rpcsvc_set_allow_insecure (rpc_conf, options); (void) rpcsvc_set_root_squash (rpc_conf, options); + (void) rpcsvc_set_outstanding_rpc_limit (rpc_conf, options); list_for_each_entry (listeners, &(rpc_conf->listeners), list) { if (listeners->trans != NULL) { if (listeners->trans->reconfigure ) -- cgit