gNFS: RFE for NFS connection behavior

Implement reconfigure() for NFS xlator so that volume set/reset wont
restart the NFS server process. But few options can not be reconfigured
dynamically e.g. nfs.mem-factor, nfs.port etc which needs NFS to be
restarted.

Change-Id: Ic586fd55b7933c0a3175708d8c41ed0475d74a1c
BUG: 1027409
Signed-off-by: Santosh Kumar Pradhan <spradhan@redhat.com>
Reviewed-on: http://review.gluster.org/6236
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
Reviewed-by: Anand Avati <avati@redhat.com>

1 files changed, 415 insertions, 51 deletions

diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c
index 33bd36e7461..75c8fe44eaf 100644
--- a/xlators/nfs/server/src/nfs.c
+++ b/xlators/nfs/server/src/nfs.c
@@ -46,6 +46,194 @@
 #define DATADIR                         "/var/lib/glusterd"
 #define NFS_DATADIR                     DATADIR "/nfs"
 
+/* Forward declaration */
+int nfs_add_initer (struct list_head *list, nfs_version_initer_t init);
+
+static int
+nfs_init_version (xlator_t *this, nfs_version_initer_t init)
+{
+        int                       ret       = -1;
+        struct nfs_initer_list    *version  = NULL;
+        struct nfs_initer_list    *tmp      = NULL;
+        rpcsvc_program_t          *prog     = NULL;
+        struct list_head          *versions = NULL;
+        struct nfs_state          *nfs      = NULL;
+        gf_boolean_t              found     = _gf_false;
+
+        if ((!this) || (!this->private) || (!init))
+                return (-1);
+
+        nfs = (struct nfs_state *)this->private;
+
+        ret = nfs_add_initer (&nfs->versions, init);
+        if (ret == -1) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Failed to add protocol initializer");
+                goto err;
+        }
+
+        versions = &nfs->versions;
+        list_for_each_entry_safe (version, tmp, versions, list) {
+                prog = version->program;
+                if (version->init == init) {
+                        prog = init(this);
+                        if (!prog) {
+                                ret = -1;
+                                goto err;
+                        }
+                        version->program = prog;
+                        found = _gf_true;
+                        break;
+                }
+        }
+
+        /* program not added */
+        if (!found) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Program: %s NOT found", prog->progname);
+                goto err;
+        }
+
+        /* Check if nfs.port is configured */
+        if (nfs->override_portnum)
+                prog->progport = nfs->override_portnum;
+
+        gf_log (GF_NFS, GF_LOG_DEBUG, "Starting program: %s", prog->progname);
+
+        ret = rpcsvc_program_register (nfs->rpcsvc, prog);
+        if (ret == -1) {
+                gf_log (GF_NFS, GF_LOG_ERROR, "Program: %s init failed",
+                                                      prog->progname);
+                goto err;
+        }
+
+        /* Registration with portmapper is disabled, Nothing to do */
+        if (!nfs->register_portmap)
+                goto err;
+
+        ret = rpcsvc_program_register_portmap (prog, prog->progport);
+        if (ret == -1) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Program  %s registration failed",
+                                prog->progname);
+                goto err;
+        }
+        ret = 0; /* All well */
+err:
+        return ret;
+}
+
+static int
+nfs_deinit_version (struct nfs_state *nfs, nfs_version_initer_t init)
+{
+        int                       ret       = -1;
+        struct nfs_initer_list    *version  = NULL;
+        struct nfs_initer_list    *tmp      = NULL;
+        rpcsvc_program_t          *prog     = NULL;
+        struct list_head          *versions = NULL;
+
+        if ((!nfs) || (!init))
+                return (-1);
+
+        versions = &nfs->versions;
+        list_for_each_entry_safe (version, tmp, versions, list) {
+                prog = version->program;
+                if (version->init == init) {
+                        prog = version->program;
+                        ret = rpcsvc_program_unregister (nfs->rpcsvc, prog);
+                        if (ret != 0)
+                                return (-1);
+                        list_del (&version->list);
+                        GF_FREE (version);
+                        return (0);
+                }
+        }
+
+        return (-1);
+}
+
+static int
+nfs_reconfigure_acl3 (xlator_t *this)
+{
+        struct nfs_state          *nfs     = NULL;
+
+        if ((!this) || (!this->private))
+                return (-1);
+
+        nfs = (struct nfs_state *)this->private;
+
+        /* ACL is enabled */
+        if (nfs->enable_acl)
+                return nfs_init_version (this, acl3svc_init);
+
+        /* ACL is disabled */
+        return nfs_deinit_version (nfs, acl3svc_init);
+}
+
+static int
+nfs_reconfigure_nlm4 (xlator_t *this)
+{
+        struct nfs_state          *nfs     = NULL;
+
+        if ((!this) || (!this->private))
+                return (-1);
+
+        nfs = (struct nfs_state *)this->private;
+
+        /* NLM is enabled */
+        if (nfs->enable_nlm)
+                return nfs_init_version (this, nlm4svc_init);
+
+        /* NLM is disabled */
+        return nfs_deinit_version (nfs, nlm4svc_init);
+}
+
+static int
+nfs_program_register_portmap_all (struct nfs_state *nfs)
+{
+        struct list_head                *versions = NULL;
+        struct nfs_initer_list          *version = NULL;
+        struct nfs_initer_list          *tmp = NULL;
+        rpcsvc_program_t                *prog = NULL;
+
+        if (nfs == NULL)
+                return (-1);
+
+        versions = &nfs->versions;
+        list_for_each_entry_safe (version, tmp, versions, list) {
+                prog = version->program;
+                if (prog == NULL)
+                        continue;
+                if (nfs->override_portnum)
+                        prog->progport = nfs->override_portnum;
+                (void) rpcsvc_program_register_portmap (prog, prog->progport);
+        }
+
+        return (0);
+}
+
+static int
+nfs_program_unregister_portmap_all (struct nfs_state *nfs)
+{
+        struct list_head                *versions = NULL;
+        struct nfs_initer_list          *version = NULL;
+        struct nfs_initer_list          *tmp = NULL;
+        rpcsvc_program_t                *prog = NULL;
+
+        if (nfs == NULL)
+                return (-1);
+
+        versions = &nfs->versions;
+        list_for_each_entry_safe (version, tmp, versions, list) {
+                prog = version->program;
+                if (prog == NULL)
+                        continue;
+                (void) rpcsvc_program_unregister_portmap (prog);
+        }
+
+        return (0);
+}
+
 /* Every NFS version must call this function with the init function
  * for its particular version.
  */
@@ -122,7 +310,7 @@ nfs_init_versions (struct nfs_state *nfs, xlator_t *this)
                         ret = -1;
                         goto err;
                 }
-//                prog->actorxl = this;
+
                 version->program = prog;
                 if (nfs->override_portnum)
                         prog->progport = nfs->override_portnum;
@@ -131,17 +319,21 @@ nfs_init_versions (struct nfs_state *nfs, xlator_t *this)
 
                 ret = rpcsvc_program_register (nfs->rpcsvc, prog);
                 if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Program init failed");
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Program: %s init failed",
+                                                      prog->progname);
                         goto err;
                 }
-                if (rpcsvc_register_portmap_enabled(nfs->rpcsvc)) {
+                if (nfs->register_portmap) {
                         ret = rpcsvc_program_register_portmap (prog,
                                                                prog->progport);
                         if (ret == -1) {
-                                gf_log (GF_NFS, GF_LOG_ERROR, "Program registration failed");
+                                gf_log (GF_NFS, GF_LOG_ERROR,
+                                        "Program  %s registration failed",
+                                        prog->progname);
                                 goto err;
                         }
                 }
+
         }
 
         ret = 0;
@@ -158,22 +350,22 @@ nfs_add_all_initiators (struct nfs_state *nfs)
         /* Add the initializers for all versions. */
         ret = nfs_add_initer (&nfs->versions, mnt3svc_init);
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
-                        " initializer");
+                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
+                                "MOUNT3 protocol initializer");
                 goto ret;
         }
 
         ret = nfs_add_initer (&nfs->versions, mnt1svc_init);
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
-                        " initializer");
+                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
+                                "MOUNT1 protocol initializer");
                 goto ret;
         }
 
         ret = nfs_add_initer (&nfs->versions, nfs3svc_init);
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
-                        " initializer");
+                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
+                                "NFS3 protocol initializer");
                 goto ret;
         }
 
@@ -588,19 +780,10 @@ nfs_init_state (xlator_t *this)
         }
 
         nfs->enable_nlm = _gf_true;
-        if (!dict_get_str (this->options, "nfs.nlm", &optstr)) {
-
-                ret = gf_string2boolean (optstr, &boolt);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse"
-                                " bool string");
-                        goto free_foppool;
-                }
-
-                if (boolt == _gf_false) {
-                        gf_log (GF_NFS, GF_LOG_INFO, "NLM is manually disabled");
-                        nfs->enable_nlm = _gf_false;
-                }
+        ret = dict_get_str_boolean (this->options, "nfs.nlm", _gf_true);
+        if (ret == _gf_false) {
+                gf_log (GF_NFS, GF_LOG_INFO, "NLM is manually disabled");
+                nfs->enable_nlm = _gf_false;
         }
 
         nfs->enable_acl = _gf_true;
@@ -691,7 +874,7 @@ nfs_init_state (xlator_t *this)
                         nfs->mount_udp = 1;
         }
 
-        nfs->rmtab = NFS_DATADIR "/rmtab";
+        nfs->rmtab = gf_strdup (NFS_DATADIR "/rmtab");
         if (dict_get(this->options, "nfs.mount-rmtab")) {
                 ret = dict_get_str (this->options, "nfs.mount-rmtab", &nfs->rmtab);
                 if (ret == -1) {
@@ -783,6 +966,8 @@ nfs_init_state (xlator_t *this)
                 goto free_foppool;
         }
 
+        nfs->register_portmap = rpcsvc_register_portmap_enabled (nfs->rpcsvc);
+
         this->private = (void *)nfs;
         INIT_LIST_HEAD (&nfs->versions);
         nfs->generation = 1965;
@@ -820,48 +1005,227 @@ nfs_drc_init (xlator_t *this)
         return ret;
 }
 
+int
+nfs_reconfigure_state (xlator_t *this, dict_t *options)
+{
+        int                 ret = 0;
+        int                 keyindx = 0;
+        char                *optstr = NULL;
+        gf_boolean_t        optbool;
+        uint32_t            optuint32;
+        struct nfs_state    *nfs = NULL;
+        char                *blacklist_keys[] = {
+                                        "nfs.port",
+                                        "nfs.transport-type",
+                                        "nfs.mem-factor",
+                                        NULL};
+
+        GF_VALIDATE_OR_GOTO (GF_NFS, this, out);
+        GF_VALIDATE_OR_GOTO (GF_NFS, this->private, out);
+        GF_VALIDATE_OR_GOTO (GF_NFS, options, out);
+
+        nfs = (struct nfs_state *)this->private;
+
+        /* Black listed options can't be reconfigured, they need
+         * NFS to be restarted. There are two cases 1. SET 2. UNSET.
+         * 1. SET */
+        while (blacklist_keys[keyindx]) {
+                if (dict_get (options, blacklist_keys[keyindx])) {
+                        gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Reconfiguring %s needs NFS restart",
+                                blacklist_keys[keyindx]);
+                        goto out;
+                }
+                keyindx ++;
+        }
+
+        /* UNSET for nfs.mem-factor */
+        if ((!dict_get (options, "nfs.mem-factor")) &&
+            (nfs->memfactor != GF_NFS_DEFAULT_MEMFACTOR)) {
+                gf_log (GF_NFS, GF_LOG_INFO,
+                        "Reconfiguring nfs.mem-factor needs NFS restart");
+                goto out;
+        }
+
+        /* UNSET for nfs.port */
+        if ((!dict_get (options, "nfs.port")) &&
+            (nfs->override_portnum)) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "Reconfiguring nfs.port needs NFS restart");
+                goto out;
+        }
+
+        /* reconfig nfs.mount-rmtab */
+        optstr = NFS_DATADIR "/rmtab";
+        if (dict_get (options, "nfs.mount-rmtab")) {
+                ret = dict_get_str (options, "nfs.mount-rmtab", &optstr);
+                if (ret < 0) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to read "
+                                "reconfigured option: nfs.mount-rmtab");
+                        goto out;
+                }
+                gf_path_strip_trailing_slashes (optstr);
+        }
+        if (strcmp (nfs->rmtab, optstr) != 0) {
+                mount_rewrite_rmtab (nfs->mstate, optstr);
+                gf_log (GF_NFS, GF_LOG_INFO,
+                                "Reconfigured nfs.mount-rmtab path: %s",
+                                nfs->rmtab);
+        }
+
+        GF_OPTION_RECONF (OPT_SERVER_AUX_GIDS, optbool,
+                                               options, bool, out);
+        if (nfs->server_aux_gids != optbool) {
+                nfs->server_aux_gids = optbool;
+                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured %s with value %d",
+                               OPT_SERVER_AUX_GIDS, optbool);
+        }
+
+        GF_OPTION_RECONF (OPT_SERVER_GID_CACHE_TIMEOUT, optuint32,
+                                               options, uint32, out);
+        if (nfs->server_aux_gids_max_age != optuint32) {
+                nfs->server_aux_gids_max_age = optuint32;
+                gid_cache_reconf (&nfs->gid_cache, optuint32);
+                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured %s with value %d",
+                               OPT_SERVER_GID_CACHE_TIMEOUT, optuint32);
+        }
+
+        /* reconfig nfs.dynamic-volumes */
+        ret = dict_get_str_boolean (options, "nfs.dynamic-volumes",
+                                             GF_NFS_DVM_OFF);
+        switch (ret) {
+        case GF_NFS_DVM_ON:
+        case GF_NFS_DVM_OFF:
+                optbool = ret;
+                break;
+        default:
+                optbool = GF_NFS_DVM_OFF;
+                break;
+        }
+        if (nfs->dynamicvolumes != optbool) {
+                nfs->dynamicvolumes = optbool;
+                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured nfs.dynamic-volumes"
+                                            " with value %d", optbool);
+        }
+
+        optbool = _gf_false;
+        if (dict_get (options, "nfs.enable-ino32")) {
+                ret = dict_get_str_boolean (options, "nfs.enable-ino32",
+                                                      _gf_false);
+                if (ret < 0) {
+                        gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Failed to read reconfigured option: "
+                                "nfs.enable-ino32");
+                        goto out;
+                }
+                optbool = ret;
+        }
+        if (nfs->enable_ino32 != optbool) {
+                nfs->enable_ino32 = optbool;
+                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured nfs.enable-ino32"
+                                            " with value %d", optbool);
+        }
+
+        /* nfs.nlm is enabled by default */
+        ret = dict_get_str_boolean (options, "nfs.nlm", _gf_true);
+        if (ret < 0) {
+                optbool = _gf_true;
+        } else {
+                optbool = ret;
+        }
+        if (nfs->enable_nlm != optbool) {
+                gf_log (GF_NFS, GF_LOG_INFO, "NLM is manually %s",
+                                (optbool ? "enabled":"disabled"));
+                nfs->enable_nlm = optbool;
+                nfs_reconfigure_nlm4 (this);
+        }
+
+        /* nfs.acl is enabled by default */
+        ret = dict_get_str_boolean (options, "nfs.acl", _gf_true);
+        if (ret < 0) {
+                optbool = _gf_true;
+        } else {
+                optbool = ret;
+        }
+        if (nfs->enable_acl != optbool) {
+                gf_log (GF_NFS, GF_LOG_INFO, "ACL is manually %s",
+                                (optbool ? "enabled":"disabled"));
+                nfs->enable_acl = optbool;
+                nfs_reconfigure_acl3 (this);
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
 
-#if 0
-/* reconfigure() is currently not used for the NFS-server. Upon setting an
- * option for the NFS-xlator, the glusterfs process it restarted.
- *
- * This current implementation makes sure to read the currently used rmtab and
- * merge it with the new rmtab.
- *
- * As this function is never called, it is provided for the future, for when
- * the NFS-server supports reloading.
+/*
+ * reconfigure() for NFS server xlator.
  */
 int
 reconfigure (xlator_t *this, dict_t *options)
 {
-        int                       ret = 0;
-        char                     *rmtab = NULL;
+        int                      ret = 0;
         struct nfs_state         *nfs = NULL;
+        gf_boolean_t             regpmap = _gf_true;
+
+        if ((!this) || (!this->private) || (!options))
+                return (-1);
 
         nfs = (struct nfs_state *)this->private;
 
-        if (!nfs) {
-                gf_log_callingfn (this->name, GF_LOG_DEBUG, "conf == null!!!");
-                goto out;
+        /* Reconfigure nfs options */
+        ret = nfs_reconfigure_state(this, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "nfs reconfigure state failed");
+                return (-1);
         }
 
-        ret = dict_get_str (options, "nfs.mount-rmtab", &rmtab);
+        /* Reconfigure nfs3 options */
+        ret = nfs3_reconfigure_state(this, options);
         if (ret) {
-                goto out;
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "nfs3 reconfigure state failed");
+                return (-1);
         }
-        gf_path_strip_trailing_slashes (rmtab);
 
-        if (strcmp (nfs->rmtab, rmtab) != 0) {
-                mount_rewrite_rmtab (nfs->mstate, rmtab);
+        /* Reconfigure mount options */
+        ret = mount_reconfigure_state(this, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "mount reconfigure state failed");
+                return (-1);
+        }
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "Reconfigured nfs.mount-rmtab path: %s", nfs->rmtab);
+        /* Reconfigure rpc layer */
+        ret = rpcsvc_reconfigure_options (nfs->rpcsvc, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "rpcsvc reconfigure options failed");
+                return (-1);
+        }
+        regpmap = rpcsvc_register_portmap_enabled(nfs->rpcsvc);
+        if (nfs->register_portmap != regpmap) {
+                nfs->register_portmap = regpmap;
+                if (regpmap) {
+                        (void) nfs_program_register_portmap_all (nfs);
+                } else {
+                        (void) nfs_program_unregister_portmap_all (nfs);
+                }
         }
 
-out:
-        return ret;
+        /* Reconfigure drc */
+        ret = rpcsvc_drc_reconfigure (nfs->rpcsvc, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "rpcsvc DRC reconfigure failed");
+                return (-1);
+        }
+
+        return (0);
 }
-#endif /* glusterfs/nfs is restarted and reconfigure() is never called */
 
 int
 init (xlator_t *this) {
@@ -1361,7 +1725,7 @@ struct volume_options options[] = {
         },
         { .key  = {"rpc.outstanding-rpc-limit"},
           .type = GF_OPTION_TYPE_INT,
-          .min  = 0,
+          .min  = RPCSVC_MIN_OUTSTANDING_RPC_LIMIT,
           .max  = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT,
           .default_value = TOSTRING(RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT),
           .description = "Parameter to throttle the number of incoming RPC "
@@ -1446,8 +1810,8 @@ struct volume_options options[] = {
         },
         { .key  = {"nfs.drc"},
           .type = GF_OPTION_TYPE_STR,
-          .default_value = "off",
-          .description = "Enable Duplicate Request Cache in gNfs server to "
+          .default_value = "on",
+          .description = "Enable Duplicate Request Cache in gNFS server to "
                          "improve correctness of non-idempotent operations like "
                          "write, delete, link, et al"
         },