event: Idle connection management

Summary:
- This diff adds support for detecting and tracking idle client connections.
- It allows *service translators* (server, nfs) to opt-in to detect and close idle client connections.
- Right now it explicitly restricts the service to NFS as a safety.

Here are the debug logs when a client connection gets closed:

  [2016-03-29 17:27:06.154232] W [socket.c:2426:socket_timeout_handler] 0-socket: Shutting down idle client connection (idle=20s,fd=20,conn=[2401:db00:11:d0af:face:0:3:0:957]->[2401:db00:11:d0af:face:0:3:0:2049])!
  [2016-03-29 17:27:06.154292] D [event-epoll.c:655:__event_epoll_timeout_slot] 0-epoll: Connection on slot->fd=9 was idle for 20 seconds!
  [2016-03-29 17:27:06.163282] D [socket.c:629:__socket_rwv] 0-socket.nfs-server: EOF on socket
  [2016-03-29 17:27:06.163298] D [socket.c:2474:socket_event_handler] 0-transport: disconnecting now
  [2016-03-29 17:27:06.163316] D [event-epoll.c:614:event_dispatch_epoll_handler] 0-epoll: generation bumped on idx=9 from gen=4 to slot->gen=5, fd=20, slot->fd=20

Test Plan: - Used stuck NFS mounts to create idle clients and unstuck them.

Reviewers: kvigor, rwareing

Reviewed By: rwareing

Subscribers: dld, moox, dph

Differential Revision: https://phabricator.fb.com/D3112099

Change-Id: Ic06c89e03f87daabab7f07f892390edd1a1fcc20
Signed-off-by: Jeff Darcy <jdarcy@fb.com>
Reviewed-on: https://review.gluster.org/18265
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>
Tested-by: Jeff Darcy <jeff@pl.atyp.us>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>

3 files changed, 70 insertions, 2 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index bb4efdf2c21..a1fa19c3207 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -2503,7 +2503,24 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .type        = GLOBAL_DOC,
           .op_version  = GD_OP_VERSION_3_7_0
         },
-
+        {  .key        = "nfs.idle-connection-check-interval",
+           .voltype    = "nfs/server",
+           .option     = "nfs.idle-connection-check-interval",
+           .type       = NO_DOC,
+           .op_version = 2
+        },
+        {  .key        = "nfs.client-max-idle-seconds",
+           .voltype    = "nfs/server",
+           .option     = "nfs.client-max-idle-seconds",
+           .type       = NO_DOC,
+           .op_version = 2
+        },
+        {  .key        = "nfs.close-idle-clients",
+           .voltype    = "nfs/server",
+           .option     = "nfs.close-idle-clients",
+           .type       = NO_DOC,
+           .op_version = 2
+        },
         /* Other options which don't fit any place above */
         { .key        = "features.read-only",
           .voltype    = "features/read-only",
diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c
index e94cb03b771..5329194bf1d 100644
--- a/xlators/nfs/server/src/nfs.c
+++ b/xlators/nfs/server/src/nfs.c
@@ -1223,6 +1223,15 @@ nfs_init_state (xlator_t *this)
 
         nfs->register_portmap = rpcsvc_register_portmap_enabled (nfs->rpcsvc);
 
+	GF_OPTION_INIT ("nfs.client-max-idle-seconds", nfs->client_max_idle_seconds, uint32, free_foppool);
+	GF_OPTION_INIT ("nfs.close-idle-clients", nfs->close_idle_clients, bool, free_foppool);
+	GF_OPTION_INIT ("nfs.idle-connection-check-interval", nfs->idle_conn_check_interval, uint32, free_foppool);
+
+        event_configure_idle_conns (this->ctx->event_pool,
+                                    nfs->client_max_idle_seconds,
+                                    (int)nfs->close_idle_clients,
+                                    nfs->idle_conn_check_interval);
+
         this->private = (void *)nfs;
         INIT_LIST_HEAD (&nfs->versions);
         nfs->generation = 1965;
@@ -1279,6 +1288,7 @@ nfs_reconfigure_state (xlator_t *this, dict_t *options)
                                         "nfs.mem-factor",
                                         NULL};
         char                *exports_auth_enable = NULL;
+        char                *optstr = NULL;
 
         GF_VALIDATE_OR_GOTO (GF_NFS, this, out);
         GF_VALIDATE_OR_GOTO (GF_NFS, this->private, out);
@@ -1465,6 +1475,15 @@ nfs_reconfigure_state (xlator_t *this, dict_t *options)
                 nfs_reconfigure_acl3 (this);
         }
 
+        GF_OPTION_RECONF ("nfs.client-max-idle-seconds", nfs->client_max_idle_seconds, options, uint32, out);
+        GF_OPTION_RECONF ("nfs.close-idle-clients", nfs->close_idle_clients, options, bool, out);
+	GF_OPTION_RECONF ("nfs.idle-connection-check-interval", nfs->idle_conn_check_interval, options, uint32, out);
+
+        event_configure_idle_conns (this->ctx->event_pool,
+                                    nfs->client_max_idle_seconds,
+                                    (int)nfs->close_idle_clients,
+                                    nfs->idle_conn_check_interval);
+
         ret = 0;
 out:
         return ret;
@@ -2209,6 +2228,34 @@ struct volume_options options[] = {
           .description = "When this option is set to off NFS falls back to "
                          "standard readdir instead of readdirp"
         },
-
+        { .key = {"nfs.idle-connection-check-interval"},
+          .type = GF_OPTION_TYPE_SIZET,
+          .min = 10,
+          .max = UINT32_MAX,
+          .default_value = "20",
+          .description = "The amount of time between walks of the event table"
+                         " to figure out which clients are idle."
+        },
+        { .key = {"nfs.client-max-idle-seconds"},
+          .type = GF_OPTION_TYPE_SIZET,
+          .min = 0,
+          .max = UINT32_MAX,
+          .default_value = "0",
+          .description = "The maximum amount of time this NFS daemon allows clients"
+                         " to be idle before it treats the clients as idle."
+                         " This option enables *tracking* of idle clients, but does not"
+                         " perform any actions on idle clients. See \"nfs.close-idle-clients\""
+                         " to close idle client connections."
+                         " A value of 0 disables this feature."
+        },
+        { .key = {"nfs.close-idle-clients"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option is used to control whether NFS will close"
+                         " idle client connections. Idle clients are detected "
+                         " and tracked via the option \"nfs.client-max-idle-seconds\"."
+                         " A value of \"off\" means that we won't close idle connections."
+                         " A value of \"on\" means that idle connections will be closed."
+        },
         { .key  = {NULL} },
 };
diff --git a/xlators/nfs/server/src/nfs.h b/xlators/nfs/server/src/nfs.h
index 4f5faf29f6b..6a02ae14d24 100644
--- a/xlators/nfs/server/src/nfs.h
+++ b/xlators/nfs/server/src/nfs.h
@@ -102,6 +102,10 @@ struct nfs_state {
         char                    *rpc_statd;
         char                    *rpc_statd_pid_file;
         gf_boolean_t            rdirplus;
+
+        unsigned int            client_max_idle_seconds;
+        unsigned int            idle_conn_check_interval;
+        gf_boolean_t            close_idle_clients;
 };
 
 struct nfs_inode_ctx {