rpc: add options to manage socket keepalive lifespan

Problem:
Default values for handling socket timeouts for brick responses are
insufficient for aggressive applications such as databases.

Solution:
Add 1:1 gluster options for keepalive, keepalive-idle,
keepalive-interval and keepalive-timeout as per the socket level options
available as per tcp(7) man page.

Default values for options are NOT agressive and continue to be values
which result in default timeout when only the keep alive option is
turned on.

These options are Linux specific and will not be applicable to the
*BSDs.

mainline:
> BUG: 1426059
> Signed-off-by: Milind Changire <mchangir@redhat.com>
> Reviewed-on: https://review.gluster.org/16731
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
(cherry picked from commit 6b8df081b46ac4f485c86a5052fc30472e74bfbb)

Change-Id: I2a08ecd949ca8ceb3e090d336ad634341e2dbf14
BUG: 1452038
Signed-off-by: Milind Changire <mchangir@redhat.com>
Reviewed-on: https://review.gluster.org/17330
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra Talur <rtalur@redhat.com>

4 files changed, 164 insertions, 44 deletions

diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h
index 0d5dde1f6b7..f93be102930 100644
--- a/libglusterfs/src/globals.h
+++ b/libglusterfs/src/globals.h
@@ -38,7 +38,7 @@
  */
 #define GD_OP_VERSION_MIN  1 /* MIN is the fresh start op-version, mostly
                                 should not change */
-#define GD_OP_VERSION_MAX  GD_OP_VERSION_3_10_0 /* MAX VERSION is the maximum
+#define GD_OP_VERSION_MAX  GD_OP_VERSION_3_10_4 /* MAX VERSION is the maximum
                                                   count in VME table, should
                                                   keep changing with
                                                   introduction of newer
@@ -78,6 +78,16 @@
 
 #define GD_OP_VERSION_3_10_0   31000 /* Op-version for GlusterFS 3.10.0 */
 
+#define GD_OP_VERSION_3_10_1   31001 /* Op-version for GlusterFS 3.10.1 */
+
+#define GD_OP_VERSION_3_10_2   31002 /* Op-version for GlusterFS 3.10.2 */
+
+#define GD_OP_VERSION_3_10_3   31003 /* Op-version for GlusterFS 3.10.3 */
+
+#define GD_OP_VERSION_3_10_4   31004 /* Op-version for GlusterFS 3.10.4 */
+
+#define GD_OP_VERSION_4_0_0    40000 /* Op-version for GlusterFS 4.0.0 */
+
 #define GD_OP_VER_PERSISTENT_AFR_XATTRS GD_OP_VERSION_3_6_0
 
 #include "xlator.h"
diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index e393f2704bf..71eacf83f2a 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -838,8 +838,8 @@ __socket_nodelay (int fd)
 
 
 static int
-__socket_keepalive (int fd, int family, int keepalive_intvl,
-                    int keepalive_idle, int timeout)
+__socket_keepalive (int fd, int family, int keepaliveintvl,
+                    int keepaliveidle, int keepalivecnt, int timeout)
 {
         int     on = 1;
         int     ret = -1;
@@ -852,16 +852,16 @@ __socket_keepalive (int fd, int family, int keepalive_intvl,
                 goto err;
         }
 
-        if (keepalive_intvl == GF_USE_DEFAULT_KEEPALIVE)
+        if (keepaliveintvl == GF_USE_DEFAULT_KEEPALIVE)
                 goto done;
 
 #if !defined(GF_LINUX_HOST_OS) && !defined(__NetBSD__)
 #if defined(GF_SOLARIS_HOST_OS) || defined(__FreeBSD__)
-        ret = setsockopt (fd, SOL_SOCKET, SO_KEEPALIVE, &keepalive_intvl,
-                          sizeof (keepalive_intvl));
+        ret = setsockopt (fd, SOL_SOCKET, SO_KEEPALIVE, &keepaliveintvl,
+                          sizeof (keepaliveintvl));
 #else
-        ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPALIVE, &keepalive_intvl,
-                          sizeof (keepalive_intvl));
+        ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPALIVE, &keepaliveintvl,
+                          sizeof (keepaliveintvl));
 #endif
         if (ret == -1) {
                 gf_log ("socket", GF_LOG_WARNING,
@@ -872,20 +872,20 @@ __socket_keepalive (int fd, int family, int keepalive_intvl,
         if (family != AF_INET && family != AF_INET6)
                 goto done;
 
-        ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepalive_idle,
-                          sizeof (keepalive_idle));
+        ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepaliveidle,
+                          sizeof (keepaliveidle));
         if (ret == -1) {
                 gf_log ("socket", GF_LOG_WARNING,
                         "failed to set keep idle %d on socket %d, %s",
-                        keepalive_idle, fd, strerror(errno));
+                        keepaliveidle, fd, strerror(errno));
                 goto err;
         }
-        ret = setsockopt (fd, IPPROTO_TCP , TCP_KEEPINTVL, &keepalive_intvl,
-                          sizeof (keepalive_intvl));
+        ret = setsockopt (fd, IPPROTO_TCP , TCP_KEEPINTVL, &keepaliveintvl,
+                          sizeof (keepaliveintvl));
         if (ret == -1) {
                 gf_log ("socket", GF_LOG_WARNING,
                         "failed to set keep interval %d on socket %d, %s",
-                        keepalive_intvl, fd, strerror(errno));
+                        keepaliveintvl, fd, strerror(errno));
                 goto err;
         }
 
@@ -901,12 +901,23 @@ __socket_keepalive (int fd, int family, int keepalive_intvl,
                 goto err;
         }
 #endif
+#if defined(TCP_KEEPCNT)
+        ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPCNT, &keepalivecnt,
+                          sizeof (keepalivecnt));
+        if (ret == -1) {
+                gf_log ("socket", GF_LOG_WARNING, "failed to set "
+                        "TCP_KEEPCNT %d on socket %d, %s", keepalivecnt, fd,
+                        strerror(errno));
+                goto err;
+        }
+#endif
 #endif
 
 done:
-        gf_log (THIS->name, GF_LOG_TRACE, "Keep-alive enabled for socket %d, "
-                "interval %d, idle: %d, timeout: %d", fd, keepalive_intvl,
-                keepalive_idle, timeout);
+        gf_log (THIS->name, GF_LOG_TRACE, "Keep-alive enabled for socket: %d, "
+                "(idle: %d, interval: %d, max-probes: %d, timeout: %d)",
+                fd, keepaliveidle, keepaliveintvl, keepalivecnt,
+                timeout);
 
 err:
         return ret;
@@ -2703,6 +2714,7 @@ socket_server_event_handler (int fd, int idx, void *data,
                                                   new_sockaddr.ss_family,
                                                   priv->keepaliveintvl,
                                                   priv->keepaliveidle,
+                                                  priv->keepalivecnt,
                                                   priv->timeout);
                         if (ret == -1)
                                 gf_log (this->name, GF_LOG_WARNING,
@@ -3135,6 +3147,7 @@ socket_connect (rpc_transport_t *this, int port)
                                                   sa_family,
                                                   priv->keepaliveintvl,
                                                   priv->keepaliveidle,
+                                                  priv->keepalivecnt,
                                                   priv->timeout);
                         if (ret == -1)
                                 gf_log (this->name, GF_LOG_ERROR,
@@ -3776,6 +3789,9 @@ reconfigure (rpc_transport_t *this, dict_t *options)
         int               ret           = 0;
         uint64_t          windowsize    = 0;
         uint32_t          timeout       = 0;
+        int               keepaliveidle  = GF_KEEPALIVE_TIME;
+        int               keepaliveintvl = GF_KEEPALIVE_INTERVAL;
+        int               keepalivecnt   = GF_KEEPALIVE_COUNT;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
@@ -3804,12 +3820,30 @@ reconfigure (rpc_transport_t *this, dict_t *options)
         else
                 priv->keepalive = 1;
 
-        if (dict_get_uint32 (this->options, "transport.tcp-user-timeout",
-                             &timeout) == 0) {
+        if (dict_get_int32 (this->options, "transport.tcp-user-timeout",
+                            &(priv->timeout)) != 0)
                 priv->timeout = timeout;
-                gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
-                        "transport.tcp-user-timeout=%d", timeout);
-        }
+        gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+                "transport.tcp-user-timeout=%d", priv->timeout);
+
+        if (dict_get_int32 (this->options, "transport.socket.keepalive-time",
+                            &(priv->keepaliveidle)) != 0)
+                priv->keepaliveidle = keepaliveidle;
+        gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+                "transport.socket.keepalive-time=%d", priv->keepaliveidle);
+
+        if (dict_get_int32 (this->options,
+                            "transport.socket.keepalive-interval",
+                            &(priv->keepaliveintvl)) != 0)
+                priv->keepaliveintvl = keepaliveintvl;
+        gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+                "transport.socket.keepalive-interval=%d", priv->keepaliveintvl);
+
+        if (dict_get_int32 (this->options, "transport.socket.keepalive-count",
+                            &(priv->keepalivecnt)) != 0)
+                priv->keepalivecnt = keepalivecnt;
+        gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+                "transport.socket.keepalive-count=%d", priv->keepalivecnt);
 
         optstr = NULL;
         if (dict_get_str (this->options, "tcp-window-size",
@@ -3954,8 +3988,10 @@ socket_init (rpc_transport_t *this)
         gf_boolean_t      tmp_bool = 0;
         uint64_t          windowsize = GF_DEFAULT_SOCKET_WINDOW_SIZE;
         char             *optstr = NULL;
-        uint32_t          keepalive = 0;
         uint32_t          timeout = 0;
+        int               keepaliveidle  = GF_KEEPALIVE_TIME;
+        int               keepaliveintvl = GF_KEEPALIVE_INTERVAL;
+        int               keepalivecnt   = GF_KEEPALIVE_COUNT;
         uint32_t          backlog = 0;
 	int               session_id = 0;
         int32_t           cert_depth = DEFAULT_VERIFY_DEPTH;
@@ -4047,8 +4083,9 @@ socket_init (rpc_transport_t *this)
         optstr = NULL;
         /* Enable Keep-alive by default. */
         priv->keepalive = 1;
-        priv->keepaliveintvl = 2;
-        priv->keepaliveidle = 20;
+        priv->keepaliveintvl = GF_KEEPALIVE_INTERVAL;
+        priv->keepaliveidle = GF_KEEPALIVE_TIME;
+        priv->keepalivecnt = GF_KEEPALIVE_COUNT;
         if (dict_get_str (this->options, "transport.socket.keepalive",
                           &optstr) == 0) {
                 if (gf_string2boolean (optstr, &tmp_bool) == -1) {
@@ -4062,24 +4099,29 @@ socket_init (rpc_transport_t *this)
                         priv->keepalive = 0;
         }
 
-        if (dict_get_uint32 (this->options,
-                             "transport.socket.keepalive-interval",
-                             &keepalive) == 0) {
-                priv->keepaliveintvl = keepalive;
-        }
+        if (dict_get_int32 (this->options, "transport.tcp-user-timeout",
+                            &(priv->timeout)) != 0)
+                priv->timeout = timeout;
+        gf_log (this->name, GF_LOG_DEBUG, "Configued "
+                "transport.tcp-user-timeout=%d", priv->timeout);
 
-        if (dict_get_uint32 (this->options,
-                             "transport.socket.keepalive-time",
-                             &keepalive) == 0) {
-                priv->keepaliveidle = keepalive;
+        if (dict_get_int32 (this->options,
+                            "transport.socket.keepalive-time",
+                            &(priv->keepaliveidle)) != 0) {
+                priv->keepaliveidle = keepaliveidle;
         }
 
-        if (dict_get_uint32 (this->options, "transport.tcp-user-timeout",
-                             &timeout) == 0) {
-                priv->timeout = timeout;
+        if (dict_get_int32 (this->options,
+                            "transport.socket.keepalive-interval",
+                            &(priv->keepaliveintvl)) != 0) {
+                priv->keepaliveintvl = keepaliveintvl;
         }
-        gf_log (this->name, GF_LOG_DEBUG, "Configued "
-                "transport.tcp-user-timeout=%d", priv->timeout);
+
+        if (dict_get_int32 (this->options, "transport.socket.keepalive-count",
+                            &(priv->keepalivecnt)) != 0)
+                priv->keepalivecnt = keepalivecnt;
+        gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+                "transport.keepalivecnt=%d", keepalivecnt);
 
         if (dict_get_uint32 (this->options,
                              "transport.socket.listen-backlog",
@@ -4479,21 +4521,30 @@ struct volume_options options[] = {
         },
         { .key   = {"transport.tcp-user-timeout"},
           .type  = GF_OPTION_TYPE_INT,
+          .default_value = "0"
         },
         { .key   = {"transport.socket.nodelay"},
-          .type  = GF_OPTION_TYPE_BOOL
+          .type  = GF_OPTION_TYPE_BOOL,
+          .default_value = "1"
         },
         { .key   = {"transport.socket.lowlat"},
           .type  = GF_OPTION_TYPE_BOOL
         },
         { .key   = {"transport.socket.keepalive"},
-          .type  = GF_OPTION_TYPE_BOOL
+          .type  = GF_OPTION_TYPE_BOOL,
+          .default_value = "1"
         },
         { .key   = {"transport.socket.keepalive-interval"},
-          .type  = GF_OPTION_TYPE_INT
+          .type  = GF_OPTION_TYPE_INT,
+          .default_value = "2"
         },
         { .key   = {"transport.socket.keepalive-time"},
-          .type  = GF_OPTION_TYPE_INT
+          .type  = GF_OPTION_TYPE_INT,
+          .default_value = "20"
+        },
+        { .key   = {"transport.socket.keepalive-count"},
+          .type  = GF_OPTION_TYPE_INT,
+          .default_value = "9"
         },
         { .key   = {"transport.socket.listen-backlog"},
           .type  = GF_OPTION_TYPE_INT
diff --git a/rpc/rpc-transport/socket/src/socket.h b/rpc/rpc-transport/socket/src/socket.h
index 8528bdeba8d..6c8875f7fb7 100644
--- a/rpc/rpc-transport/socket/src/socket.h
+++ b/rpc/rpc-transport/socket/src/socket.h
@@ -50,6 +50,10 @@
 #define GF_MIN_SOCKET_WINDOW_SIZE       (0)
 #define GF_USE_DEFAULT_KEEPALIVE        (-1)
 
+#define GF_KEEPALIVE_TIME               (20)
+#define GF_KEEPALIVE_INTERVAL           (2)
+#define GF_KEEPALIVE_COUNT              (9)
+
 typedef enum {
         SP_STATE_NADA = 0,
         SP_STATE_COMPLETE,
@@ -224,6 +228,7 @@ typedef struct {
         int                    keepalive;
         int                    keepaliveidle;
         int                    keepaliveintvl;
+        int                    keepalivecnt;
         int                    timeout;
         uint32_t               backlog;
         gf_boolean_t           read_fail_log;
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index bb3e2100901..d5f9fcf00a8 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -1681,6 +1681,7 @@ struct volopt_map_entry glusterd_volopt_map[] = {
         { .key        = "network.ping-timeout",
           .voltype    = "protocol/client",
           .op_version = 1,
+          .value      = "42",
           .flags      = OPT_FLAG_CLIENT_OPT
         },
         { .key        = "network.tcp-window-size",
@@ -1724,12 +1725,40 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .voltype     = "protocol/client",
           .op_version  = GD_OP_VERSION_3_7_0,
         },
+        { .key         = "client.tcp-user-timeout",
+          .voltype     = "protocol/client",
+          .option      = "transport.tcp-user-timeout",
+          .op_version  = GD_OP_VERSION_3_10_4,
+          .value       = "0",           /* 0 - implies "use system default" */
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "client.keepalive-time",
+          .voltype     = "protocol/client",
+          .option      = "transport.socket.keepalive-time",
+          .op_version  = GD_OP_VERSION_3_10_4,
+          .value       = "20",
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "client.keepalive-interval",
+          .voltype     = "protocol/client",
+          .option      = "transport.socket.keepalive-interval",
+          .op_version  = GD_OP_VERSION_3_10_4,
+          .value       = "2",
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "client.keepalive-count",
+          .voltype     = "protocol/client",
+          .option      = "transport.socket.keepalive-count",
+          .op_version  = GD_OP_VERSION_3_10_4,
+          .value       = "9",
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
 
         /* Server xlator options */
         { .key         = "network.ping-timeout",
           .voltype     = "protocol/server",
-          .option      = "transport.tcp-user-timeout",
           .op_version  = GD_OP_VERSION_3_7_0,
+          .value       = "42",
         },
         { .key         = "network.tcp-window-size",
           .voltype     = "protocol/server",
@@ -1755,6 +1784,7 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .voltype     = "protocol/server",
           .option      = "transport.socket.keepalive",
           .type        = NO_DOC,
+          .value       = "1",
           .op_version  = 1
         },
         { .key         = "server.allow-insecure",
@@ -1842,6 +1872,30 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .voltype     = "protocol/server",
           .op_version  = GD_OP_VERSION_3_7_0,
         },
+        { .key         = "server.tcp-user-timeout",
+          .voltype     = "protocol/server",
+          .option      = "transport.tcp-user-timeout",
+          .op_version  = GD_OP_VERSION_3_10_4,
+          .value       = "0",           /* 0 - implies "use system default" */
+        },
+        { .key         = "server.keepalive-time",
+          .voltype     = "protocol/server",
+          .option      = "transport.socket.keepalive-time",
+          .op_version  = GD_OP_VERSION_3_10_4,
+          .value       = "20",
+        },
+        { .key         = "server.keepalive-interval",
+          .voltype     = "protocol/server",
+          .option      = "transport.socket.keepalive-interval",
+          .op_version  = GD_OP_VERSION_3_10_4,
+          .value       = "2",
+        },
+        { .key         = "server.keepalive-count",
+          .voltype     = "protocol/server",
+          .option      = "transport.socket.keepalive-count",
+          .op_version  = GD_OP_VERSION_3_10_4,
+          .value       = "9",
+        },
 
         /* Generic transport options */
         { .key         = SSL_OWN_CERT_OPT,