socket: use TCP_USER_TIMEOUT to detect client failures quicker

Use the network.ping-timeout to set the TCP_USER_TIMEOUT socket option
(see 'man 7 tcp'). The option sets the transport.tcp-user-timeout option
that is handled in the rpc/socket layer on the protocol/server side.
This socket option makes detecting unclean disconnected clients more
reliable.

When the socket gets closed, any locks that the client held are been
released. This makes it possible to reduce the fail-over time for
applications that run on systems that became unreachable due to
a network partition or general system error client-side (kernel panic,
hang, ...).

It is not trivial to create a test-case for this at the moment. We need
a client that unclean disconnects and an other client that tries to take
over the lock from the disconnected client.

URL: http://supercolony.gluster.org/pipermail/gluster-devel/2014-May/040755.html
Change-Id: I5e5f540a49abfb5f398291f1818583a63a5f4bb4
BUG: 1129787
Signed-off-by: Niels de Vos <ndevos@redhat.com>
Reviewed-on: http://review.gluster.org/8065
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: soumya k <skoduri@redhat.com>
Reviewed-by: Santosh Pradhan <santosh.pradhan@gmail.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>

7 files changed, 76 insertions, 12 deletions

diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c
index f6774b72353..23fbf37360d 100644
--- a/rpc/rpc-lib/src/rpc-transport.c
+++ b/rpc/rpc-lib/src/rpc-transport.c
@@ -572,7 +572,7 @@ out:
 //why call it if you dont set it.
 int
 rpc_transport_keepalive_options_set (dict_t *options, int32_t interval,
-                                     int32_t time)
+                                     int32_t time, int32_t timeout)
 {
         int                     ret = -1;
 
@@ -588,6 +588,11 @@ rpc_transport_keepalive_options_set (dict_t *options, int32_t interval,
                 "transport.socket.keepalive-time", time);
         if (ret)
                 goto out;
+
+        ret = dict_set_int32 (options,
+                "transport.tcp-user-timeout", timeout);
+        if (ret)
+                goto out;
 out:
         return ret;
 }
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index d1c650e7ec1..d0572a16333 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -306,7 +306,7 @@ rpc_transport_pollin_destroy (rpc_transport_pollin_t *pollin);
 
 int
 rpc_transport_keepalive_options_set (dict_t *options, int32_t interval,
-                                     int32_t time);
+                                     int32_t time, int32_t timeout);
 
 int
 rpc_transport_unix_options_build (dict_t **options, char *filepath,
diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index 2b61eb417d2..054a1c0b1db 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -31,9 +31,15 @@
 #include "xdr-nfs3.h"
 #include "rpcsvc.h"
 
+/* for TCP_USER_TIMEOUT */
+#if !defined(TCP_USER_TIMEOUT) && defined(GF_LINUX_HOST_OS)
+#include <linux/tcp.h>
+#else
+#include <netinet/tcp.h>
+#endif
+
 #include <fcntl.h>
 #include <errno.h>
-#include <netinet/tcp.h>
 #include <rpc/xdr.h>
 #include <sys/ioctl.h>
 #define GF_LOG_ERRNO(errno) ((errno == ENOTCONN) ? GF_LOG_DEBUG : GF_LOG_ERROR)
@@ -857,10 +863,12 @@ __socket_nodelay (int fd)
 
 
 static int
-__socket_keepalive (int fd, int family, int keepalive_intvl, int keepalive_idle)
+__socket_keepalive (int fd, int family, int keepalive_intvl,
+                    int keepalive_idle, int timeout)
 {
         int     on = 1;
         int     ret = -1;
+        int     timeout_ms = timeout * 1000;
 
         ret = setsockopt (fd, SOL_SOCKET, SO_KEEPALIVE, &on, sizeof (on));
         if (ret == -1) {
@@ -890,7 +898,7 @@ __socket_keepalive (int fd, int family, int keepalive_intvl, int keepalive_idle)
                 goto done;
 
         ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepalive_idle,
-                          sizeof (keepalive_intvl));
+                          sizeof (keepalive_idle));
         if (ret == -1) {
                 gf_log ("socket", GF_LOG_WARNING,
                         "failed to set keep idle %d on socket %d, %s",
@@ -905,11 +913,23 @@ __socket_keepalive (int fd, int family, int keepalive_intvl, int keepalive_idle)
                         keepalive_intvl, fd, strerror(errno));
                 goto err;
         }
+
+#if defined(TCP_USER_TIMEOUT)
+        ret = setsockopt (fd, IPPROTO_TCP , TCP_USER_TIMEOUT, &timeout_ms,
+                          sizeof (timeout_ms));
+        if (ret == -1) {
+                gf_log ("socket", GF_LOG_WARNING, "failed to set "
+                        "TCP_USER_TIMEOUT %d on socket %d, %s", timeout_ms, fd,
+                        strerror(errno));
+                goto err;
+        }
+#endif
 #endif
 
 done:
-        gf_log (THIS->name, GF_LOG_TRACE, "Keep-alive enabled for socket %d, interval "
-                "%d, idle: %d", fd, keepalive_intvl, keepalive_idle);
+        gf_log (THIS->name, GF_LOG_TRACE, "Keep-alive enabled for socket %d, "
+                "interval %d, idle: %d, timeout: %d", fd, keepalive_intvl,
+                keepalive_idle, timeout);
 
 err:
         return ret;
@@ -2642,7 +2662,8 @@ socket_server_event_handler (int fd, int idx, void *data,
                                 ret = __socket_keepalive (new_sock,
                                                           new_sockaddr.ss_family,
                                                           priv->keepaliveintvl,
-                                                          priv->keepaliveidle);
+                                                          priv->keepaliveidle,
+                                                          priv->timeout);
                                 if (ret == -1)
                                         gf_log (this->name, GF_LOG_WARNING,
                                                 "Failed to set keep-alive: %s",
@@ -2986,7 +3007,8 @@ socket_connect (rpc_transport_t *this, int port)
                         ret = __socket_keepalive (priv->sock,
                                                   sa_family,
                                                   priv->keepaliveintvl,
-                                                  priv->keepaliveidle);
+                                                  priv->keepaliveidle,
+                                                  priv->timeout);
                         if (ret == -1)
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "Failed to set keep-alive: %s",
@@ -3577,6 +3599,7 @@ reconfigure (rpc_transport_t *this, dict_t *options)
         char             *optstr        = NULL;
         int               ret           = 0;
         uint64_t          windowsize    = 0;
+        uint32_t          timeout       = 0;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
@@ -3605,6 +3628,13 @@ reconfigure (rpc_transport_t *this, dict_t *options)
         else
                 priv->keepalive = 1;
 
+        if (dict_get_uint32 (this->options, "transport.tcp-user-timeout",
+                             &timeout) == 0) {
+                priv->timeout = timeout;
+                gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+                        "transport.tcp-user-timeout=%d", timeout);
+        }
+
         optstr = NULL;
         if (dict_get_str (this->options, "tcp-window-size",
                           &optstr) == 0) {
@@ -3659,6 +3689,7 @@ socket_init (rpc_transport_t *this)
         uint64_t          windowsize = GF_DEFAULT_SOCKET_WINDOW_SIZE;
         char             *optstr = NULL;
         uint32_t          keepalive = 0;
+        uint32_t          timeout = 0;
         uint32_t          backlog = 0;
 	int               session_id = 0;
         int32_t           cert_depth = 1;
@@ -3771,6 +3802,13 @@ socket_init (rpc_transport_t *this)
                 priv->keepaliveidle = keepalive;
         }
 
+        if (dict_get_uint32 (this->options, "transport.tcp-user-timeout",
+                             &timeout) == 0) {
+                priv->timeout = timeout;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "Configued "
+                "transport.tcp-user-timeout=%d", priv->timeout);
+
         if (dict_get_uint32 (this->options,
                              "transport.socket.listen-backlog",
                              &backlog) == 0) {
@@ -4027,6 +4065,9 @@ struct volume_options options[] = {
           .min   = GF_MIN_SOCKET_WINDOW_SIZE,
           .max   = GF_MAX_SOCKET_WINDOW_SIZE
         },
+        { .key   = {"transport.tcp-user-timeout"},
+          .type  = GF_OPTION_TYPE_INT,
+        },
         { .key   = {"transport.socket.nodelay"},
           .type  = GF_OPTION_TYPE_BOOL
         },
diff --git a/rpc/rpc-transport/socket/src/socket.h b/rpc/rpc-transport/socket/src/socket.h
index 6a8ab870ab7..2a84e264b81 100644
--- a/rpc/rpc-transport/socket/src/socket.h
+++ b/rpc/rpc-transport/socket/src/socket.h
@@ -216,6 +216,7 @@ typedef struct {
         int                    keepalive;
         int                    keepaliveidle;
         int                    keepaliveintvl;
+        int                    timeout;
         uint32_t               backlog;
         gf_boolean_t           read_fail_log;
         gf_boolean_t           ssl_enabled;     /* outbound I/O */
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index cc97baf6f21..1c33f3febb3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -2994,7 +2994,8 @@ out:
 }
 
 int
-glusterd_transport_keepalive_options_get (int *interval, int *time)
+glusterd_transport_keepalive_options_get (int *interval, int *time,
+                                          int *timeout)
 {
         int     ret = 0;
         xlator_t *this = NULL;
@@ -3008,6 +3009,9 @@ glusterd_transport_keepalive_options_get (int *interval, int *time)
         ret = dict_get_int32 (this->options,
                               "transport.socket.keepalive-time",
                               time);
+        ret = dict_get_int32 (this->options,
+                              "transport.tcp-user-timeout",
+                              timeout);
         return 0;
 }
 
@@ -3018,6 +3022,7 @@ glusterd_transport_inet_options_build (dict_t **options, const char *hostname,
         dict_t  *dict = NULL;
         int32_t interval = -1;
         int32_t time     = -1;
+        int32_t timeout  = -1;
         int     ret = 0;
 
         GF_ASSERT (options);
@@ -3044,10 +3049,11 @@ glusterd_transport_inet_options_build (dict_t **options, const char *hostname,
         }
 
         /* Set keepalive options */
-        glusterd_transport_keepalive_options_get (&interval, &time);
+        glusterd_transport_keepalive_options_get (&interval, &time, &timeout);
 
         if ((interval > 0) || (time > 0))
-                ret = rpc_transport_keepalive_options_set (dict, interval, time);
+                ret = rpc_transport_keepalive_options_set (dict, interval,
+                                                           time, timeout);
         *options = dict;
 out:
         gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 77f6853dd51..b0a3b0c8c39 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -842,6 +842,11 @@ struct volopt_map_entry glusterd_volopt_map[] = {
         },
 
         /* Server xlator options */
+        { .key         = "network.ping-timeout",
+          .voltype     = "protocol/server",
+          .option      = "transport.tcp-user-timeout",
+          .op_version  = GD_OP_VERSION_3_7_0,
+        },
         { .key         = "network.tcp-window-size",
           .voltype     = "protocol/server",
           .op_version  = 1
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index 023f2a6234f..aea88b623cc 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -1198,6 +1198,12 @@ struct volume_options options[] = {
         { .key   = {"volume-filename.*"},
           .type  = GF_OPTION_TYPE_PATH,
         },
+        { .key   = {"transport.tcp-user-timeout"},
+          .type  = GF_OPTION_TYPE_TIME,
+          .min   = 0,
+          .max   = 1013,
+          .default_value = "42", /* default like network.ping-timeout */
+        },
         { .key   = {"transport.*"},
           .type  = GF_OPTION_TYPE_ANY,
         },