glusterd: Disable ping-timer between glusterd and brick process

When there are too many IO happening, brick process epoll thread
will be busy and fails to respond to the glusterd pick packet within
30sec.
Also epoll thread can be blocked by a big-lock.

Solution is to disable ping-timer by default and only enable where ever
required

Later when the epoll thread model changed and made lighter,
we need to revert back this change. http://review.gluster.com/3842 is
one such approach.

Change-Id: I7f80ad3eb00f7d9c4d4527305932f7cf4920e73f
BUG: 1097224
Signed-off-by: Vijaikumar M <vmallika@redhat.com>
Reviewed-on: http://review.gluster.org/7753
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

6 files changed, 19 insertions, 11 deletions

diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.c b/rpc/rpc-lib/src/rpc-clnt-ping.c
index 845ce235a2b..26831ba11f3 100644
--- a/rpc/rpc-lib/src/rpc-clnt-ping.c
+++ b/rpc/rpc-lib/src/rpc-clnt-ping.c
@@ -147,7 +147,7 @@ rpc_clnt_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         } else {
                                 /* timer expired and transport bailed out */
                                 gf_log (this->name, GF_LOG_WARNING,
-                                        "timer must have expired");
+                                        "socket disconnected");
 
                         }
                         conn->ping_started = 0;
@@ -226,7 +226,8 @@ rpc_clnt_start_ping (void *rpc_ptr)
         conn = &rpc->conn;
 
         if (conn->ping_timeout == 0) {
-                gf_log (THIS->name, GF_LOG_INFO, "ping timeout is 0, returning");
+                gf_log (THIS->name, GF_LOG_DEBUG, "ping timeout is 0,"
+                        " returning");
                 return;
         }
 
diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.h b/rpc/rpc-lib/src/rpc-clnt-ping.h
index e7fbf3ced9d..4edc416cee9 100644
--- a/rpc/rpc-lib/src/rpc-clnt-ping.h
+++ b/rpc/rpc-lib/src/rpc-clnt-ping.h
@@ -14,5 +14,6 @@
 #include "config.h"
 #endif
 
+#define RPC_DEFAULT_PING_TIMEOUT 30
 void
 rpc_clnt_start_ping (void *rpc_ptr);
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 475afb49739..44324a80431 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -984,9 +984,11 @@ rpc_clnt_connection_init (struct rpc_clnt *clnt, glusterfs_ctx_t *ctx,
                 gf_log (name, GF_LOG_DEBUG,
                         "setting ping-timeout to %d", conn->ping_timeout);
         } else {
-                gf_log (name, GF_LOG_INFO,
-                        "defaulting ping-timeout to 30secs");
-                conn->ping_timeout = 30;
+                /*TODO: Once the epoll thread model is fixed,
+                  change the default ping-timeout to 30sec */
+                gf_log (name, GF_LOG_DEBUG,
+                        "disable ping-timeout");
+                conn->ping_timeout = 0;
         }
 
         trans = rpc_transport_load (ctx, options, name);
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index e98b5a948de..165fdd882b1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -2250,6 +2250,9 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                                               "%s", brick->brick_id);
                 if (!xl)
                         goto out;
+                ret = xlator_set_option (xl, "ping-timeout", "42");
+                if (ret)
+                        goto out;
                 ret = xlator_set_option (xl, "remote-host", brick->hostname);
                 if (ret)
                         goto out;
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index d89c1ef7bcf..a5927e66a96 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -40,6 +40,7 @@
 #include "glusterd-locks.h"
 #include "common-utils.h"
 #include "run.h"
+#include "rpc-clnt-ping.h"
 
 #include "syncop.h"
 
@@ -1623,10 +1624,10 @@ struct volume_options options[] = {
           .description = "directory where the bricks for the snapshots will be created"
         },
         { .key  = {"ping-timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 1,
-          .max  = 100,
-          .default_value = "30",
+          .type = GF_OPTION_TYPE_TIME,
+          .min  = 0,
+          .max  = 300,
+          .default_value = TOSTRING(RPC_DEFAULT_PING_TIMEOUT),
         },
         { .key   = {NULL} },
 };
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index aecd8f8fb07..e181292f433 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -21,7 +21,7 @@
 #include "statedump.h"
 #include "compat-errno.h"
 
-#include "xdr-rpc.h" 
+#include "xdr-rpc.h"
 #include "glusterfs3.h"
 
 extern rpc_clnt_prog_t clnt_handshake_prog;
@@ -2841,7 +2841,7 @@ struct volume_options options[] = {
         },
         { .key   = {"ping-timeout"},
           .type  = GF_OPTION_TYPE_TIME,
-          .min   = 1,
+          .min   = 0,
           .max   = 1013,
           .default_value = "42",
           .description = "Time duration for which the client waits to "