glusterd: Ping timer implmentation

This patch refactors the existing client ping timer implementation, and makes
use of the common code for implementing both client ping timer and the
glusterd ping timer.

A new gluster rpc program for ping is introduced. The ping timer is only
started for peers that have this new program. The deafult glusterd ping
timeout is 30 seconds. It is configurable by setting the option
'ping-timeout' in glusterd.vol .

Also, this patch introduces changes in the glusterd-handshake path. The client
programs for a peer are now set in the callback of dump_versions, for both
the older handshake and the newer op-version handshake. This is the only place
in the handshake process where we know what programs a peer supports.

Change-Id: I035815ac13449ca47080ecc3253c0a9afbe9016a
BUG: 1038261
Signed-off-by: Vijaikumar M <vmallika@redhat.com>
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/5202
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

19 files changed, 351 insertions, 306 deletions

diff --git a/extras/glusterd.vol b/extras/glusterd.vol
index 9bac52ab7fe..fe2e3e2c772 100644
--- a/extras/glusterd.vol
+++ b/extras/glusterd.vol
@@ -5,5 +5,6 @@ volume management
     option transport.socket.keepalive-time 10
     option transport.socket.keepalive-interval 2
     option transport.socket.read-fail-log off
+    option ping-timeout 30
 #   option base-port 49152
 end-volume
diff --git a/rpc/rpc-lib/src/Makefile.am b/rpc/rpc-lib/src/Makefile.am
index ee2f27b259b..6a098c9a8ce 100644
--- a/rpc/rpc-lib/src/Makefile.am
+++ b/rpc/rpc-lib/src/Makefile.am
@@ -2,13 +2,13 @@ lib_LTLIBRARIES = libgfrpc.la
 
 libgfrpc_la_SOURCES = auth-unix.c rpcsvc-auth.c rpcsvc.c auth-null.c \
 	rpc-transport.c xdr-rpc.c xdr-rpcclnt.c rpc-clnt.c auth-glusterfs.c \
-	rpc-drc.c $(CONTRIBDIR)/sunrpc/xdr_sizeof.c
+	rpc-drc.c $(CONTRIBDIR)/sunrpc/xdr_sizeof.c  rpc-clnt-ping.c
 
 libgfrpc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 libgfrpc_la_LDFLAGS = -version-info $(LIBGFRPC_LT_VERSION) $(GF_LDFLAGS)
 
 noinst_HEADERS = rpcsvc.h rpc-transport.h xdr-common.h xdr-rpc.h xdr-rpcclnt.h \
-	rpc-clnt.h rpcsvc-common.h protocol-common.h rpc-drc.h
+	rpc-clnt.h rpcsvc-common.h protocol-common.h rpc-drc.h rpc-clnt-ping.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/rpc/xdr/src \
diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.c b/rpc/rpc-lib/src/rpc-clnt-ping.c
new file mode 100644
index 00000000000..b3bd5e11deb
--- /dev/null
+++ b/rpc/rpc-lib/src/rpc-clnt-ping.c
@@ -0,0 +1,266 @@
+/*
+  Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "rpc-clnt.h"
+#include "rpc-clnt-ping.h"
+#include "byte-order.h"
+#include "xdr-rpcclnt.h"
+#include "rpc-transport.h"
+#include "protocol-common.h"
+#include "mem-pool.h"
+#include "xdr-rpc.h"
+#include "rpc-common-xdr.h"
+
+
+char *clnt_ping_procs[GF_DUMP_MAXVALUE] = {
+        [GF_DUMP_PING] = "NULL",
+};
+struct rpc_clnt_program clnt_ping_prog = {
+        .progname  = "GF-DUMP",
+        .prognum   = GLUSTER_DUMP_PROGRAM,
+        .progver   = GLUSTER_DUMP_VERSION,
+        .procnames = clnt_ping_procs,
+};
+
+void
+rpc_clnt_ping_timer_expired (void *rpc_ptr)
+{
+        struct rpc_clnt         *rpc                = NULL;
+        rpc_transport_t         *trans              = NULL;
+        rpc_clnt_connection_t   *conn               = NULL;
+        int                      disconnect         = 0;
+        int                      transport_activity = 0;
+        struct timespec          timeout            = {0, };
+        struct timeval           current            = {0, };
+
+        rpc = (struct rpc_clnt*) rpc_ptr;
+        conn = &rpc->conn;
+        trans = conn->trans;
+
+        if (!trans) {
+                gf_log ("ping-timer", GF_LOG_WARNING,
+                        "transport not initialized");
+                goto out;
+        }
+
+        pthread_mutex_lock (&conn->lock);
+        {
+                if (conn->ping_timer) {
+                        gf_timer_call_cancel (rpc->ctx,
+                                              conn->ping_timer);
+                        conn->ping_timer = NULL;
+                        rpc_clnt_unref (rpc);
+                }
+                gettimeofday (&current, NULL);
+
+                if (((current.tv_sec - conn->last_received.tv_sec) <
+                     conn->ping_timeout)
+                    || ((current.tv_sec - conn->last_sent.tv_sec) <
+                        conn->ping_timeout)) {
+                        transport_activity = 1;
+                }
+
+                if (transport_activity) {
+                        gf_log (trans->name, GF_LOG_TRACE,
+                                "ping timer expired but transport activity "
+                                "detected - not bailing transport");
+                        timeout.tv_sec = conn->ping_timeout;
+                        timeout.tv_nsec = 0;
+
+                        rpc_clnt_ref (rpc);
+                        conn->ping_timer =
+                                gf_timer_call_after (rpc->ctx, timeout,
+                                                     rpc_clnt_ping_timer_expired,
+                                                     (void *) rpc);
+                        if (conn->ping_timer == NULL) {
+                                gf_log (trans->name, GF_LOG_WARNING,
+                                        "unable to setup ping timer");
+                                rpc_clnt_unref (rpc);
+                        }
+                } else {
+                        conn->ping_started = 0;
+                        disconnect = 1;
+                }
+        }
+        pthread_mutex_unlock (&conn->lock);
+
+        if (disconnect) {
+                gf_log (trans->name, GF_LOG_CRITICAL,
+                        "server %s has not responded in the last %d "
+                        "seconds, disconnecting.",
+                        trans->peerinfo.identifier,
+                        conn->ping_timeout);
+
+                rpc_transport_disconnect (conn->trans);
+        }
+
+out:
+        return;
+}
+
+int
+rpc_clnt_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+        struct rpc_clnt       *rpc     = NULL;
+        xlator_t              *this    = NULL;
+        rpc_clnt_connection_t *conn    = NULL;
+        call_frame_t          *frame   = NULL;
+
+        if (!myframe) {
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "frame with the request is NULL");
+                goto out;
+        }
+
+        frame = myframe;
+        this = frame->this;
+        rpc  = frame->local;
+        frame->local = NULL; /* Prevent STACK_DESTROY from segfaulting */
+        conn = &rpc->conn;
+
+        if (req->rpc_status == -1) {
+                pthread_mutex_lock (&conn->lock);
+                {
+                        if (conn->ping_timer != NULL) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "socket or ib related error");
+                                gf_timer_call_cancel (rpc->ctx,
+                                                      conn->ping_timer);
+                                conn->ping_timer = NULL;
+                                rpc_clnt_unref (rpc);
+                        } else {
+                                /* timer expired and transport bailed out */
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "timer must have expired");
+                        }
+                        conn->ping_started = 0;
+                }
+                pthread_mutex_unlock (&conn->lock);
+        }
+out:
+        if (frame)
+                STACK_DESTROY (frame->root);
+        if (rpc)
+                rpc_clnt_unref (rpc);
+        return 0;
+}
+
+int
+rpc_clnt_ping (struct rpc_clnt *rpc)
+{
+        call_frame_t *frame = NULL;
+        int32_t       ret   = -1;
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (!frame)
+                goto fail;
+
+        frame->local = rpc_clnt_ref (rpc);
+
+        ret = rpc_clnt_submit (rpc, &clnt_ping_prog,
+                               GF_DUMP_PING, rpc_clnt_ping_cbk, NULL, 0,
+                               NULL, 0, NULL, frame, NULL, 0, NULL, 0, NULL);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to start ping timer");
+        }
+
+        return ret;
+
+fail:
+        if (frame) {
+                STACK_DESTROY (frame->root);
+        }
+
+        return ret;
+
+}
+
+void
+rpc_clnt_start_ping (void *rpc_ptr)
+{
+        struct rpc_clnt         *rpc         = NULL;
+        rpc_clnt_connection_t   *conn        = NULL;
+        struct timespec          timeout     = {0, };
+        int                      frame_count = 0;
+
+        rpc = (struct rpc_clnt*) rpc_ptr;
+        conn = &rpc->conn;
+
+        if (conn->ping_timeout == 0) {
+                gf_log (THIS->name, GF_LOG_INFO, "ping timeout is 0, returning");
+                return;
+        }
+
+        pthread_mutex_lock (&conn->lock);
+        {
+                if (conn->ping_started) {
+                        pthread_mutex_unlock (&conn->lock);
+                        return;
+                }
+
+                if (conn->ping_timer) {
+                        gf_timer_call_cancel (rpc->ctx, conn->ping_timer);
+                        conn->ping_timer = NULL;
+                        rpc_clnt_unref (rpc);
+                }
+
+                if (conn->saved_frames)
+                        /* treat the case where conn->saved_frames is NULL
+                           as no pending frames */
+                        frame_count = conn->saved_frames->count;
+
+                if ((frame_count == 0) || !conn->connected) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "returning as transport is already disconnected"
+                                " OR there are no frames (%d || %d)",
+                                frame_count, !conn->connected);
+
+                        pthread_mutex_unlock (&conn->lock);
+                        return;
+                }
+
+                if (frame_count < 0) {
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "saved_frames->count is %"PRId64,
+                                conn->saved_frames->count);
+                        conn->saved_frames->count = 0;
+                }
+
+                timeout.tv_sec = conn->ping_timeout;
+                timeout.tv_nsec = 0;
+
+                rpc_clnt_ref (rpc);
+                conn->ping_timer =
+                        gf_timer_call_after (rpc->ctx, timeout,
+                                             rpc_clnt_ping_timer_expired,
+                                             (void *) rpc);
+
+                if (conn->ping_timer == NULL) {
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "unable to setup ping timer");
+                        rpc_clnt_unref (rpc);
+                        pthread_mutex_unlock (&conn->lock);
+                        return;
+                } else {
+                        conn->ping_started = 1;
+                }
+        }
+        pthread_mutex_unlock (&conn->lock);
+
+        rpc_clnt_ping(rpc);
+}
diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.h b/rpc/rpc-lib/src/rpc-clnt-ping.h
new file mode 100644
index 00000000000..e7fbf3ced9d
--- /dev/null
+++ b/rpc/rpc-lib/src/rpc-clnt-ping.h
@@ -0,0 +1,18 @@
+/*
+  Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+void
+rpc_clnt_start_ping (void *rpc_ptr);
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 3106342e970..46ee0eebcd0 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -17,6 +17,7 @@
 #define RPC_CLNT_DEFAULT_REQUEST_COUNT 512
 
 #include "rpc-clnt.h"
+#include "rpc-clnt-ping.h"
 #include "byte-order.h"
 #include "xdr-rpcclnt.h"
 #include "rpc-transport.h"
@@ -552,6 +553,7 @@ rpc_clnt_connection_cleanup (rpc_clnt_connection_t *conn)
                         gf_timer_call_cancel (clnt->ctx, conn->ping_timer);
                         conn->ping_timer = NULL;
                         conn->ping_started = 0;
+                        rpc_clnt_unref (clnt);
                 }
         }
         pthread_mutex_unlock (&conn->lock);
@@ -1000,6 +1002,17 @@ rpc_clnt_connection_init (struct rpc_clnt *clnt, glusterfs_ctx_t *ctx,
         }
         conn->rpc_clnt = clnt;
 
+        ret = dict_get_int32 (options, "ping-timeout",
+                              &conn->ping_timeout);
+        if (ret >= 0) {
+                gf_log (name, GF_LOG_DEBUG,
+                        "setting ping-timeout to %d", conn->ping_timeout);
+        } else {
+                gf_log (name, GF_LOG_INFO,
+                        "defaulting ping-timeout to 30secs");
+                conn->ping_timeout = 30;
+        }
+
         trans = rpc_transport_load (ctx, options, name);
         if (!trans) {
                 gf_log (name, GF_LOG_WARNING, "loading of new rpc-transport"
@@ -1592,6 +1605,7 @@ rpc_clnt_submit (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
                 goto out;
         }
 
+        rpc_clnt_start_ping (rpc);
         ret = 0;
 
 out:
@@ -1734,6 +1748,7 @@ rpc_clnt_disable (struct rpc_clnt *rpc)
                         gf_timer_call_cancel (rpc->ctx, conn->ping_timer);
                         conn->ping_timer = NULL;
                         conn->ping_started = 0;
+                        rpc_clnt_unref (rpc);
                 }
                 trans = conn->trans;
                 conn->trans = NULL;
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index bd4820605d8..f439d538761 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -145,6 +145,7 @@ struct rpc_clnt_connection {
 	struct timeval           last_received;
 	int32_t                  ping_started;
         char                    *name;
+	int32_t                  ping_timeout;
 };
 typedef struct rpc_clnt_connection rpc_clnt_connection_t;
 
@@ -186,7 +187,6 @@ typedef struct rpc_clnt {
         char                  disabled;
 } rpc_clnt_t;
 
-
 struct rpc_clnt *rpc_clnt_new (dict_t *options, glusterfs_ctx_t *ctx,
                                char *name, uint32_t reqpool_size);
 
diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index 1c13048f223..be9f9a861f0 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -1844,6 +1844,32 @@ out:
 }
 
 static int
+rpcsvc_ping (rpcsvc_request_t *req)
+{
+        char          rsp_buf[8 * 1024] = {0,};
+        gf_common_rsp rsp               = {0,};
+        struct iovec  iov               = {0,};
+        int           ret               = -1;
+        uint32_t      ping_rsp_len      = 0;
+
+        ping_rsp_len = xdr_sizeof ((xdrproc_t) xdr_gf_common_rsp,
+                                   &rsp);
+
+        iov.iov_base = rsp_buf;
+        iov.iov_len  = ping_rsp_len;
+
+        ret = xdr_serialize_generic (iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+        if (ret < 0) {
+                ret = RPCSVC_ACTOR_ERROR;
+        } else {
+                rsp.op_ret = 0;
+                rpcsvc_submit_generic (req, &iov, 1, NULL, 0, NULL);
+        }
+
+        return 0;
+}
+
+static int
 rpcsvc_dump (rpcsvc_request_t *req)
 {
         char         rsp_buf[8 * 1024] = {0,};
@@ -2585,6 +2611,7 @@ out:
 rpcsvc_actor_t gluster_dump_actors[] = {
         [GF_DUMP_NULL]      = {"NULL",     GF_DUMP_NULL,     NULL,        NULL, 0, DRC_NA},
         [GF_DUMP_DUMP]      = {"DUMP",     GF_DUMP_DUMP,     rpcsvc_dump, NULL, 0, DRC_NA},
+        [GF_DUMP_PING]      = {"PING",     GF_DUMP_PING,     rpcsvc_ping, NULL, 0, DRC_NA},
         [GF_DUMP_MAXVALUE]  = {"MAXVALUE", GF_DUMP_MAXVALUE, NULL,        NULL, 0, DRC_NA},
 };
 
@@ -2594,5 +2621,5 @@ struct rpcsvc_program gluster_dump_prog = {
         .prognum   = GLUSTER_DUMP_PROGRAM,
         .progver   = GLUSTER_DUMP_VERSION,
         .actors    = gluster_dump_actors,
-        .numactors = 2,
+        .numactors = sizeof (gluster_dump_actors) / sizeof (gluster_dump_actors[0]) - 1,
 };
diff --git a/rpc/rpc-lib/src/xdr-common.h b/rpc/rpc-lib/src/xdr-common.h
index 631f4b284ad..39b8f240a1e 100644
--- a/rpc/rpc-lib/src/xdr-common.h
+++ b/rpc/rpc-lib/src/xdr-common.h
@@ -29,6 +29,7 @@
 enum gf_dump_procnum {
         GF_DUMP_NULL,
         GF_DUMP_DUMP,
+        GF_DUMP_PING,
         GF_DUMP_MAXVALUE,
 };
 
diff --git a/rpc/xdr/src/glusterfs3-xdr.x b/rpc/xdr/src/glusterfs3-xdr.x
index feef7e2649e..72f528e0a1c 100644
--- a/rpc/xdr/src/glusterfs3-xdr.x
+++ b/rpc/xdr/src/glusterfs3-xdr.x
@@ -692,12 +692,6 @@ struct gfs3_release_req {
         opaque   xdata<>; /* Extra data */
 }  ;
 
-struct gf_common_rsp {
-       int    op_ret;
-       int    op_errno;
-        opaque   xdata<>; /* Extra data */
-} ;
-
 struct gfs3_dirlist {
        u_quad_t d_ino;
        u_quad_t d_off;
diff --git a/rpc/xdr/src/rpc-common-xdr.x b/rpc/xdr/src/rpc-common-xdr.x
index ee66500e032..e94077543b7 100644
--- a/rpc/xdr/src/rpc-common-xdr.x
+++ b/rpc/xdr/src/rpc-common-xdr.x
@@ -37,3 +37,10 @@ struct gf_dump_rsp {
 	int op_errno;
 	struct gf_prog_detail *prog;
 };
+
+
+struct gf_common_rsp {
+       int    op_ret;
+       int    op_errno;
+       opaque   xdata<>; /* Extra data */
+} ;
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index c840803fbf0..0a43fd69ae5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -4260,6 +4260,7 @@ __glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
         switch (event) {
         case RPC_CLNT_CONNECT:
         {
+                rpc_clnt_set_connected (&rpc->conn);
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
                 peerinfo->connected = 1;
                 peerinfo->quorum_action = _gf_true;
@@ -4272,6 +4273,7 @@ __glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
 
         case RPC_CLNT_DISCONNECT:
         {
+                rpc_clnt_unset_connected (&rpc->conn);
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_DISCONNECT %d",
                         peerinfo->state.state);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 1a6aa81d30c..691d410cd44 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1648,5 +1648,11 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_STR,
           .description = "directory where the bricks for the snapshots will be created"
         },
+        { .key  = {"ping-timeout"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 1,
+          .max  = 100,
+          .default_value = "30",
+        },
         { .key   = {NULL} },
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 9e3eb417929..cbcaecd919d 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -179,6 +179,7 @@ typedef struct {
         char                      *snap_bricks_directory;
         gf_store_handle_t         *missed_snaps_list_shandle;
         struct list_head           missed_snaps_list;
+        int           ping_timeout;
 } glusterd_conf_t;
 
 
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 85b0f757bf4..cf91c3b723c 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -29,9 +29,6 @@
 extern rpc_clnt_prog_t clnt3_3_fop_prog;
 extern rpc_clnt_prog_t clnt_pmap_prog;
 
-int client_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                     void *myframe);
-
 int client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
                                int count, void *myframe);
 
@@ -44,265 +41,6 @@ typedef struct client_fd_lk_local {
         clnt_fd_ctx_t *fdctx;
 }clnt_fd_lk_local_t;
 
-/* Handshake */
-
-void
-rpc_client_ping_timer_expired (void *data)
-{
-        rpc_transport_t         *trans              = NULL;
-        rpc_clnt_connection_t   *conn               = NULL;
-        int                      disconnect         = 0;
-        int                      transport_activity = 0;
-        struct timespec          timeout            = {0, };
-        struct timeval           current            = {0, };
-        struct rpc_clnt         *clnt               = NULL;
-        xlator_t                *this               = NULL;
-        clnt_conf_t             *conf               = NULL;
-
-        this = data;
-
-        if (!this || !this->private) {
-                gf_log (THIS->name, GF_LOG_WARNING, "xlator initialization not done");
-                goto out;
-        }
-
-        conf = this->private;
-
-        clnt = conf->rpc;
-        if (!clnt) {
-                gf_log (this->name, GF_LOG_WARNING, "rpc not initialized");
-                goto out;
-        }
-
-        conn = &clnt->conn;
-        trans = conn->trans;
-
-        if (!trans) {
-                gf_log (this->name, GF_LOG_WARNING, "transport not initialized");
-                goto out;
-        }
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                if (conn->ping_timer)
-                        gf_timer_call_cancel (this->ctx,
-                                              conn->ping_timer);
-                gettimeofday (&current, NULL);
-
-                if (((current.tv_sec - conn->last_received.tv_sec) <
-                     conf->opt.ping_timeout)
-                    || ((current.tv_sec - conn->last_sent.tv_sec) <
-                        conf->opt.ping_timeout)) {
-                        transport_activity = 1;
-                }
-
-                if (transport_activity) {
-                        gf_log (trans->name, GF_LOG_TRACE,
-                                "ping timer expired but transport activity "
-                                "detected - not bailing transport");
-                        timeout.tv_sec = conf->opt.ping_timeout;
-                        timeout.tv_nsec = 0;
-
-                        conn->ping_timer =
-                                gf_timer_call_after (this->ctx, timeout,
-                                                     rpc_client_ping_timer_expired,
-                                                     (void *) this);
-                        if (conn->ping_timer == NULL)
-                                gf_log (trans->name, GF_LOG_WARNING,
-                                        "unable to setup ping timer");
-
-                } else {
-                        conn->ping_started = 0;
-                        conn->ping_timer = NULL;
-                        disconnect = 1;
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        if (disconnect) {
-                gf_log (trans->name, GF_LOG_CRITICAL,
-                        "server %s has not responded in the last %d "
-                        "seconds, disconnecting.",
-                        conn->trans->peerinfo.identifier,
-                        conf->opt.ping_timeout);
-
-                rpc_transport_disconnect (conn->trans);
-        }
-
-out:
-        return;
-}
-
-void
-client_start_ping (void *data)
-{
-        xlator_t                *this        = NULL;
-        clnt_conf_t             *conf        = NULL;
-        rpc_clnt_connection_t   *conn        = NULL;
-        int32_t                  ret         = -1;
-        struct timespec          timeout     = {0, };
-        call_frame_t            *frame       = NULL;
-        int                      frame_count = 0;
-
-        this = data;
-        if (!this || !this->private) {
-                gf_log (THIS->name, GF_LOG_WARNING, "xlator not initialized");
-                goto fail;
-        }
-
-        conf  = this->private;
-        if (!conf->rpc) {
-                gf_log (this->name, GF_LOG_WARNING, "rpc not initialized");
-                goto fail;
-        }
-        conn = &conf->rpc->conn;
-
-        if (conf->opt.ping_timeout == 0) {
-                gf_log (this->name, GF_LOG_INFO, "ping timeout is 0, returning");
-                return;
-        }
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                if (conn->ping_timer)
-                        gf_timer_call_cancel (this->ctx, conn->ping_timer);
-
-                conn->ping_timer = NULL;
-                conn->ping_started = 0;
-
-                if (conn->saved_frames)
-                        /* treat the case where conn->saved_frames is NULL
-                           as no pending frames */
-                        frame_count = conn->saved_frames->count;
-
-                if ((frame_count == 0) || !conn->connected) {
-                        /* using goto looked ugly here,
-                         * hence getting out this way */
-                        /* unlock */
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "returning as transport is already disconnected"
-                                " OR there are no frames (%d || %d)",
-                                frame_count, !conn->connected);
-
-                        pthread_mutex_unlock (&conn->lock);
-                        return;
-                }
-
-                if (frame_count < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "saved_frames->count is %"PRId64,
-                                conn->saved_frames->count);
-                        conn->saved_frames->count = 0;
-                }
-
-                timeout.tv_sec = conf->opt.ping_timeout;
-                timeout.tv_nsec = 0;
-
-                conn->ping_timer =
-                        gf_timer_call_after (this->ctx, timeout,
-                                             rpc_client_ping_timer_expired,
-                                             (void *) this);
-
-                if (conn->ping_timer == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "unable to setup ping timer");
-                } else {
-                        conn->ping_started = 1;
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto fail;
-
-        ret = client_submit_request (this, NULL, frame, conf->handshake,
-                                     GF_HNDSK_PING, client_ping_cbk, NULL,
-                                     NULL, 0, NULL, 0, NULL, (xdrproc_t)NULL);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to start ping timer");
-        }
-
-        return;
-
-fail:
-        if (frame) {
-                STACK_DESTROY (frame->root);
-        }
-
-        return;
-}
-
-
-int
-client_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                 void *myframe)
-{
-        xlator_t              *this    = NULL;
-        rpc_clnt_connection_t *conn    = NULL;
-        struct timespec        timeout = {0, };
-        call_frame_t          *frame   = NULL;
-        clnt_conf_t           *conf    = NULL;
-
-        if (!myframe) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "frame with the request is NULL");
-                goto out;
-        }
-        frame = myframe;
-        this = frame->this;
-        if (!this || !this->private) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "xlator private is not set");
-                goto out;
-        }
-
-        conf = this->private;
-        conn = &conf->rpc->conn;
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                if (req->rpc_status == -1) {
-                        if (conn->ping_timer != NULL) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "socket or ib related error");
-                                gf_timer_call_cancel (this->ctx,
-                                                      conn->ping_timer);
-                                conn->ping_timer = NULL;
-                        } else {
-                                /* timer expired and transport bailed out */
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "timer must have expired");
-                        }
-
-                        goto unlock;
-                }
-
-
-                timeout.tv_sec  = conf->opt.ping_timeout;
-                timeout.tv_nsec = 0;
-
-                gf_timer_call_cancel (this->ctx,
-                                      conn->ping_timer);
-
-                conn->ping_timer =
-                        gf_timer_call_after (this->ctx, timeout,
-                                             client_start_ping, (void *)this);
-
-                if (conn->ping_timer == NULL)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set the ping timer");
-        }
-unlock:
-        pthread_mutex_unlock (&conn->lock);
-out:
-        if (frame)
-                STACK_DESTROY (frame->root);
-        return 0;
-}
-
-
 int
 client3_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
diff --git a/xlators/protocol/client/src/client-rpc-fops.c b/xlators/protocol/client/src/client-rpc-fops.c
index 99a6f6d7492..e77da7a27d6 100644
--- a/xlators/protocol/client/src/client-rpc-fops.c
+++ b/xlators/protocol/client/src/client-rpc-fops.c
@@ -14,12 +14,12 @@
 #endif
 
 #include "client.h"
+#include "rpc-common-xdr.h"
 #include "glusterfs3-xdr.h"
 #include "glusterfs3.h"
 #include "compat-errno.h"
 
 int32_t client3_getspec (call_frame_t *frame, xlator_t *this, void *data);
-void client_start_ping (void *data);
 rpc_clnt_prog_t clnt3_3_fop_prog;
 
 
@@ -35,13 +35,10 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
         struct iovec    iov        = {0, };
         struct iobuf   *iobuf      = NULL;
         int             count      = 0;
-        int             start_ping = 0;
         struct iobref  *new_iobref = NULL;
         ssize_t         xdr_size   = 0;
         struct rpc_req  rpcreq     = {0, };
 
-        start_ping = 0;
-
         conf = this->private;
 
         if (req && xdrproc) {
@@ -95,19 +92,6 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
                 gf_log (this->name, GF_LOG_DEBUG, "rpc_clnt_submit failed");
         }
 
-        if (ret == 0) {
-                pthread_mutex_lock (&conf->rpc->conn.lock);
-                {
-                        if (!conf->rpc->conn.ping_started) {
-                                start_ping = 1;
-                        }
-                }
-                pthread_mutex_unlock (&conf->rpc->conn.lock);
-        }
-
-        if (start_ping)
-                client_start_ping ((void *) this);
-
         if (new_iobref)
                 iobref_unref (new_iobref);
 
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 98fb10c34eb..7726c0b8445 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -29,7 +29,6 @@ extern rpc_clnt_prog_t clnt_dump_prog;
 extern struct rpcclnt_cb_program gluster_cbk_prog;
 
 int client_handshake (xlator_t *this, struct rpc_clnt *rpc);
-void client_start_ping (void *data);
 int client_init_rpc (xlator_t *this);
 int client_destroy_rpc (xlator_t *this);
 int client_mark_fd_bad (xlator_t *this);
@@ -156,7 +155,6 @@ client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
         struct iovec    iov        = {0, };
         struct iobuf   *iobuf      = NULL;
         int             count      = 0;
-        char            start_ping = 0;
         struct iobref  *new_iobref = NULL;
         ssize_t         xdr_size   = 0;
         struct rpc_req  rpcreq     = {0, };
@@ -235,19 +233,6 @@ client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
                 gf_log (this->name, GF_LOG_DEBUG, "rpc_clnt_submit failed");
         }
 
-        if (ret == 0) {
-                pthread_mutex_lock (&conf->rpc->conn.lock);
-                {
-                        if (!conf->rpc->conn.ping_started) {
-                                start_ping = 1;
-                        }
-                }
-                pthread_mutex_unlock (&conf->rpc->conn.lock);
-        }
-
-        if (start_ping)
-                client_start_ping ((void *) this);
-
         ret = 0;
 
         if (new_iobref)
diff --git a/xlators/protocol/server/src/server-handshake.c b/xlators/protocol/server/src/server-handshake.c
index a459f89e182..f3ae96ef2ae 100644
--- a/xlators/protocol/server/src/server-handshake.c
+++ b/xlators/protocol/server/src/server-handshake.c
@@ -16,6 +16,7 @@
 
 #include "server.h"
 #include "server-helpers.h"
+#include "rpc-common-xdr.h"
 #include "glusterfs3-xdr.h"
 #include "compat-errno.h"
 #include "glusterfs3.h"
diff --git a/xlators/protocol/server/src/server-rpc-fops.c b/xlators/protocol/server/src/server-rpc-fops.c
index ebe62ae7c42..ce844197d90 100644
--- a/xlators/protocol/server/src/server-rpc-fops.c
+++ b/xlators/protocol/server/src/server-rpc-fops.c
@@ -18,6 +18,7 @@
 
 #include "server.h"
 #include "server-helpers.h"
+#include "rpc-common-xdr.h"
 #include "glusterfs3-xdr.h"
 #include "glusterfs3.h"
 #include "compat-errno.h"
diff --git a/xlators/protocol/server/src/server.h b/xlators/protocol/server/src/server.h
index 782327d775b..c5ab579140d 100644
--- a/xlators/protocol/server/src/server.h
+++ b/xlators/protocol/server/src/server.h
@@ -174,8 +174,6 @@ struct _server_state {
 
 extern struct rpcsvc_program gluster_handshake_prog;
 extern struct rpcsvc_program glusterfs3_3_fop_prog;
-extern struct rpcsvc_program gluster_ping_prog;
-
 
 typedef struct _server_ctx {
         gf_lock_t            fdtable_lock;