rpc-clnt*: several code changes to reduce conn lock times

Assorted code refactoring to reduce lock contention.
Also, took the opportunity to reorder structs more properly.
Removed dead code.

Hopefully, no functional changes.
Compile-tested only!

updates: bz#1193929
Signed-off-by: Yaniv Kaul <ykaul@redhat.com>

Change-Id: I5de6124ad071fd5e2c31832364d602b5f6d6fe28

3 files changed, 25 insertions, 55 deletions

diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.c b/rpc/rpc-lib/src/rpc-clnt-ping.c
index a98a83dd8c8..7c2026f9086 100644
--- a/rpc/rpc-lib/src/rpc-clnt-ping.c
+++ b/rpc/rpc-lib/src/rpc-clnt-ping.c
@@ -108,7 +108,6 @@ rpc_clnt_ping_timer_expired(void *rpc_ptr)
     rpc_transport_t *trans = NULL;
     rpc_clnt_connection_t *conn = NULL;
     int disconnect = 0;
-    int transport_activity = 0;
     struct timespec current = {
         0,
     };
@@ -123,28 +122,22 @@ rpc_clnt_ping_timer_expired(void *rpc_ptr)
         goto out;
     }
 
+    clock_gettime(CLOCK_REALTIME, &current);
     pthread_mutex_lock(&conn->lock);
     {
         unref = rpc_clnt_remove_ping_timer_locked(rpc);
 
-        clock_gettime(CLOCK_REALTIME, &current);
         if (((current.tv_sec - conn->last_received.tv_sec) <
              conn->ping_timeout) ||
             ((current.tv_sec - conn->last_sent.tv_sec) < conn->ping_timeout)) {
-            transport_activity = 1;
-        }
-
-        if (transport_activity) {
             gf_log(trans->name, GF_LOG_TRACE,
                    "ping timer expired but transport activity "
                    "detected - not bailing transport");
-
             if (__rpc_clnt_rearm_ping_timer(rpc, rpc_clnt_ping_timer_expired) ==
                 -1) {
                 gf_log(trans->name, GF_LOG_WARNING,
                        "unable to setup ping timer");
             }
-
         } else {
             conn->ping_started = 0;
             disconnect = 1;
@@ -198,14 +191,16 @@ rpc_clnt_ping_cbk(struct rpc_req *req, struct iovec *iov, int count,
     timespec_sub(&local->submit_time, &now, &delta);
     latency_msec = delta.tv_sec * 1000 + delta.tv_nsec / 1000000;
 
+    gf_log(THIS->name, GF_LOG_DEBUG, "Ping latency is %" PRIu64 "ms",
+           latency_msec);
+    call_notify = _gf_true;
+
     pthread_mutex_lock(&conn->lock);
     {
-        gf_log(THIS->name, GF_LOG_DEBUG, "Ping latency is %" PRIu64 "ms",
-               latency_msec);
-
-        call_notify = _gf_true;
+        unref = rpc_clnt_remove_ping_timer_locked(local->rpc);
         if (req->rpc_status == -1) {
-            unref = rpc_clnt_remove_ping_timer_locked(local->rpc);
+            conn->ping_started = 0;
+            pthread_mutex_unlock(&conn->lock);
             if (unref) {
                 gf_log(this->name, GF_LOG_WARNING,
                        "socket or ib related error");
@@ -214,19 +209,20 @@ rpc_clnt_ping_cbk(struct rpc_req *req, struct iovec *iov, int count,
                 /* timer expired and transport bailed out */
                 gf_log(this->name, GF_LOG_WARNING, "socket disconnected");
             }
-            conn->ping_started = 0;
-            goto unlock;
+            goto after_unlock;
         }
 
-        unref = rpc_clnt_remove_ping_timer_locked(local->rpc);
         if (__rpc_clnt_rearm_ping_timer(local->rpc, rpc_clnt_start_ping) ==
             -1) {
+            /* unlock before logging error */
+            pthread_mutex_unlock(&conn->lock);
             gf_log(this->name, GF_LOG_WARNING, "failed to set the ping timer");
+        } else {
+            /* just unlock the mutex */
+            pthread_mutex_unlock(&conn->lock);
         }
     }
-unlock:
-    pthread_mutex_unlock(&conn->lock);
-
+after_unlock:
     if (call_notify) {
         ret = local->rpc->notifyfn(local->rpc, this, RPC_CLNT_PING,
                                    (void *)(uintptr_t)latency_msec);
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 9b8e79070ec..56bde4ed8ab 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -67,13 +67,13 @@ __saved_frames_put(struct saved_frames *frames, void *frame,
     }
     /* THIS should be saved and set back */
 
-    memset(saved_frame, 0, sizeof(*saved_frame));
     INIT_LIST_HEAD(&saved_frame->list);
 
     saved_frame->capital_this = THIS;
     saved_frame->frame = frame;
     saved_frame->rpcreq = rpcreq;
     gettimeofday(&saved_frame->saved_at, NULL);
+    memset(&saved_frame->rsp, 0, sizeof(rpc_transport_rsp_t));
 
     if (_is_lock_fop(saved_frame))
         list_add_tail(&saved_frame->list, &frames->lk_sf.list);
@@ -780,8 +780,7 @@ is_rpc_clnt_disconnected(rpc_clnt_connection_t *conn)
 
     pthread_mutex_lock(&conn->lock);
     {
-        if (conn->disconnected == _gf_false)
-            disconnected = _gf_false;
+        disconnected = conn->disconnected;
     }
     pthread_mutex_unlock(&conn->lock);
 
@@ -1849,28 +1848,6 @@ rpc_clnt_unref(struct rpc_clnt *rpc)
     return rpc;
 }
 
-char
-rpc_clnt_is_disabled(struct rpc_clnt *rpc)
-{
-    rpc_clnt_connection_t *conn = NULL;
-    char disabled = 0;
-
-    if (!rpc) {
-        goto out;
-    }
-
-    conn = &rpc->conn;
-
-    pthread_mutex_lock(&conn->lock);
-    {
-        disabled = rpc->disabled;
-    }
-    pthread_mutex_unlock(&conn->lock);
-
-out:
-    return disabled;
-}
-
 void
 rpc_clnt_disable(struct rpc_clnt *rpc)
 {
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index 28b6f0e7f4d..d122382e446 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -133,24 +133,23 @@ struct rpc_clnt_connection {
     gf_timer_t *timer;
     gf_timer_t *ping_timer;
     struct rpc_clnt *rpc_clnt;
-    char connected;
-    gf_boolean_t disconnected;
     struct saved_frames *saved_frames;
-    int32_t frame_timeout;
     struct timespec last_sent;
     struct timespec last_received;
-    int32_t ping_started;
-    char *name;
-    int32_t ping_timeout;
     uint64_t pingcnt;
     uint64_t msgcnt;
     uint64_t cleanup_gen;
+    char *name;
+    int32_t ping_started;
+    int32_t frame_timeout;
+    int32_t ping_timeout;
+    gf_boolean_t disconnected;
+    char connected;
 };
 typedef struct rpc_clnt_connection rpc_clnt_connection_t;
 
 struct rpc_req {
     rpc_clnt_connection_t *conn;
-    uint32_t xid;
     struct iovec req[2];
     int reqcnt;
     struct iobref *req_iobref;
@@ -163,6 +162,7 @@ struct rpc_req {
     int procnum;
     fop_cbk_fn_t cbkfn;
     void *conn_private;
+    uint32_t xid;
 };
 
 typedef struct rpc_clnt {
@@ -183,8 +183,8 @@ typedef struct rpc_clnt {
     glusterfs_ctx_t *ctx;
     gf_atomic_t refcount;
     int auth_value;
-    char disabled;
     xlator_t *owner;
+    char disabled;
 } rpc_clnt_t;
 
 struct rpc_clnt *
@@ -256,9 +256,6 @@ rpc_clnt_disable(struct rpc_clnt *rpc);
 void
 rpc_clnt_disconnect(struct rpc_clnt *rpc);
 
-char
-rpc_clnt_is_disabled(struct rpc_clnt *rpc);
-
 int
 rpc_clnt_mgmt_pmap_signout(glusterfs_ctx_t *ctx, char *brick_name);