diff options
| author | Mohit Agrawal <moagrawa@redhat.com> | 2017-08-14 10:15:45 +0530 |
|---|---|---|
| committer | Jeff Darcy <jeff@pl.atyp.us> | 2017-11-28 13:02:06 +0000 |
| commit | 04fb94c99160ae1c6dea02624fefd47eb48da810 (patch) | |
| tree | 43dcdc56e7834734523a899a810a3544ef5f4dc9 | |
| parent | 0805771ad3e676224760a8bbfe7a265d0e132a7e (diff) | |
rpc: Eliminate conn->lock contention by using more granular locks
rpc_clnt_submit() acquires conn->lock before call to
rpc_transport_submit_request() and subsequent queuing of frame into
saved_frames list. However, as part of handling RPC_TRANSPORT_MSG_RECEIVED
and RPC_TRANSPORT_MSG_SENT notifications in rpc_clnt_notify(), conn->lock
is again used to atomically update conn->last_received and conn->last_sent
event timestamps.
So when conn->lock is acquired as part of submitting a request,
a parallel POLLIN notification gets blocked at rpc layer until the request
submission completes and the lock is released.
To get around this, this patch call clock_gettime (instead to call gettimeofday)
to update event timestamps in conn->last_received and conn->last_sent and to
call clock_gettime don't need to call mutex_lock because it (clock_gettime)
is thread safe call.
Note: Run fio on vm after apply the patch, iops is improved after apply
the patch.
Change-Id: I347b5031d61c426b276bc5e07136a7172645d763
BUG: 1467614
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>
| -rw-r--r-- | rpc/rpc-lib/src/rpc-clnt-ping.c | 4 | ||||
| -rw-r--r-- | rpc/rpc-lib/src/rpc-clnt.c | 12 | ||||
| -rw-r--r-- | rpc/rpc-lib/src/rpc-clnt.h | 4 |
3 files changed, 6 insertions, 14 deletions
diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.c b/rpc/rpc-lib/src/rpc-clnt-ping.c index 2f11341b0d5..40077e7aefd 100644 --- a/rpc/rpc-lib/src/rpc-clnt-ping.c +++ b/rpc/rpc-lib/src/rpc-clnt-ping.c @@ -112,7 +112,7 @@ rpc_clnt_ping_timer_expired (void *rpc_ptr) rpc_clnt_connection_t *conn = NULL; int disconnect = 0; int transport_activity = 0; - struct timeval current = {0, }; + struct timespec current = {0, }; int unref = 0; rpc = (struct rpc_clnt*) rpc_ptr; @@ -129,7 +129,7 @@ rpc_clnt_ping_timer_expired (void *rpc_ptr) { unref = rpc_clnt_remove_ping_timer_locked (rpc); - gettimeofday (¤t, NULL); + clock_gettime (CLOCK_REALTIME, ¤t); if (((current.tv_sec - conn->last_received.tv_sec) < conn->ping_timeout) || ((current.tv_sec - conn->last_sent.tv_sec) < diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c index adb8e3d4a60..cb59408dcf6 100644 --- a/rpc/rpc-lib/src/rpc-clnt.c +++ b/rpc/rpc-lib/src/rpc-clnt.c @@ -965,11 +965,7 @@ rpc_clnt_notify (rpc_transport_t *trans, void *mydata, case RPC_TRANSPORT_MSG_RECEIVED: { - pthread_mutex_lock (&conn->lock); - { - gettimeofday (&conn->last_received, NULL); - } - pthread_mutex_unlock (&conn->lock); + clock_gettime (CLOCK_REALTIME, &conn->last_received); pollin = data; if (pollin->is_reply) @@ -984,11 +980,7 @@ rpc_clnt_notify (rpc_transport_t *trans, void *mydata, case RPC_TRANSPORT_MSG_SENT: { - pthread_mutex_lock (&conn->lock); - { - gettimeofday (&conn->last_sent, NULL); - } - pthread_mutex_unlock (&conn->lock); + clock_gettime (CLOCK_REALTIME, &conn->last_sent); ret = 0; break; diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h index 952ecbce1a1..428ffb80650 100644 --- a/rpc/rpc-lib/src/rpc-clnt.h +++ b/rpc/rpc-lib/src/rpc-clnt.h @@ -143,8 +143,8 @@ struct rpc_clnt_connection { gf_boolean_t disconnected; struct saved_frames *saved_frames; int32_t frame_timeout; - struct timeval last_sent; - struct timeval last_received; + struct timespec last_sent; + struct timespec last_received; int32_t ping_started; char *name; int32_t ping_timeout; |