diff options
| author | Raghavendra G <raghavendra@gluster.com> | 2010-09-24 07:51:53 +0000 |
|---|---|---|
| committer | Vijay Bellur <vijay@dev.gluster.com> | 2010-09-24 07:10:27 -0700 |
| commit | 6e952607f17dd5ea6e3748f2bbc26234f4b1e553 (patch) | |
| tree | b31bbfc39505c238058be5df1cb7baad7b826493 /rpc | |
| parent | f2c5894f254d9b3d17b058adce7c7b4152da2f5e (diff) | |
rpc/rpc-transport: fix frame-loss during rdma-reads.v3.1.0qa31
- total no of rdma reads in progress was tracked using refcount of post
(refcount = total rdma reads + 1). However doing rdma_post_unref _after_
rdma_do_reads, can result in race condition between execution of
rdma_post_unref and procedure handling rdma read completion. This makes
it impossible to find out whether the current rdma read was the last
rdma-read being done in procedure handling rdma read-completion.
The fix to this should either do rdma_post_unref before posting an rdma
read or use another variable to track the number of rdma-reads.
This patch implements second method.
Signed-off-by: Raghavendra G <raghavendra@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>
BUG: 1700 (write fops are bailing out on a distribute setup)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1700
Diffstat (limited to 'rpc')
| -rw-r--r-- | rpc/rpc-transport/rdma/src/rdma.c | 20 | ||||
| -rw-r--r-- | rpc/rpc-transport/rdma/src/rdma.h | 1 |
2 files changed, 18 insertions, 3 deletions
diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c index 7790971f867..27a64d86ead 100644 --- a/rpc/rpc-transport/rdma/src/rdma.c +++ b/rpc/rpc-transport/rdma/src/rdma.c @@ -2921,6 +2921,15 @@ rdma_do_reads (rdma_peer_t *peer, rdma_post_t *post, rdma_read_chunk_t *readch) size += readch[i].rc_target.rs_length; } + if (i == 0) { + gf_log (RDMA_LOG_NAME, GF_LOG_DEBUG, + "message type specified as rdma-read but there are no " + "rdma read-chunks present"); + goto out; + } + + post->ctx.rdma_reads = i; + if (size > peer->trans->ctx->page_size) { gf_log (RDMA_LOG_NAME, GF_LOG_ERROR, "total size of rdma-read (%lu) is greater than " @@ -3388,7 +3397,7 @@ void rdma_handle_successful_send_completion (rdma_peer_t *peer, struct ibv_wc *wc) { rdma_post_t *post = NULL; - int refcount = 0, ret = 0; + int reads = 0, ret = 0; if (wc->opcode != IBV_WC_RDMA_READ) { goto out; @@ -3396,8 +3405,13 @@ rdma_handle_successful_send_completion (rdma_peer_t *peer, struct ibv_wc *wc) post = (rdma_post_t *)(long) wc->wr_id; - refcount = rdma_post_get_refcount (post); - if (refcount != 1) { + pthread_mutex_lock (&post->lock); + { + reads = --post->ctx.rdma_reads; + } + pthread_mutex_unlock (&post->lock); + + if (reads != 0) { /* if it is not the last rdma read, we've got nothing to do */ goto out; } diff --git a/rpc/rpc-transport/rdma/src/rdma.h b/rpc/rpc-transport/rdma/src/rdma.h index 51242d6b4ad..04386a98faf 100644 --- a/rpc/rpc-transport/rdma/src/rdma.h +++ b/rpc/rpc-transport/rdma/src/rdma.h @@ -257,6 +257,7 @@ struct __rdma_post_context { int count; struct iobref *iobref; char is_request; + int rdma_reads; rdma_reply_info_t *reply_info; }; typedef struct __rdma_post_context rdma_post_context_t; |