rpc/rpc-transport: fix frame-loss during rdma-reads.v3.1.0qa31

- total no of rdma reads in progress was tracked using refcount of post
    (refcount = total rdma reads + 1). However doing rdma_post_unref _after_
    rdma_do_reads, can result in race condition between execution of
    rdma_post_unref and procedure handling rdma read completion. This makes
    it impossible to find out whether the current rdma read was the last
    rdma-read being done in procedure handling rdma read-completion.
    The fix to this should either do rdma_post_unref before posting an rdma
    read or use another variable to track the number of rdma-reads.
    This patch implements second method.

Signed-off-by: Raghavendra G <raghavendra@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>

BUG: 1700 (write fops are bailing out on a distribute setup)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1700

2 files changed, 18 insertions, 3 deletions

diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c
index 7790971f867..27a64d86ead 100644
--- a/rpc/rpc-transport/rdma/src/rdma.c
+++ b/rpc/rpc-transport/rdma/src/rdma.c
@@ -2921,6 +2921,15 @@ rdma_do_reads (rdma_peer_t *peer, rdma_post_t *post, rdma_read_chunk_t *readch)
                 size += readch[i].rc_target.rs_length;
         }
 
+        if (i == 0) {
+                gf_log (RDMA_LOG_NAME, GF_LOG_DEBUG,
+                        "message type specified as rdma-read but there are no "
+                        "rdma read-chunks present");
+                goto out;
+        }
+
+        post->ctx.rdma_reads = i;
+
         if (size > peer->trans->ctx->page_size) {
                 gf_log (RDMA_LOG_NAME, GF_LOG_ERROR,
                         "total size of rdma-read (%lu) is greater than "
@@ -3388,7 +3397,7 @@ void
 rdma_handle_successful_send_completion (rdma_peer_t *peer, struct ibv_wc *wc)
 {
         rdma_post_t            *post     = NULL;
-        int                     refcount = 0, ret = 0;
+        int                     reads    = 0, ret = 0;
 
         if (wc->opcode != IBV_WC_RDMA_READ) {
                 goto out;
@@ -3396,8 +3405,13 @@ rdma_handle_successful_send_completion (rdma_peer_t *peer, struct ibv_wc *wc)
 
         post = (rdma_post_t *)(long) wc->wr_id;
 
-        refcount = rdma_post_get_refcount (post);
-        if (refcount != 1) {
+        pthread_mutex_lock (&post->lock);
+        {
+                reads = --post->ctx.rdma_reads;
+        }
+        pthread_mutex_unlock (&post->lock);
+
+        if (reads != 0) {
                 /* if it is not the last rdma read, we've got nothing to do */
                 goto out;
         }
diff --git a/rpc/rpc-transport/rdma/src/rdma.h b/rpc/rpc-transport/rdma/src/rdma.h
index 51242d6b4ad..04386a98faf 100644
--- a/rpc/rpc-transport/rdma/src/rdma.h
+++ b/rpc/rpc-transport/rdma/src/rdma.h
@@ -257,6 +257,7 @@ struct __rdma_post_context {
         int                count;
         struct iobref     *iobref;
         char               is_request;
+        int                rdma_reads;
         rdma_reply_info_t *reply_info;
 };
 typedef struct __rdma_post_context rdma_post_context_t;