rpc-transport/rdma: Fix 2KB as threshold size for msgs that can be transferred inline.

- Any fop/mop that can result a reply whose size is greater than this threshold
    allocates and submits a buffer along with the request to receive reply.

Signed-off-by: Raghavendra G <raghavendra@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 513 (Introduce 0 copy rdma)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=513

2 files changed, 23 insertions, 13 deletions

diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c
index 4301f4119a0..b2258fbb99d 100644
--- a/rpc/rpc-transport/rdma/src/rdma.c
+++ b/rpc/rpc-transport/rdma/src/rdma.c
@@ -957,7 +957,7 @@ __rdma_ioq_churn_request (rdma_peer_t *peer, rdma_ioq_t *entry,
 
         send_size = iov_length (entry->rpchdr, entry->rpchdr_count)
                 + iov_length (entry->proghdr, entry->proghdr_count)
-                + RDMA_MAX_HEADER_SIZE;
+                + GLUSTERFS_RDMA_MAX_HEADER_SIZE;
 
         if (entry->prog_payload_count != 0) {
                 prog_payload_length
@@ -965,9 +965,10 @@ __rdma_ioq_churn_request (rdma_peer_t *peer, rdma_ioq_t *entry,
                                       entry->prog_payload_count);
         }
 
-        if (send_size > RDMA_INLINE_THRESHOLD) {
+        if (send_size > GLUSTERFS_RDMA_INLINE_THRESHOLD) {
                 rtype = rdma_areadch;
-        } else if ((send_size + prog_payload_length) < RDMA_INLINE_THRESHOLD) {
+        } else if ((send_size + prog_payload_length)
+                   < GLUSTERFS_RDMA_INLINE_THRESHOLD) {
                 rtype = rdma_noch;
         } else if (entry->prog_payload_count != 0) {
                 rtype = rdma_readch;
@@ -1151,7 +1152,7 @@ __rdma_send_reply_inline (rdma_peer_t *peer, rdma_ioq_t *entry,
                                            * reply
                                            */
 
-        if (send_size > RDMA_INLINE_THRESHOLD) {
+        if (send_size > GLUSTERFS_RDMA_INLINE_THRESHOLD) {
                 ret = __rdma_send_error (peer, entry, post, reply_info,
                                          ERR_CHUNK);
                 goto out;
@@ -1492,14 +1493,15 @@ __rdma_send_reply_type_msg (rdma_peer_t *peer, rdma_ioq_t *entry,
 
         send_size = iov_length (entry->rpchdr, entry->rpchdr_count)
                 + iov_length (entry->proghdr, entry->proghdr_count)
-                + RDMA_MAX_HEADER_SIZE;
+                + GLUSTERFS_RDMA_MAX_HEADER_SIZE;
 
-        if (send_size > RDMA_INLINE_THRESHOLD) {
+        if (send_size > GLUSTERFS_RDMA_INLINE_THRESHOLD) {
                 gf_log (RDMA_LOG_NAME, GF_LOG_DEBUG,
                         "client has provided only write chunks, but the "
                         "combined size of rpc and program header (%d) is "
                         "exceeding the size of msg that can be sent using "
-                        "RDMA send (%d)", send_size, RDMA_INLINE_THRESHOLD);
+                        "RDMA send (%d)", send_size,
+                        GLUSTERFS_RDMA_INLINE_THRESHOLD);
 
                 ret = __rdma_send_error (peer, entry, post, reply_info,
                                          ERR_CHUNK);
@@ -3550,8 +3552,8 @@ rdma_options_init (rpc_transport_t *this)
 
         /* TODO: validate arguments from options below */
 
-        options->send_size = this->ctx->page_size * 4; /* 512 KB */
-        options->recv_size = this->ctx->page_size * 4; /* 512 KB */
+        options->send_size = GLUSTERFS_RDMA_INLINE_THRESHOLD;/*this->ctx->page_size * 4;  512 KB*/
+        options->recv_size = GLUSTERFS_RDMA_INLINE_THRESHOLD;/*this->ctx->page_size * 4;  512 KB*/
         options->send_count = 32;
         options->recv_count = 32;
 
diff --git a/rpc/rpc-transport/rdma/src/rdma.h b/rpc/rpc-transport/rdma/src/rdma.h
index 4bb5a3759e1..e24ce76ccf1 100644
--- a/rpc/rpc-transport/rdma/src/rdma.h
+++ b/rpc/rpc-transport/rdma/src/rdma.h
@@ -41,11 +41,12 @@
 
 /* FIXME: give appropriate values to these macros */
 #define GF_DEFAULT_RDMA_LISTEN_PORT (GF_DEFAULT_BASE_PORT + 1)
+
+/* If you are changing RDMA_MAX_SEGMENTS, please make sure to update
+ * GLUSTERFS_RDMA_MAX_HEADER_SIZE defined in glusterfs.h .
+ */
 #define RDMA_MAX_SEGMENTS           8
-#define RDMA_MAX_HEADER_SIZE        (sizeof (rdma_header_t)                 \
-                                     + RDMA_MAX_SEGMENTS                    \
-                                     * sizeof (rdma_read_chunk_t))
-#define RDMA_INLINE_THRESHOLD       (1024 * 128)
+
 #define RDMA_VERSION                1
 #define RDMA_POOL_SIZE              512
 
@@ -76,6 +77,9 @@ typedef enum rdma_chunktype {
         rdma_replych           /* entire reply through rdma write */
 }rdma_chunktype_t;
 
+/* If you are modifying __rdma_header, please make sure to change
+ * GLUSTERFS_RDMA_MAX_HEADER_SIZE defined in glusterfs.h to reflect your changes
+ */
 struct __rdma_header {
         uint32_t rm_xid;    /* Mirrors the RPC header xid */
         uint32_t rm_vers;   /* Version of this protocol */
@@ -102,6 +106,10 @@ struct __rdma_header {
 } __attribute__((packed));
 typedef struct __rdma_header rdma_header_t;
 
+/* If you are modifying __rdma_segment or __rdma_read_chunk, please make sure
+ * to change GLUSTERFS_RDMA_MAX_HEADER_SIZE defined in glusterfs.h to reflect
+ * your changes.
+ */
 struct __rdma_segment {
         uint32_t rs_handle;       /* Registered memory handle */
         uint32_t rs_length;       /* Length of the chunk in bytes */