transport/rdma: Add attr_timeout, attr_retry_cnt, attr_rnr_retry

as configurable options

Change-Id: Ifc4710f149be35979746ddfbfb4181638601bc64
BUG: 766040
Signed-off-by: Harshavardhana <fharshav@redhat.com>
Reviewed-on: http://review.gluster.com/766
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

4 files changed, 74 insertions, 7 deletions

diff --git a/libglusterfs/src/dict.c b/libglusterfs/src/dict.c
index 40f575d52..2ad1ae1ab 100644
--- a/libglusterfs/src/dict.c
+++ b/libglusterfs/src/dict.c
@@ -22,6 +22,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <inttypes.h>
+#include <limits.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -1166,6 +1167,35 @@ data_to_uint16 (data_t *data)
         return strtol (str, NULL, 0);
 }
 
+uint8_t
+data_to_uint8 (data_t *data)
+{
+	uint32_t value = 0;
+
+        if (!data) {
+		gf_log_callingfn ("dict", GF_LOG_WARNING, "data is NULL");
+                return -1;
+	}
+
+        char *str = alloca (data->len + 1);
+        if (!str)
+                return -1;
+
+        memcpy (str, data->data, data->len);
+        str[data->len] = '\0';
+
+	errno = 0;
+	value = strtol (str, NULL, 0);
+
+	if ((UCHAR_MAX - value) < 0) {
+		errno = ERANGE;
+		gf_log_callingfn ("dict", GF_LOG_WARNING, "data conversion overflow detected (%s)", strerror(errno));
+		return -1;
+	}
+
+        return (uint8_t) value;
+}
+
 char *
 data_to_str (data_t *data)
 {
diff --git a/libglusterfs/src/dict.h b/libglusterfs/src/dict.h
index 121f84be5..c68c3af87 100644
--- a/libglusterfs/src/dict.h
+++ b/libglusterfs/src/dict.h
@@ -110,6 +110,7 @@ int8_t data_to_int8 (data_t *data);
 uint64_t data_to_uint64 (data_t *data);
 uint32_t data_to_uint32 (data_t *data);
 uint16_t data_to_uint16 (data_t *data);
+uint8_t data_to_uint8 (data_t *data);
 
 data_t *data_from_ptr (void *value);
 data_t *data_from_static_ptr (void *value);
diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c
index 43175af48..d3a8e9c8d 100644
--- a/rpc/rpc-transport/rdma/src/rdma.c
+++ b/rpc/rpc-transport/rdma/src/rdma.c
@@ -299,7 +299,6 @@ __gf_rdma_ioq_entry_free (gf_rdma_ioq_t *entry)
                 iobref_unref (entry->msg.request.rsp_iobref);
                 entry->msg.request.rsp_iobref = NULL;
         }
-        /* TODO: use mem-pool */
         mem_put (entry);
 }
 
@@ -1788,7 +1787,7 @@ gf_rdma_ioq_new (rpc_transport_t *this, rpc_transport_data_t *data)
         }
 
         priv = this->private;
-        /* TODO: use mem-pool */
+
         entry = mem_get (priv->device->ioq_pool);
         if (entry == NULL) {
                 goto out;
@@ -2426,11 +2425,10 @@ gf_rdma_connect_qp (rpc_transport_t *this)
                 return -1;
         }
 
-        /* TODO: make timeout and retry_cnt configurable from options */
         attr.qp_state       = IBV_QPS_RTS;
-        attr.timeout        = 14;
-        attr.retry_cnt      = 7;
-        attr.rnr_retry      = 7;
+        attr.timeout        = options->attr_timeout;
+        attr.retry_cnt      = options->attr_retry_cnt;
+        attr.rnr_retry      = options->attr_rnr_retry;
         attr.sq_psn         = priv->peer.local_psn;
         attr.max_rd_atomic  = 1;
         if (ibv_modify_qp (priv->peer.qp, &attr,
@@ -3615,6 +3613,9 @@ gf_rdma_options_init (rpc_transport_t *this)
         options->recv_size = GLUSTERFS_RDMA_INLINE_THRESHOLD;/*this->ctx->page_size * 4;  512 KB*/
         options->send_count = 4096;
         options->recv_count = 4096;
+	options->attr_timeout = GF_RDMA_TIMEOUT;
+	options->attr_retry_cnt = GF_RDMA_RETRY_CNT;
+	options->attr_rnr_retry = GF_RDMA_RNR_RETRY;
 
         temp = dict_get (this->options,
                          "transport.rdma.work-request-send-count");
@@ -3624,7 +3625,22 @@ gf_rdma_options_init (rpc_transport_t *this)
         temp = dict_get (this->options,
                          "transport.rdma.work-request-recv-count");
         if (temp)
-                options->recv_count = data_to_int32 (temp);
+		options->recv_count = data_to_int32 (temp);
+
+	temp = dict_get (this->options, "transport.rdma.attr-timeout");
+
+	if (temp)
+		options->attr_timeout = data_to_uint8 (temp);
+
+	temp = dict_get (this->options, "transport.rdma.attr-retry-cnt");
+
+	if (temp)
+		options->attr_retry_cnt = data_to_uint8 (temp);
+
+	temp = dict_get (this->options, "transport.rdma.attr-rnr-retry");
+
+	if (temp)
+		options->attr_rnr_retry = data_to_uint8 (temp);
 
         options->port = 1;
         temp = dict_get (this->options,
@@ -4905,6 +4921,18 @@ struct volume_options options[] = {
                     "transport.rdma.remote-port"},
           .type  = GF_OPTION_TYPE_INT
         },
+	{ .key   = {"transport.rdma.attr-timeout",
+		    "rdma-attr-timeout"},
+          .type  = GF_OPTION_TYPE_INT
+        },
+	{ .key   = {"transport.rdma.attr-retry-cnt",
+		    "rdma-attr-retry-cnt"},
+          .type  = GF_OPTION_TYPE_INT
+        },
+	{ .key   = {"transport.rdma.attr-rnr-retry",
+		    "rdma-attr-rnr-retry"},
+          .type  = GF_OPTION_TYPE_INT
+        },
         { .key   = {"transport.rdma.listen-port", "listen-port"},
           .type  = GF_OPTION_TYPE_INT
         },
diff --git a/rpc/rpc-transport/rdma/src/rdma.h b/rpc/rpc-transport/rdma/src/rdma.h
index 16149ef96..342938591 100644
--- a/rpc/rpc-transport/rdma/src/rdma.h
+++ b/rpc/rpc-transport/rdma/src/rdma.h
@@ -50,6 +50,11 @@
 #define GF_RDMA_VERSION                1
 #define GF_RDMA_POOL_SIZE              512
 
+/* Additional attributes */
+#define GF_RDMA_TIMEOUT                14
+#define GF_RDMA_RETRY_CNT              7
+#define GF_RDMA_RNR_RETRY              7
+
 typedef enum gf_rdma_errcode {
         ERR_VERS = 1,
         ERR_CHUNK = 2
@@ -148,6 +153,9 @@ struct __gf_rdma_options {
         int32_t  recv_count;
         uint64_t recv_size;
         uint64_t send_size;
+	uint8_t  attr_timeout;
+	uint8_t  attr_retry_cnt;
+	uint8_t  attr_rnr_retry;
 };
 typedef struct __gf_rdma_options gf_rdma_options_t;