rdma: pre-register iobuf_pool with rdma devices.

registring buffers with rdma device is a time consuming
operation. So performing registration in code path will
decrease the performance.
Using a pre registered memory will give a bettor performance,
ie, register iobuf_pool during rdma initialization. For
dynamically created arena, we can register with all the
device.

Change-Id: Ic79183e2efd014c43faf5911fdb6d5cfbcee64ca
BUG: 1187456
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
Reviewed-on: http://review.gluster.org/9506
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Tested-by: Raghavendra G <rgowdapp@redhat.com>

1 files changed, 11 insertions, 1 deletions

diff --git a/libglusterfs/src/iobuf.h b/libglusterfs/src/iobuf.h
index 4e07910d722..7e5cfe37a28 100644
--- a/libglusterfs/src/iobuf.h
+++ b/libglusterfs/src/iobuf.h
@@ -19,6 +19,8 @@
 
 #define GF_VARIABLE_IOBUF_COUNT 32
 
+#define GF_RDMA_DEVICE_COUNT 8
+
 /* Lets try to define the new anonymous mapping
  * flag, in case the system is still using the
  * now deprecated MAP_ANON flag.
@@ -81,6 +83,7 @@ struct iobuf_arena {
                 };
         };
 
+        struct list_head    all_list;
         size_t              page_size;  /* size of all iobufs in this arena */
         size_t              arena_size; /* this is equal to
                                            (iobuf_pool->arena_size / page_size)
@@ -110,6 +113,7 @@ struct iobuf_pool {
         size_t              default_page_size; /* default size of iobuf */
 
         int                 arena_cnt;
+        struct list_head    all_arenas;
         struct list_head    arenas[GF_VARIABLE_IOBUF_COUNT];
         /* array of arenas. Each element of the array is a list of arenas
            holding iobufs of particular page_size */
@@ -121,7 +125,13 @@ struct iobuf_pool {
         /* array of of arenas which can be purged */
 
         uint64_t            request_misses; /* mostly the requests for higher
-                                               value of iobufs */
+                                              value of iobufs */
+        int                 rdma_device_count;
+        struct list_head    *mr_list[GF_RDMA_DEVICE_COUNT];
+        void                *device[GF_RDMA_DEVICE_COUNT];
+        int (*rdma_registration)(void **, void*);
+        int (*rdma_deregistration)(struct list_head**, struct iobuf_arena *);
+
 };