iobuf: Get rid of pre allocated iobuf_pool and use per thread mem pool

The current implementation of iobuf_pool has two problems:
- prealloc of 12.5MB memory, this limits the scale factor of the gluster
  processes due to RAM requirements
- lock contention, as the current implementation has one global
  iobuf_pool lock. Credits for debugging and addressing the same goes to
  Krutika Dhananjay <kdhananj@redhat.com>. Issue: #410

Hence changing the iobuf implementation to use per thread mem pool.
This may theoritically appear to cause perf dip as there is no preallocation.
But per thread mem pool will not have significant perf impact as the last
allocated memory is kept alive for subsequent allocs, for some time.
The worst case would be if iobufs requested are of random sizes each time.
The best case is, if we get iobuf request of the same size. From the perf
tests, this patch did not seem to cause any perf decrease.

Note that, with this patch, the rdma performance is going to degrade
drastically. In one of the previous patchsets we had fixes to not
degrade rdma perf, but rdma is not supported and also not tested [1].
Hence the decision was to not have code in rdma that is not tested
and not supported.

[1] https://lists.gluster.org/pipermail/gluster-users.old/2018-July/034400.html

Updates: #325
Change-Id: Ic2ef3bd498f9250dea25f25ba0c01fde19584b27
Signed-off-by: Poornima G <pgurusid@redhat.com>

1 files changed, 14 insertions, 97 deletions

diff --git a/libglusterfs/src/glusterfs/iobuf.h b/libglusterfs/src/glusterfs/iobuf.h
index 6de0f13ae36..7516bc8034b 100644
--- a/libglusterfs/src/glusterfs/iobuf.h
+++ b/libglusterfs/src/glusterfs/iobuf.h
@@ -17,118 +17,38 @@
 #include <sys/mman.h>
 #include <sys/uio.h>
 
-#define GF_VARIABLE_IOBUF_COUNT 32
-
-#define GF_RDMA_DEVICE_COUNT 8
-
-/* Lets try to define the new anonymous mapping
- * flag, in case the system is still using the
- * now deprecated MAP_ANON flag.
- *
- * Also, this should ideally be in a centralized/common
- * header which can be used by other source files also.
- */
-#ifndef MAP_ANONYMOUS
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-
 #define GF_ALIGN_BUF(ptr, bound)                                               \
     ((void *)((unsigned long)(ptr + bound - 1) & (unsigned long)(~(bound - 1))))
 
 #define GF_IOBUF_ALIGN_SIZE 512
 
+#define GF_IOBUF_DEFAULT_PAGE_SIZE (128 * GF_UNIT_KB)
+
 /* one allocatable unit for the consumers of the IOBUF API */
 /* each unit hosts @page_size bytes of memory */
 struct iobuf;
 
-/* one region of memory mapped from the operating system */
-/* each region MMAPs @arena_size bytes of memory */
-/* each arena hosts @arena_size / @page_size IOBUFs */
-struct iobuf_arena;
-
 /* expandable and contractable pool of memory, internally broken into arenas */
 struct iobuf_pool;
 
-struct iobuf_init_config {
-    size_t pagesize;
-    int32_t num_pages;
-};
-
 struct iobuf {
-    union {
-        struct list_head list;
-        struct {
-            struct iobuf *next;
-            struct iobuf *prev;
-        };
-    };
-    struct iobuf_arena *iobuf_arena;
-
-    gf_lock_t lock;  /* for ->ptr and ->ref */
-    gf_atomic_t ref; /* 0 == passive, >0 == active */
+    gf_boolean_t stdalloc; /* indicates whether iobuf is allocated from
+                              mem pool or standard alloc*/
+    gf_lock_t lock;        /* for ->ptr and ->ref */
+    gf_atomic_t ref;       /* 0 == passive, >0 == active */
 
     void *ptr; /* usable memory region by the consumer */
 
-    void *free_ptr; /* in case of stdalloc, this is the
-                       one to be freed */
-};
-
-struct iobuf_arena {
-    union {
-        struct list_head list;
-        struct {
-            struct iobuf_arena *next;
-            struct iobuf_arena *prev;
-        };
-    };
-
-    struct list_head all_list;
-    size_t page_size; /* size of all iobufs in this arena */
-    size_t arena_size;
-    /* this is equal to rounded_size * num_iobufs.
-       (rounded_size comes with gf_iobuf_get_pagesize().) */
-    size_t page_count;
-
-    struct iobuf_pool *iobuf_pool;
-
-    void *mem_base;
-    struct iobuf *iobufs; /* allocated iobufs list */
-
-    int active_cnt;
-    struct iobuf active; /* head node iobuf
-                            (unused by itself) */
-    int passive_cnt;
-    struct iobuf passive; /* head node iobuf
-                             (unused by itself) */
-    uint64_t alloc_cnt;   /* total allocs in this pool */
-    int max_active;       /* max active buffers at a given time */
+    void *free_ptr;                /* in case of stdalloc, this is the
+                                      one to be freed */
+    size_t page_size;              /* iobuf's page size */
+    struct iobuf_pool *iobuf_pool; /* iobuf_pool iobuf is associated with */
 };
 
 struct iobuf_pool {
-    pthread_mutex_t mutex;
-    size_t arena_size;        /* size of memory region in
-                                 arena */
-    size_t default_page_size; /* default size of iobuf */
-
-    int arena_cnt;
-    struct list_head all_arenas;
-    struct list_head arenas[GF_VARIABLE_IOBUF_COUNT];
-    /* array of arenas. Each element of the array is a list of arenas
-       holding iobufs of particular page_size */
-
-    struct list_head filled[GF_VARIABLE_IOBUF_COUNT];
-    /* array of arenas without free iobufs */
-
-    struct list_head purge[GF_VARIABLE_IOBUF_COUNT];
-    /* array of of arenas which can be purged */
-
-    uint64_t request_misses; /* mostly the requests for higher
-                               value of iobufs */
-    int rdma_device_count;
-    struct list_head *mr_list[GF_RDMA_DEVICE_COUNT];
-    void *device[GF_RDMA_DEVICE_COUNT];
-    int (*rdma_registration)(void **, void *);
-    int (*rdma_deregistration)(struct list_head **, struct iobuf_arena *);
+    gf_atomic_t mem_pool_hit;
+    gf_atomic_t mem_pool_miss;
+    gf_atomic_t active_cnt;
 };
 
 struct iobuf_pool *
@@ -142,13 +62,10 @@ iobuf_unref(struct iobuf *iobuf);
 struct iobuf *
 iobuf_ref(struct iobuf *iobuf);
 void
-iobuf_pool_destroy(struct iobuf_pool *iobuf_pool);
-void
 iobuf_to_iovec(struct iobuf *iob, struct iovec *iov);
 
 #define iobuf_ptr(iob) ((iob)->ptr)
-#define iobpool_default_pagesize(iobpool) ((iobpool)->default_page_size)
-#define iobuf_pagesize(iob) (iob->iobuf_arena->page_size)
+#define iobuf_pagesize(iob) (iob->page_size)
 
 struct iobref {
     gf_lock_t lock;