summaryrefslogtreecommitdiffstats
path: root/libglusterfs/src
diff options
context:
space:
mode:
Diffstat (limited to 'libglusterfs/src')
-rw-r--r--libglusterfs/src/glusterfs/iobuf.h111
-rw-r--r--libglusterfs/src/iobuf.c836
-rw-r--r--libglusterfs/src/mem-pool.c2
3 files changed, 72 insertions, 877 deletions
diff --git a/libglusterfs/src/glusterfs/iobuf.h b/libglusterfs/src/glusterfs/iobuf.h
index 6de0f13ae36..7516bc8034b 100644
--- a/libglusterfs/src/glusterfs/iobuf.h
+++ b/libglusterfs/src/glusterfs/iobuf.h
@@ -17,118 +17,38 @@
#include <sys/mman.h>
#include <sys/uio.h>
-#define GF_VARIABLE_IOBUF_COUNT 32
-
-#define GF_RDMA_DEVICE_COUNT 8
-
-/* Lets try to define the new anonymous mapping
- * flag, in case the system is still using the
- * now deprecated MAP_ANON flag.
- *
- * Also, this should ideally be in a centralized/common
- * header which can be used by other source files also.
- */
-#ifndef MAP_ANONYMOUS
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-
#define GF_ALIGN_BUF(ptr, bound) \
((void *)((unsigned long)(ptr + bound - 1) & (unsigned long)(~(bound - 1))))
#define GF_IOBUF_ALIGN_SIZE 512
+#define GF_IOBUF_DEFAULT_PAGE_SIZE (128 * GF_UNIT_KB)
+
/* one allocatable unit for the consumers of the IOBUF API */
/* each unit hosts @page_size bytes of memory */
struct iobuf;
-/* one region of memory mapped from the operating system */
-/* each region MMAPs @arena_size bytes of memory */
-/* each arena hosts @arena_size / @page_size IOBUFs */
-struct iobuf_arena;
-
/* expandable and contractable pool of memory, internally broken into arenas */
struct iobuf_pool;
-struct iobuf_init_config {
- size_t pagesize;
- int32_t num_pages;
-};
-
struct iobuf {
- union {
- struct list_head list;
- struct {
- struct iobuf *next;
- struct iobuf *prev;
- };
- };
- struct iobuf_arena *iobuf_arena;
-
- gf_lock_t lock; /* for ->ptr and ->ref */
- gf_atomic_t ref; /* 0 == passive, >0 == active */
+ gf_boolean_t stdalloc; /* indicates whether iobuf is allocated from
+ mem pool or standard alloc*/
+ gf_lock_t lock; /* for ->ptr and ->ref */
+ gf_atomic_t ref; /* 0 == passive, >0 == active */
void *ptr; /* usable memory region by the consumer */
- void *free_ptr; /* in case of stdalloc, this is the
- one to be freed */
-};
-
-struct iobuf_arena {
- union {
- struct list_head list;
- struct {
- struct iobuf_arena *next;
- struct iobuf_arena *prev;
- };
- };
-
- struct list_head all_list;
- size_t page_size; /* size of all iobufs in this arena */
- size_t arena_size;
- /* this is equal to rounded_size * num_iobufs.
- (rounded_size comes with gf_iobuf_get_pagesize().) */
- size_t page_count;
-
- struct iobuf_pool *iobuf_pool;
-
- void *mem_base;
- struct iobuf *iobufs; /* allocated iobufs list */
-
- int active_cnt;
- struct iobuf active; /* head node iobuf
- (unused by itself) */
- int passive_cnt;
- struct iobuf passive; /* head node iobuf
- (unused by itself) */
- uint64_t alloc_cnt; /* total allocs in this pool */
- int max_active; /* max active buffers at a given time */
+ void *free_ptr; /* in case of stdalloc, this is the
+ one to be freed */
+ size_t page_size; /* iobuf's page size */
+ struct iobuf_pool *iobuf_pool; /* iobuf_pool iobuf is associated with */
};
struct iobuf_pool {
- pthread_mutex_t mutex;
- size_t arena_size; /* size of memory region in
- arena */
- size_t default_page_size; /* default size of iobuf */
-
- int arena_cnt;
- struct list_head all_arenas;
- struct list_head arenas[GF_VARIABLE_IOBUF_COUNT];
- /* array of arenas. Each element of the array is a list of arenas
- holding iobufs of particular page_size */
-
- struct list_head filled[GF_VARIABLE_IOBUF_COUNT];
- /* array of arenas without free iobufs */
-
- struct list_head purge[GF_VARIABLE_IOBUF_COUNT];
- /* array of of arenas which can be purged */
-
- uint64_t request_misses; /* mostly the requests for higher
- value of iobufs */
- int rdma_device_count;
- struct list_head *mr_list[GF_RDMA_DEVICE_COUNT];
- void *device[GF_RDMA_DEVICE_COUNT];
- int (*rdma_registration)(void **, void *);
- int (*rdma_deregistration)(struct list_head **, struct iobuf_arena *);
+ gf_atomic_t mem_pool_hit;
+ gf_atomic_t mem_pool_miss;
+ gf_atomic_t active_cnt;
};
struct iobuf_pool *
@@ -142,13 +62,10 @@ iobuf_unref(struct iobuf *iobuf);
struct iobuf *
iobuf_ref(struct iobuf *iobuf);
void
-iobuf_pool_destroy(struct iobuf_pool *iobuf_pool);
-void
iobuf_to_iovec(struct iobuf *iob, struct iovec *iov);
#define iobuf_ptr(iob) ((iob)->ptr)
-#define iobpool_default_pagesize(iobpool) ((iobpool)->default_page_size)
-#define iobuf_pagesize(iob) (iob->iobuf_arena->page_size)
+#define iobuf_pagesize(iob) (iob->page_size)
struct iobref {
gf_lock_t lock;
diff --git a/libglusterfs/src/iobuf.c b/libglusterfs/src/iobuf.c
index 0e37c4fc6e2..aff0a3121ff 100644
--- a/libglusterfs/src/iobuf.c
+++ b/libglusterfs/src/iobuf.c
@@ -12,588 +12,80 @@
#include "glusterfs/statedump.h"
#include <stdio.h>
#include "glusterfs/libglusterfs-messages.h"
-
-/*
- TODO: implement destroy margins and prefetching of arenas
-*/
-
-#define IOBUF_ARENA_MAX_INDEX \
- (sizeof(gf_iobuf_init_config) / (sizeof(struct iobuf_init_config)))
-
-/* Make sure this array is sorted based on pagesize */
-struct iobuf_init_config gf_iobuf_init_config[] = {
- /* { pagesize, num_pages }, */
- {128, 1024}, {512, 512}, {2 * 1024, 512}, {8 * 1024, 128},
- {32 * 1024, 64}, {128 * 1024, 32}, {256 * 1024, 8}, {1 * 1024 * 1024, 2},
-};
-
-static int
-gf_iobuf_get_arena_index(const size_t page_size)
-{
- int i;
-
- for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
- if (page_size <= gf_iobuf_init_config[i].pagesize)
- return i;
- }
-
- return -1;
-}
-
-static size_t
-gf_iobuf_get_pagesize(const size_t page_size)
-{
- int i;
- size_t size = 0;
-
- for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
- size = gf_iobuf_init_config[i].pagesize;
- if (page_size <= size)
- return size;
- }
-
- return -1;
-}
-
-void
-__iobuf_arena_init_iobufs(struct iobuf_arena *iobuf_arena)
-{
- int iobuf_cnt = 0;
- struct iobuf *iobuf = NULL;
- int offset = 0;
- int i = 0;
-
- GF_VALIDATE_OR_GOTO("iobuf", iobuf_arena, out);
-
- iobuf_cnt = iobuf_arena->page_count;
-
- iobuf_arena->iobufs = GF_CALLOC(sizeof(*iobuf), iobuf_cnt,
- gf_common_mt_iobuf);
- if (!iobuf_arena->iobufs)
- return;
-
- iobuf = iobuf_arena->iobufs;
- for (i = 0; i < iobuf_cnt; i++) {
- INIT_LIST_HEAD(&iobuf->list);
- LOCK_INIT(&iobuf->lock);
-
- iobuf->iobuf_arena = iobuf_arena;
-
- iobuf->ptr = iobuf_arena->mem_base + offset;
-
- list_add(&iobuf->list, &iobuf_arena->passive.list);
- iobuf_arena->passive_cnt++;
-
- offset += iobuf_arena->page_size;
- iobuf++;
- }
-
-out:
- return;
-}
-
-void
-__iobuf_arena_destroy_iobufs(struct iobuf_arena *iobuf_arena)
-{
- int iobuf_cnt = 0;
- struct iobuf *iobuf = NULL;
- int i = 0;
-
- GF_VALIDATE_OR_GOTO("iobuf", iobuf_arena, out);
-
- iobuf_cnt = iobuf_arena->page_count;
-
- if (!iobuf_arena->iobufs) {
- gf_msg_callingfn(THIS->name, GF_LOG_ERROR, 0, LG_MSG_IOBUFS_NOT_FOUND,
- "iobufs not found");
- return;
- }
-
- iobuf = iobuf_arena->iobufs;
- for (i = 0; i < iobuf_cnt; i++) {
- GF_ASSERT(GF_ATOMIC_GET(iobuf->ref) == 0);
-
- LOCK_DESTROY(&iobuf->lock);
- list_del_init(&iobuf->list);
- iobuf++;
- }
-
- GF_FREE(iobuf_arena->iobufs);
-
-out:
- return;
-}
-
-void
-__iobuf_arena_destroy(struct iobuf_pool *iobuf_pool,
- struct iobuf_arena *iobuf_arena)
-{
- GF_VALIDATE_OR_GOTO("iobuf", iobuf_arena, out);
-
- if (iobuf_pool->rdma_deregistration)
- iobuf_pool->rdma_deregistration(iobuf_pool->mr_list, iobuf_arena);
-
- __iobuf_arena_destroy_iobufs(iobuf_arena);
-
- if (iobuf_arena->mem_base && iobuf_arena->mem_base != MAP_FAILED)
- munmap(iobuf_arena->mem_base, iobuf_arena->arena_size);
-
- GF_FREE(iobuf_arena);
-out:
- return;
-}
-
-struct iobuf_arena *
-__iobuf_arena_alloc(struct iobuf_pool *iobuf_pool, size_t page_size,
- int32_t num_iobufs)
-{
- struct iobuf_arena *iobuf_arena = NULL;
- size_t rounded_size = 0;
-
- GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
-
- iobuf_arena = GF_CALLOC(sizeof(*iobuf_arena), 1, gf_common_mt_iobuf_arena);
- if (!iobuf_arena)
- goto err;
-
- INIT_LIST_HEAD(&iobuf_arena->list);
- INIT_LIST_HEAD(&iobuf_arena->all_list);
- INIT_LIST_HEAD(&iobuf_arena->active.list);
- INIT_LIST_HEAD(&iobuf_arena->passive.list);
- iobuf_arena->iobuf_pool = iobuf_pool;
-
- rounded_size = gf_iobuf_get_pagesize(page_size);
-
- iobuf_arena->page_size = rounded_size;
- iobuf_arena->page_count = num_iobufs;
-
- iobuf_arena->arena_size = rounded_size * num_iobufs;
-
- iobuf_arena->mem_base = mmap(NULL, iobuf_arena->arena_size,
- PROT_READ | PROT_WRITE,
- MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
- if (iobuf_arena->mem_base == MAP_FAILED) {
- gf_msg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_MAPPING_FAILED,
- "mapping failed");
- goto err;
- }
-
- if (iobuf_pool->rdma_registration) {
- iobuf_pool->rdma_registration(iobuf_pool->device, iobuf_arena);
- }
-
- list_add_tail(&iobuf_arena->all_list, &iobuf_pool->all_arenas);
-
- __iobuf_arena_init_iobufs(iobuf_arena);
- if (!iobuf_arena->iobufs) {
- gf_msg(THIS->name, GF_LOG_ERROR, 0, LG_MSG_INIT_IOBUF_FAILED,
- "init failed");
- goto err;
- }
-
- iobuf_pool->arena_cnt++;
-
- return iobuf_arena;
-
-err:
- __iobuf_arena_destroy(iobuf_pool, iobuf_arena);
-
-out:
- return NULL;
-}
-
-static struct iobuf_arena *
-__iobuf_arena_unprune(struct iobuf_pool *iobuf_pool, const size_t page_size,
- const int index)
-{
- struct iobuf_arena *iobuf_arena = NULL;
- struct iobuf_arena *tmp = NULL;
-
- GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
-
- list_for_each_entry(tmp, &iobuf_pool->purge[index], list)
- {
- list_del_init(&tmp->list);
- iobuf_arena = tmp;
- break;
- }
-out:
- return iobuf_arena;
-}
-
-static struct iobuf_arena *
-__iobuf_pool_add_arena(struct iobuf_pool *iobuf_pool, const size_t page_size,
- const int32_t num_pages, const int index)
-{
- struct iobuf_arena *iobuf_arena = NULL;
-
- iobuf_arena = __iobuf_arena_unprune(iobuf_pool, page_size, index);
-
- if (!iobuf_arena) {
- iobuf_arena = __iobuf_arena_alloc(iobuf_pool, page_size, num_pages);
- if (!iobuf_arena) {
- gf_msg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_ARENA_NOT_FOUND,
- "arena not found");
- return NULL;
- }
- }
- list_add(&iobuf_arena->list, &iobuf_pool->arenas[index]);
-
- return iobuf_arena;
-}
-
-/* This function destroys all the iobufs and the iobuf_pool */
-void
-iobuf_pool_destroy(struct iobuf_pool *iobuf_pool)
-{
- struct iobuf_arena *iobuf_arena = NULL;
- struct iobuf_arena *tmp = NULL;
- int i = 0;
-
- GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
-
- pthread_mutex_lock(&iobuf_pool->mutex);
- {
- for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
- list_for_each_entry_safe(iobuf_arena, tmp, &iobuf_pool->arenas[i],
- list)
- {
- list_del_init(&iobuf_arena->list);
- iobuf_pool->arena_cnt--;
-
- __iobuf_arena_destroy(iobuf_pool, iobuf_arena);
- }
- list_for_each_entry_safe(iobuf_arena, tmp, &iobuf_pool->purge[i],
- list)
- {
- list_del_init(&iobuf_arena->list);
- iobuf_pool->arena_cnt--;
- __iobuf_arena_destroy(iobuf_pool, iobuf_arena);
- }
- /* If there are no iobuf leaks, there should be no
- * arenas in the filled list. If at all there are any
- * arenas in the filled list, the below function will
- * assert.
- */
- list_for_each_entry_safe(iobuf_arena, tmp, &iobuf_pool->filled[i],
- list)
- {
- list_del_init(&iobuf_arena->list);
- iobuf_pool->arena_cnt--;
- __iobuf_arena_destroy(iobuf_pool, iobuf_arena);
- }
- /* If there are no iobuf leaks, there shoould be
- * no standard allocated arenas, iobuf_put will free
- * such arenas.
- * TODO: Free the stdalloc arenas forcefully if present?
- */
- }
- }
- pthread_mutex_unlock(&iobuf_pool->mutex);
-
- pthread_mutex_destroy(&iobuf_pool->mutex);
-
- GF_FREE(iobuf_pool);
-
-out:
- return;
-}
-
-static void
-iobuf_create_stdalloc_arena(struct iobuf_pool *iobuf_pool)
-{
- struct iobuf_arena *iobuf_arena = NULL;
-
- /* No locking required here as its called only once during init */
- iobuf_arena = GF_CALLOC(sizeof(*iobuf_arena), 1, gf_common_mt_iobuf_arena);
- if (!iobuf_arena)
- goto err;
-
- INIT_LIST_HEAD(&iobuf_arena->list);
- INIT_LIST_HEAD(&iobuf_arena->active.list);
- INIT_LIST_HEAD(&iobuf_arena->passive.list);
-
- iobuf_arena->iobuf_pool = iobuf_pool;
-
- iobuf_arena->page_size = 0x7fffffff;
-
- list_add_tail(&iobuf_arena->list,
- &iobuf_pool->arenas[IOBUF_ARENA_MAX_INDEX]);
-
-err:
- return;
-}
+#include "glusterfs/atomic.h"
struct iobuf_pool *
iobuf_pool_new(void)
{
struct iobuf_pool *iobuf_pool = NULL;
- int i = 0;
- size_t page_size = 0;
- size_t arena_size = 0;
- int32_t num_pages = 0;
- int index;
iobuf_pool = GF_CALLOC(sizeof(*iobuf_pool), 1, gf_common_mt_iobuf_pool);
if (!iobuf_pool)
goto out;
- INIT_LIST_HEAD(&iobuf_pool->all_arenas);
- pthread_mutex_init(&iobuf_pool->mutex, NULL);
- for (i = 0; i <= IOBUF_ARENA_MAX_INDEX; i++) {
- INIT_LIST_HEAD(&iobuf_pool->arenas[i]);
- INIT_LIST_HEAD(&iobuf_pool->filled[i]);
- INIT_LIST_HEAD(&iobuf_pool->purge[i]);
- }
-
- iobuf_pool->default_page_size = 128 * GF_UNIT_KB;
-
- iobuf_pool->rdma_registration = NULL;
- iobuf_pool->rdma_deregistration = NULL;
-
- for (i = 0; i < GF_RDMA_DEVICE_COUNT; i++) {
- iobuf_pool->device[i] = NULL;
- iobuf_pool->mr_list[i] = NULL;
- }
-
- pthread_mutex_lock(&iobuf_pool->mutex);
- {
- for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
- page_size = gf_iobuf_init_config[i].pagesize;
- num_pages = gf_iobuf_init_config[i].num_pages;
-
- index = gf_iobuf_get_arena_index(page_size);
- if (index == -1) {
- pthread_mutex_unlock(&iobuf_pool->mutex);
- gf_msg("iobuf", GF_LOG_ERROR, 0, LG_MSG_PAGE_SIZE_EXCEEDED,
- "page_size (%zu) of iobufs in arena being added is "
- "greater than max available",
- page_size);
- return NULL;
- }
-
- __iobuf_pool_add_arena(iobuf_pool, page_size, num_pages, index);
-
- arena_size += page_size * num_pages;
- }
- }
- pthread_mutex_unlock(&iobuf_pool->mutex);
- /* Need an arena to handle all the bigger iobuf requests */
- iobuf_create_stdalloc_arena(iobuf_pool);
+ GF_ATOMIC_INIT(iobuf_pool->mem_pool_hit, 0);
+ GF_ATOMIC_INIT(iobuf_pool->mem_pool_miss, 0);
+ GF_ATOMIC_INIT(iobuf_pool->active_cnt, 0);
- iobuf_pool->arena_size = arena_size;
out:
-
return iobuf_pool;
}
-static void
-__iobuf_arena_prune(struct iobuf_pool *iobuf_pool,
- struct iobuf_arena *iobuf_arena, const int index)
-{
- /* code flow comes here only if the arena is in purge list and we can
- * free the arena only if we have at least one arena in 'arenas' list
- * (ie, at least few iobufs free in arena), that way, there won't
- * be spurious mmap/unmap of buffers
- */
- if (list_empty(&iobuf_pool->arenas[index]))
- goto out;
-
- /* All cases matched, destroy */
- list_del_init(&iobuf_arena->list);
- list_del_init(&iobuf_arena->all_list);
- iobuf_pool->arena_cnt--;
-
- __iobuf_arena_destroy(iobuf_pool, iobuf_arena);
-
-out:
- return;
-}
-
void
-iobuf_pool_prune(struct iobuf_pool *iobuf_pool)
+iobuf_pool_destroy(struct iobuf_pool *iobuf_pool)
{
- struct iobuf_arena *iobuf_arena = NULL;
- struct iobuf_arena *tmp = NULL;
- int i = 0;
-
- GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
+ if (!iobuf_pool)
+ return;
- pthread_mutex_lock(&iobuf_pool->mutex);
- {
- for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
- if (list_empty(&iobuf_pool->arenas[i])) {
- continue;
- }
-
- list_for_each_entry_safe(iobuf_arena, tmp, &iobuf_pool->purge[i],
- list)
- {
- __iobuf_arena_prune(iobuf_pool, iobuf_arena, i);
- }
- }
- }
- pthread_mutex_unlock(&iobuf_pool->mutex);
+ if (GF_ATOMIC_GET(iobuf_pool->active_cnt) != 0)
+ gf_msg_callingfn(THIS->name, GF_LOG_ERROR, 0, LG_MSG_IOBUFS_NOT_FOUND,
+ "iobuf_pool_destroy called, but there"
+ " are unfreed active iobufs:%" PRId64,
+ GF_ATOMIC_GET(iobuf_pool->active_cnt));
-out:
+ GF_FREE(iobuf_pool);
return;
}
-/* Always called under the iobuf_pool mutex lock */
-static struct iobuf_arena *
-__iobuf_select_arena(struct iobuf_pool *iobuf_pool, const size_t page_size,
- const int index)
-{
- struct iobuf_arena *iobuf_arena = NULL;
- struct iobuf_arena *trav = NULL;
-
- /* look for unused iobuf from the head-most arena */
- list_for_each_entry(trav, &iobuf_pool->arenas[index], list)
- {
- if (trav->passive_cnt) {
- iobuf_arena = trav;
- break;
- }
- }
-
- if (!iobuf_arena) {
- /* all arenas were full, find the right count to add */
- iobuf_arena = __iobuf_pool_add_arena(
- iobuf_pool, page_size, gf_iobuf_init_config[index].num_pages,
- index);
- }
-
- return iobuf_arena;
-}
-
-/* Always called under the iobuf_pool mutex lock */
-static struct iobuf *
-__iobuf_get(struct iobuf_pool *iobuf_pool, const size_t page_size,
- const int index)
-{
- struct iobuf *iobuf = NULL;
- struct iobuf_arena *iobuf_arena = NULL;
-
- /* most eligible arena for picking an iobuf */
- iobuf_arena = __iobuf_select_arena(iobuf_pool, page_size, index);
- if (!iobuf_arena)
- return NULL;
-
- list_for_each_entry(iobuf, &iobuf_arena->passive.list, list) break;
-
- list_del(&iobuf->list);
- iobuf_arena->passive_cnt--;
-
- list_add(&iobuf->list, &iobuf_arena->active.list);
- iobuf_arena->active_cnt++;
-
- /* no resetting requied for this element */
- iobuf_arena->alloc_cnt++;
-
- if (iobuf_arena->max_active < iobuf_arena->active_cnt)
- iobuf_arena->max_active = iobuf_arena->active_cnt;
-
- if (iobuf_arena->passive_cnt == 0) {
- list_del(&iobuf_arena->list);
- list_add(&iobuf_arena->list, &iobuf_pool->filled[index]);
- }
-
- return iobuf;
-}
-
struct iobuf *
-iobuf_get_from_stdalloc(struct iobuf_pool *iobuf_pool, size_t page_size)
+iobuf_get2(struct iobuf_pool *iobuf_pool, size_t page_size)
{
struct iobuf *iobuf = NULL;
- struct iobuf_arena *iobuf_arena = NULL;
- struct iobuf_arena *trav = NULL;
- int ret = -1;
+ gf_boolean_t hit = _gf_false;
- /* The first arena in the 'MAX-INDEX' will always be used for misc */
- list_for_each_entry(trav, &iobuf_pool->arenas[IOBUF_ARENA_MAX_INDEX], list)
- {
- iobuf_arena = trav;
- break;
+ if (page_size == 0) {
+ page_size = GF_IOBUF_DEFAULT_PAGE_SIZE;
}
- iobuf = GF_CALLOC(1, sizeof(*iobuf), gf_common_mt_iobuf);
+ iobuf = mem_pool_get0(sizeof(struct iobuf), &hit);
if (!iobuf)
goto out;
- /* 4096 is the alignment */
- iobuf->free_ptr = GF_CALLOC(1, ((page_size + GF_IOBUF_ALIGN_SIZE) - 1),
- gf_common_mt_char);
- if (!iobuf->free_ptr)
+ iobuf->free_ptr = mem_pool_get(page_size, &hit);
+ if (!iobuf->free_ptr) {
+ iobuf->free_ptr = GF_MALLOC(page_size, gf_common_mt_char);
+ iobuf->stdalloc = _gf_true;
+ }
+ if (!iobuf->free_ptr) {
+ mem_put(iobuf);
+ iobuf = NULL;
goto out;
+ }
+ if (hit == _gf_true)
+ GF_ATOMIC_INC(iobuf_pool->mem_pool_hit);
+ else
+ GF_ATOMIC_INC(iobuf_pool->mem_pool_miss);
- iobuf->ptr = GF_ALIGN_BUF(iobuf->free_ptr, GF_IOBUF_ALIGN_SIZE);
- iobuf->iobuf_arena = iobuf_arena;
+ iobuf->ptr = iobuf->free_ptr;
LOCK_INIT(&iobuf->lock);
- /* Hold a ref because you are allocating and using it */
- GF_ATOMIC_INIT(iobuf->ref, 1);
+ iobuf->page_size = page_size;
+ iobuf->iobuf_pool = iobuf_pool;
- ret = 0;
+ /* Hold a ref because you are allocating and using it */
+ iobuf_ref(iobuf);
+ GF_ATOMIC_INC(iobuf_pool->active_cnt);
out:
- if (ret && iobuf) {
- GF_FREE(iobuf->free_ptr);
- GF_FREE(iobuf);
- iobuf = NULL;
- }
-
- return iobuf;
-}
-
-struct iobuf *
-iobuf_get2(struct iobuf_pool *iobuf_pool, size_t page_size)
-{
- struct iobuf *iobuf = NULL;
- size_t rounded_size = 0;
- int index = 0;
-
- if (page_size == 0) {
- page_size = iobuf_pool->default_page_size;
- }
-
- rounded_size = gf_iobuf_get_pagesize(page_size);
- if (rounded_size == -1) {
- /* make sure to provide the requested buffer with standard
- memory allocations */
- iobuf = iobuf_get_from_stdalloc(iobuf_pool, page_size);
-
- gf_msg_debug("iobuf", 0,
- "request for iobuf of size %zu "
- "is serviced using standard calloc() (%p) as it "
- "exceeds the maximum available buffer size",
- page_size, iobuf);
-
- iobuf_pool->request_misses++;
- return iobuf;
- }
-
- index = gf_iobuf_get_arena_index(page_size);
- if (index == -1) {
- gf_msg("iobuf", GF_LOG_ERROR, 0, LG_MSG_PAGE_SIZE_EXCEEDED,
- "page_size (%zu) of iobufs in arena being added is "
- "greater than max available",
- page_size);
- return NULL;
- }
-
- pthread_mutex_lock(&iobuf_pool->mutex);
- {
- iobuf = __iobuf_get(iobuf_pool, rounded_size, index);
- if (!iobuf) {
- gf_msg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_IOBUF_NOT_FOUND,
- "iobuf not found");
- goto unlock;
- }
-
- iobuf_ref(iobuf);
- }
-unlock:
- pthread_mutex_unlock(&iobuf_pool->mutex);
-
return iobuf;
}
@@ -607,23 +99,13 @@ iobuf_get_page_aligned(struct iobuf_pool *iobuf_pool, size_t page_size,
req_size = page_size;
if (req_size == 0) {
- req_size = iobuf_pool->default_page_size;
+ req_size = GF_IOBUF_DEFAULT_PAGE_SIZE;
}
iobuf = iobuf_get2(iobuf_pool, req_size + align_size);
if (!iobuf)
return NULL;
- /* If std allocation was used, then free_ptr will be non-NULL. In this
- * case, we do not want to modify the original free_ptr.
- * On the other hand, if the buf was gotten through the available
- * arenas, then we use iobuf->free_ptr to store the original
- * pointer to the offset into the mmap'd block of memory and in turn
- * reuse iobuf->ptr to hold the page-aligned address. And finally, in
- * iobuf_put(), we copy iobuf->free_ptr into iobuf->ptr - back to where
- * it was originally when __iobuf_get() returned this iobuf.
- */
- if (!iobuf->free_ptr)
- iobuf->free_ptr = iobuf->ptr;
+
iobuf->ptr = GF_ALIGN_BUF(iobuf->ptr, align_size);
return iobuf;
@@ -632,118 +114,22 @@ iobuf_get_page_aligned(struct iobuf_pool *iobuf_pool, size_t page_size,
struct iobuf *
iobuf_get(struct iobuf_pool *iobuf_pool)
{
- struct iobuf *iobuf = NULL;
- int index = 0;
-
- GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
-
- index = gf_iobuf_get_arena_index(iobuf_pool->default_page_size);
- if (index == -1) {
- gf_msg("iobuf", GF_LOG_ERROR, 0, LG_MSG_PAGE_SIZE_EXCEEDED,
- "page_size (%zu) of iobufs in arena being added is "
- "greater than max available",
- iobuf_pool->default_page_size);
- return NULL;
- }
-
- pthread_mutex_lock(&iobuf_pool->mutex);
- {
- iobuf = __iobuf_get(iobuf_pool, iobuf_pool->default_page_size, index);
- if (!iobuf) {
- gf_msg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_IOBUF_NOT_FOUND,
- "iobuf not found");
- goto unlock;
- }
-
- iobuf_ref(iobuf);
- }
-unlock:
- pthread_mutex_unlock(&iobuf_pool->mutex);
-
-out:
- return iobuf;
-}
-
-void
-__iobuf_put(struct iobuf *iobuf, struct iobuf_arena *iobuf_arena)
-{
- struct iobuf_pool *iobuf_pool = NULL;
- int index = 0;
-
- GF_VALIDATE_OR_GOTO("iobuf", iobuf_arena, out);
- GF_VALIDATE_OR_GOTO("iobuf", iobuf, out);
-
- iobuf_pool = iobuf_arena->iobuf_pool;
-
- index = gf_iobuf_get_arena_index(iobuf_arena->page_size);
- if (index == -1) {
- gf_msg_debug("iobuf", 0,
- "freeing the iobuf (%p) "
- "allocated with standard calloc()",
- iobuf);
-
- /* free up properly without bothering about lists and all */
- LOCK_DESTROY(&iobuf->lock);
- GF_FREE(iobuf->free_ptr);
- GF_FREE(iobuf);
- return;
- }
-
- if (iobuf_arena->passive_cnt == 0) {
- list_del(&iobuf_arena->list);
- list_add_tail(&iobuf_arena->list, &iobuf_pool->arenas[index]);
- }
-
- list_del_init(&iobuf->list);
- iobuf_arena->active_cnt--;
-
- if (iobuf->free_ptr) {
- iobuf->ptr = iobuf->free_ptr;
- iobuf->free_ptr = NULL;
- }
-
- list_add(&iobuf->list, &iobuf_arena->passive.list);
- iobuf_arena->passive_cnt++;
-
- if (iobuf_arena->active_cnt == 0) {
- list_del(&iobuf_arena->list);
- list_add_tail(&iobuf_arena->list, &iobuf_pool->purge[index]);
- GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
- __iobuf_arena_prune(iobuf_pool, iobuf_arena, index);
- }
-out:
- return;
+ return iobuf_get2(iobuf_pool, GF_IOBUF_DEFAULT_PAGE_SIZE);
}
void
iobuf_put(struct iobuf *iobuf)
{
- struct iobuf_arena *iobuf_arena = NULL;
- struct iobuf_pool *iobuf_pool = NULL;
-
- GF_VALIDATE_OR_GOTO("iobuf", iobuf, out);
+ LOCK_DESTROY(&iobuf->lock);
- iobuf_arena = iobuf->iobuf_arena;
- if (!iobuf_arena) {
- gf_msg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_ARENA_NOT_FOUND,
- "arena not found");
- return;
- }
-
- iobuf_pool = iobuf_arena->iobuf_pool;
- if (!iobuf_pool) {
- gf_msg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_POOL_NOT_FOUND,
- "iobuf pool not found");
- return;
- }
+ if (iobuf->stdalloc)
+ GF_FREE(iobuf->free_ptr);
+ else
+ mem_put(iobuf->free_ptr);
- pthread_mutex_lock(&iobuf_pool->mutex);
- {
- __iobuf_put(iobuf, iobuf_arena);
- }
- pthread_mutex_unlock(&iobuf_pool->mutex);
+ GF_ATOMIC_DEC(iobuf->iobuf_pool->active_cnt);
+ mem_put(iobuf);
-out:
return;
}
@@ -967,25 +353,10 @@ out:
size_t
iobuf_size(struct iobuf *iobuf)
{
- size_t size = 0;
-
- GF_VALIDATE_OR_GOTO("iobuf", iobuf, out);
-
- if (!iobuf->iobuf_arena) {
- gf_msg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_ARENA_NOT_FOUND,
- "arena not found");
- goto out;
- }
-
- if (!iobuf->iobuf_arena->iobuf_pool) {
- gf_msg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_POOL_NOT_FOUND,
- "pool not found");
- goto out;
- }
+ if (!iobuf)
+ return 0;
- size = iobuf->iobuf_arena->page_size;
-out:
- return size;
+ return iobuf->page_size;
}
size_t
@@ -1010,113 +381,20 @@ out:
}
void
-iobuf_info_dump(struct iobuf *iobuf, const char *key_prefix)
-{
- char key[GF_DUMP_MAX_BUF_LEN];
- struct iobuf my_iobuf;
- int ret = 0;
-
- GF_VALIDATE_OR_GOTO("iobuf", iobuf, out);
-
- ret = TRY_LOCK(&iobuf->lock);
- if (ret) {
- return;
- }
- memcpy(&my_iobuf, iobuf, sizeof(my_iobuf));
- UNLOCK(&iobuf->lock);
-
- gf_proc_dump_build_key(key, key_prefix, "ref");
- gf_proc_dump_write(key, "%" GF_PRI_ATOMIC, GF_ATOMIC_GET(my_iobuf.ref));
- gf_proc_dump_build_key(key, key_prefix, "ptr");
- gf_proc_dump_write(key, "%p", my_iobuf.ptr);
-
-out:
- return;
-}
-
-void
-iobuf_arena_info_dump(struct iobuf_arena *iobuf_arena, const char *key_prefix)
-{
- char key[GF_DUMP_MAX_BUF_LEN];
- int i = 1;
- struct iobuf *trav;
-
- GF_VALIDATE_OR_GOTO("iobuf", iobuf_arena, out);
-
- gf_proc_dump_build_key(key, key_prefix, "mem_base");
- gf_proc_dump_write(key, "%p", iobuf_arena->mem_base);
- gf_proc_dump_build_key(key, key_prefix, "active_cnt");
- gf_proc_dump_write(key, "%d", iobuf_arena->active_cnt);
- gf_proc_dump_build_key(key, key_prefix, "passive_cnt");
- gf_proc_dump_write(key, "%d", iobuf_arena->passive_cnt);
- gf_proc_dump_build_key(key, key_prefix, "alloc_cnt");
- gf_proc_dump_write(key, "%" PRIu64, iobuf_arena->alloc_cnt);
- gf_proc_dump_build_key(key, key_prefix, "max_active");
- gf_proc_dump_write(key, "%d", iobuf_arena->max_active);
- gf_proc_dump_build_key(key, key_prefix, "page_size");
- gf_proc_dump_write(key, "%" GF_PRI_SIZET, iobuf_arena->page_size);
- list_for_each_entry(trav, &iobuf_arena->active.list, list)
- {
- gf_proc_dump_build_key(key, key_prefix, "active_iobuf.%d", i++);
- gf_proc_dump_add_section("%s", key);
- iobuf_info_dump(trav, key);
- }
-
-out:
- return;
-}
-
-void
iobuf_stats_dump(struct iobuf_pool *iobuf_pool)
{
- char msg[1024];
- struct iobuf_arena *trav = NULL;
- int i = 1;
- int j = 0;
- int ret = -1;
-
GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
- ret = pthread_mutex_trylock(&iobuf_pool->mutex);
-
- if (ret) {
- return;
- }
gf_proc_dump_add_section("iobuf.global");
gf_proc_dump_write("iobuf_pool", "%p", iobuf_pool);
- gf_proc_dump_write("iobuf_pool.default_page_size", "%" GF_PRI_SIZET,
- iobuf_pool->default_page_size);
- gf_proc_dump_write("iobuf_pool.arena_size", "%" GF_PRI_SIZET,
- iobuf_pool->arena_size);
- gf_proc_dump_write("iobuf_pool.arena_cnt", "%d", iobuf_pool->arena_cnt);
+ gf_proc_dump_write("iobuf_pool.default_page_size", "%llu",
+ GF_IOBUF_DEFAULT_PAGE_SIZE);
+ gf_proc_dump_write("iobuf_pool.request_hits", "%" PRId64,
+ GF_ATOMIC_GET(iobuf_pool->mem_pool_hit));
gf_proc_dump_write("iobuf_pool.request_misses", "%" PRId64,
- iobuf_pool->request_misses);
-
- for (j = 0; j < IOBUF_ARENA_MAX_INDEX; j++) {
- list_for_each_entry(trav, &iobuf_pool->arenas[j], list)
- {
- snprintf(msg, sizeof(msg), "arena.%d", i);
- gf_proc_dump_add_section("%s", msg);
- iobuf_arena_info_dump(trav, msg);
- i++;
- }
- list_for_each_entry(trav, &iobuf_pool->purge[j], list)
- {
- snprintf(msg, sizeof(msg), "purge.%d", i);
- gf_proc_dump_add_section("%s", msg);
- iobuf_arena_info_dump(trav, msg);
- i++;
- }
- list_for_each_entry(trav, &iobuf_pool->filled[j], list)
- {
- snprintf(msg, sizeof(msg), "filled.%d", i);
- gf_proc_dump_add_section("%s", msg);
- iobuf_arena_info_dump(trav, msg);
- i++;
- }
- }
-
- pthread_mutex_unlock(&iobuf_pool->mutex);
+ GF_ATOMIC_GET(iobuf_pool->mem_pool_miss));
+ gf_proc_dump_write("iobuf_pool.active_cnt", "%" PRId64,
+ GF_ATOMIC_GET(iobuf_pool->active_cnt));
out:
return;
diff --git a/libglusterfs/src/mem-pool.c b/libglusterfs/src/mem-pool.c
index 81badc0ba0b..4f46b1e3ca0 100644
--- a/libglusterfs/src/mem-pool.c
+++ b/libglusterfs/src/mem-pool.c
@@ -920,7 +920,7 @@ mem_pool_get(unsigned long sizeof_type, gf_boolean_t *hit)
sizeof_type |= (1 << POOL_SMALLEST) - 1;
power = sizeof(sizeof_type) * 8 - __builtin_clzl(sizeof_type - 1) + 1;
if (power > POOL_LARGEST) {
- gf_msg_callingfn("mem-pool", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+ gf_msg_callingfn("mem-pool", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG,
"invalid argument");
return NULL;
}
generated by cgit (git 2.34.1) at 2025-11-04 02:16:07 +0000