IOBUF: Bring in variable iobuf support

Rebasing Raghavendra's patch to master

Change-Id: Ie7a0c9a7b8e73cfe48c4573a5153460d5126208e
BUG: 2472
Reviewed-on: http://review.gluster.com/12
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Amar Tumballi <amar@gluster.com>
Reviewed-by: Anand Avati <avati@gluster.com>

10 files changed, 292 insertions, 123 deletions

diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 5f9f4a5c089..cb1e5a4453b 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -1783,6 +1783,60 @@ out:
 
         return flag;
 }
+/*
+ * rounds up nr to power of two. If nr is already a power of two, just returns
+ * nr
+ */
+
+inline int32_t
+gf_roundup_power_of_two (uint32_t nr)
+{
+        uint32_t result = 1;
+
+        if (nr < 0) {
+                gf_log ("common-utils", GF_LOG_WARNING,
+                        "negative number passed");
+                result = -1;
+                goto out;
+        }
+
+        while (result < nr) {
+                result *= 2;
+        }
+
+out:
+        return result;
+}
+
+/*
+ * rounds up nr to next power of two. If nr is already a power of two, next
+ * power of two is returned.
+ */
+
+/*
+ * rounds up nr to next power of two. If nr is already a power of two, next
+ * power of two is returned.
+ */
+
+inline int32_t
+gf_roundup_next_power_of_two (uint32_t nr)
+{
+        int32_t result = 1;
+
+        if (nr < 0) {
+                gf_log ("common-utils", GF_LOG_WARNING,
+                        "negative number passed");
+                result = -1;
+                goto out;
+        }
+
+        while (result <= nr) {
+                result *= 2;
+        }
+
+out:
+        return result;
+}
 
 int
 validate_brick_name (char *brick)
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index 9afa2805822..7b3cf4fde54 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -307,6 +307,19 @@ memdup (const void *ptr, size_t size)
 	return newptr;
 }
 
+/*
+ * rounds up nr to power of two. If nr is already a power of two, just returns
+ * nr
+ */
+
+int32_t gf_roundup_power_of_two (uint32_t nr);
+
+/*
+ * rounds up nr to next power of two. If nr is already a power of two, next
+ * power of two is returned.
+ */
+
+int32_t gf_roundup_next_power_of_two (uint32_t nr);
 
 char *gf_trim (char *string);
 int gf_strsplit (const char *str, const char *delim,
diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index bbb774be492..2a1dcb789a8 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -37,27 +37,6 @@ gf_fd_fdtable_expand (fdtable_t *fdtable, uint32_t nr);
 fd_t *
 _fd_ref (fd_t *fd);
 
-/*
-  Allocate in memory chunks of power of 2 starting from 1024B
-  Assumes fdtable->lock is held
-*/
-static inline int
-gf_roundup_power_of_two (uint32_t nr)
-{
-        uint32_t result = 1;
-
-        if (nr < 0) {
-                gf_log ("fd", GF_LOG_ERROR, "negative number passed");
-                return -1;
-        }
-
-        while (result <= nr)
-                result *= 2;
-
-        return result;
-}
-
-
 static int
 gf_fd_chain_fd_entries (fdentry_t *entries, uint32_t startidx,
                         uint32_t endcount)
@@ -96,7 +75,7 @@ gf_fd_fdtable_expand (fdtable_t *fdtable, uint32_t nr)
         }
 
         nr /= (1024 / sizeof (fdentry_t));
-        nr = gf_roundup_power_of_two (nr + 1);
+        nr = gf_roundup_next_power_of_two (nr + 1);
         nr *= (1024 / sizeof (fdentry_t));
 
         oldfds = fdtable->fdentries;
diff --git a/libglusterfs/src/iobuf.c b/libglusterfs/src/iobuf.c
index 8b6fddf5eb8..fc5630fcc71 100644
--- a/libglusterfs/src/iobuf.c
+++ b/libglusterfs/src/iobuf.c
@@ -39,8 +39,8 @@ __iobuf_arena_init_iobufs (struct iobuf_arena *iobuf_arena)
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf_arena, out);
 
-        arena_size = iobuf_arena->iobuf_pool->arena_size;
-        page_size  = iobuf_arena->iobuf_pool->page_size;
+        arena_size = iobuf_arena->arena_size;
+        page_size  = iobuf_arena->page_size;
         iobuf_cnt  = arena_size / page_size;
 
         iobuf_arena->iobufs = GF_CALLOC (sizeof (*iobuf), iobuf_cnt,
@@ -80,8 +80,8 @@ __iobuf_arena_destroy_iobufs (struct iobuf_arena *iobuf_arena)
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf_arena, out);
 
-        arena_size = iobuf_arena->iobuf_pool->arena_size;
-        page_size  = iobuf_arena->iobuf_pool->page_size;
+        arena_size = iobuf_arena->arena_size;
+        page_size  = iobuf_arena->page_size;
         iobuf_cnt  = arena_size / page_size;
 
         if (!iobuf_arena->iobufs) {
@@ -127,10 +127,10 @@ out:
 
 
 struct iobuf_arena *
-__iobuf_arena_alloc (struct iobuf_pool *iobuf_pool)
+__iobuf_arena_alloc (struct iobuf_pool *iobuf_pool, size_t page_size)
 {
         struct iobuf_arena *iobuf_arena = NULL;
-        size_t              arena_size = 0;
+        size_t              arena_size = 0, rounded_size = 0;
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf_pool, out);
 
@@ -145,6 +145,16 @@ __iobuf_arena_alloc (struct iobuf_pool *iobuf_pool)
         iobuf_arena->iobuf_pool = iobuf_pool;
 
         arena_size = iobuf_pool->arena_size;
+
+        rounded_size = gf_roundup_power_of_two (page_size);
+        iobuf_arena->page_size = rounded_size;
+
+        if ((arena_size % rounded_size) != 0) {
+                arena_size = (arena_size / rounded_size) * rounded_size;
+        }
+
+        iobuf_arena->arena_size = arena_size;
+
         iobuf_arena->mem_base = mmap (NULL, arena_size, PROT_READ|PROT_WRITE,
                                       MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
         if (iobuf_arena->mem_base == MAP_FAILED) {
@@ -171,50 +181,69 @@ out:
 
 
 struct iobuf_arena *
-__iobuf_arena_unprune (struct iobuf_pool *iobuf_pool)
+__iobuf_arena_unprune (struct iobuf_pool *iobuf_pool, size_t page_size)
 {
-        struct iobuf_arena *iobuf_arena = NULL;
-        struct iobuf_arena *tmp = NULL;
-
+        struct iobuf_arena *iobuf_arena  = NULL;
+        struct iobuf_arena *tmp          = NULL;
+        int                 index        = 0;
+ 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf_pool, out);
+ 
+        index = log_base2 (page_size);
+        if (index > GF_VARIABLE_IOBUF_COUNT) {
+                gf_log ("iobuf", GF_LOG_DEBUG, "no arena corresponding to "
+                        "page_size (%"GF_PRI_SIZET") is present. max supported "
+                        "size (%llu)", page_size,
+                        1LL << GF_VARIABLE_IOBUF_COUNT);
+                return NULL;
+        }
 
-        list_for_each_entry (tmp, &iobuf_pool->purge.list, list) {
+        list_for_each_entry (tmp, &iobuf_pool->purge[index], list) {
                 list_del_init (&tmp->list);
                 iobuf_arena = tmp;
                 break;
         }
-
 out:
         return iobuf_arena;
 }
 
 
 struct iobuf_arena *
-__iobuf_pool_add_arena (struct iobuf_pool *iobuf_pool)
+__iobuf_pool_add_arena (struct iobuf_pool *iobuf_pool, size_t page_size)
 {
-        struct iobuf_arena *iobuf_arena = NULL;
+        struct iobuf_arena *iobuf_arena  = NULL;
+        int                 index        = 0;
+        uint32_t            rounded_size = 0;
 
-        GF_VALIDATE_OR_GOTO ("iobuf", iobuf_pool, out);
+        rounded_size = gf_roundup_power_of_two (page_size);
+
+        index = log_base2 (rounded_size);
+        if (index > GF_VARIABLE_IOBUF_COUNT) {
+                gf_log ("iobuf", GF_LOG_DEBUG, "page_size %u of "
+                        "iobufs in arena being added is greater than max "
+                        "supported size (%llu)", rounded_size,
+                        1ULL << GF_VARIABLE_IOBUF_COUNT);
+                return NULL;
+        }
 
-        iobuf_arena = __iobuf_arena_unprune (iobuf_pool);
+        iobuf_arena = __iobuf_arena_unprune (iobuf_pool, rounded_size);
 
         if (!iobuf_arena)
-                iobuf_arena = __iobuf_arena_alloc (iobuf_pool);
+                iobuf_arena = __iobuf_arena_alloc (iobuf_pool, rounded_size);
 
         if (!iobuf_arena) {
                 gf_log (THIS->name, GF_LOG_WARNING, "arena not found");
                 return NULL;
         }
 
-        list_add_tail (&iobuf_arena->list, &iobuf_pool->arenas.list);
+        list_add_tail (&iobuf_arena->list, &iobuf_pool->arenas[index]);
 
-out:
         return iobuf_arena;
 }
 
 
 struct iobuf_arena *
-iobuf_pool_add_arena (struct iobuf_pool *iobuf_pool)
+iobuf_pool_add_arena (struct iobuf_pool *iobuf_pool, size_t page_size)
 {
         struct iobuf_arena *iobuf_arena = NULL;
 
@@ -222,7 +251,7 @@ iobuf_pool_add_arena (struct iobuf_pool *iobuf_pool)
 
         pthread_mutex_lock (&iobuf_pool->mutex);
         {
-                iobuf_arena = __iobuf_pool_add_arena (iobuf_pool);
+                iobuf_arena = __iobuf_pool_add_arena (iobuf_pool, page_size);
         }
         pthread_mutex_unlock (&iobuf_pool->mutex);
 
@@ -235,17 +264,19 @@ void
 iobuf_pool_destroy (struct iobuf_pool *iobuf_pool)
 {
         struct iobuf_arena *iobuf_arena = NULL;
-        struct iobuf_arena *tmp = NULL;
+        struct iobuf_arena *tmp         = NULL;
+        int                 i           = 0;
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf_pool, out);
 
-        list_for_each_entry_safe (iobuf_arena, tmp, &iobuf_pool->arenas.list,
-                                  list) {
-
-                list_del_init (&iobuf_arena->list);
-                iobuf_pool->arena_cnt--;
+        for (i = 0; i < GF_VARIABLE_IOBUF_COUNT; i++) {
+                list_for_each_entry_safe (iobuf_arena, tmp,
+                                          &iobuf_pool->arenas[i], list) {
+                        list_del_init (&iobuf_arena->list);
+                        iobuf_pool->arena_cnt--;
+                        __iobuf_arena_destroy (iobuf_arena);
+                }
 
-                __iobuf_arena_destroy (iobuf_arena);
         }
 
 out:
@@ -257,28 +288,34 @@ struct iobuf_pool *
 iobuf_pool_new (size_t arena_size, size_t page_size)
 {
         struct iobuf_pool  *iobuf_pool = NULL;
+        int                 i          = 0;
+        unsigned long long  max_size   = 0;
 
-        if (arena_size < page_size) {
-                gf_log (THIS->name, GF_LOG_WARNING,
+        max_size = ((1ULL << (GF_VARIABLE_IOBUF_COUNT)) - 1);
+        if ((arena_size < page_size) || (max_size < arena_size)) {
+                gf_log ("", GF_LOG_WARNING,
                         "arena size (%zu) is less than page size(%zu)",
                         arena_size, page_size);
-                return NULL;
+                goto out;
         }
 
         iobuf_pool = GF_CALLOC (sizeof (*iobuf_pool), 1,
                                 gf_common_mt_iobuf_pool);
         if (!iobuf_pool)
-                return NULL;
+                goto out;
 
         pthread_mutex_init (&iobuf_pool->mutex, NULL);
-        INIT_LIST_HEAD (&iobuf_pool->arenas.list);
-        INIT_LIST_HEAD (&iobuf_pool->filled.list);
-        INIT_LIST_HEAD (&iobuf_pool->purge.list);
+        for (i = 0; i < GF_VARIABLE_IOBUF_COUNT; i++) {
+                INIT_LIST_HEAD (&iobuf_pool->arenas[i]);
+                INIT_LIST_HEAD (&iobuf_pool->filled[i]);
+                INIT_LIST_HEAD (&iobuf_pool->purge[i]);
+        }
 
         iobuf_pool->arena_size = arena_size;
-        iobuf_pool->page_size  = page_size;
+        iobuf_pool->default_page_size  = page_size;
 
-        iobuf_pool_add_arena (iobuf_pool);
+        iobuf_pool_add_arena (iobuf_pool, page_size);
+out:
 
         return iobuf_pool;
 }
@@ -289,23 +326,25 @@ __iobuf_pool_prune (struct iobuf_pool *iobuf_pool)
 {
         struct iobuf_arena *iobuf_arena = NULL;
         struct iobuf_arena *tmp = NULL;
+        int                 i   = 0;
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf_pool, out);
 
-        if (list_empty (&iobuf_pool->arenas.list))
-                /* buffering - preserve this one arena (if at all)
-                   for __iobuf_arena_unprune */
-                return;
-
-        list_for_each_entry_safe (iobuf_arena, tmp, &iobuf_pool->purge.list,
-                                  list) {
-                if (iobuf_arena->active_cnt)
+        for (i = 0; i < GF_VARIABLE_IOBUF_COUNT; i++) {
+                if (list_empty (&iobuf_pool->arenas[i])) {
                         continue;
+                }
 
-                list_del_init (&iobuf_arena->list);
-                iobuf_pool->arena_cnt--;
+                list_for_each_entry_safe (iobuf_arena, tmp,
+                                          &iobuf_pool->purge[i], list) {
+                        if (iobuf_arena->active_cnt)
+                                continue;
 
-                __iobuf_arena_destroy (iobuf_arena);
+                        list_del_init (&iobuf_arena->list);
+                        iobuf_pool->arena_cnt--;
+
+                        __iobuf_arena_destroy (iobuf_arena);
+                }
         }
 
 out:
@@ -330,15 +369,27 @@ out:
 
 
 struct iobuf_arena *
-__iobuf_select_arena (struct iobuf_pool *iobuf_pool)
+__iobuf_select_arena (struct iobuf_pool *iobuf_pool, size_t page_size)
 {
-        struct iobuf_arena *iobuf_arena = NULL;
-        struct iobuf_arena *trav = NULL;
+        struct iobuf_arena *iobuf_arena  = NULL;
+        struct iobuf_arena *trav         = NULL;
+        size_t              rounded_size = 0;
+        int                 index        = 0;
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf_pool, out);
 
+        rounded_size = gf_roundup_power_of_two (page_size);
+
+        index = log_base2 (rounded_size);
+        if (index > GF_VARIABLE_IOBUF_COUNT) {
+                gf_log ("iobuf", GF_LOG_DEBUG, "size of iobuf requested (%"
+                        GF_PRI_SIZET") is greater than max supported size (%"
+                        "llu)", rounded_size, 1ULL << GF_VARIABLE_IOBUF_COUNT);
+                return NULL;
+        }
+
         /* look for unused iobuf from the head-most arena */
-        list_for_each_entry (trav, &iobuf_pool->arenas.list, list) {
+        list_for_each_entry (trav, &iobuf_pool->arenas[index], list) {
                 if (trav->passive_cnt) {
                         iobuf_arena = trav;
                         break;
@@ -347,7 +398,7 @@ __iobuf_select_arena (struct iobuf_pool *iobuf_pool)
 
         if (!iobuf_arena) {
                 /* all arenas were full */
-                iobuf_arena = __iobuf_pool_add_arena (iobuf_pool);
+                iobuf_arena = __iobuf_pool_add_arena (iobuf_pool, rounded_size);
         }
 
 out:
@@ -372,12 +423,12 @@ __iobuf_unref (struct iobuf *iobuf)
         return iobuf;
 }
 
-
 struct iobuf *
-__iobuf_get (struct iobuf_arena *iobuf_arena)
+__iobuf_get (struct iobuf_arena *iobuf_arena, size_t page_size)
 {
-        struct iobuf      *iobuf = NULL;
-        struct iobuf_pool *iobuf_pool = NULL;
+        struct iobuf      *iobuf        = NULL;
+        struct iobuf_pool *iobuf_pool   = NULL;
+        int                index        = 0;
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf_arena, out);
 
@@ -393,33 +444,67 @@ __iobuf_get (struct iobuf_arena *iobuf_arena)
         iobuf_arena->active_cnt++;
 
         if (iobuf_arena->passive_cnt == 0) {
+                index = log_base2 (page_size);
                 list_del (&iobuf_arena->list);
-                list_add (&iobuf_arena->list, &iobuf_pool->filled.list);
+                list_add (&iobuf_arena->list, &iobuf_pool->filled[index]);
         }
 
 out:
         return iobuf;
 }
 
+struct iobuf *
+iobuf_get2 (struct iobuf_pool *iobuf_pool, size_t page_size)
+{
+        struct iobuf       *iobuf        = NULL;
+        struct iobuf_arena *iobuf_arena  = NULL;
+        size_t              rounded_size = 0;
+
+        if (page_size == 0) {
+                page_size = iobuf_pool->default_page_size;
+        }
+
+        rounded_size = gf_roundup_power_of_two (page_size);
+
+        pthread_mutex_lock (&iobuf_pool->mutex);
+        {
+                /* most eligible arena for picking an iobuf */
+                iobuf_arena = __iobuf_select_arena (iobuf_pool, rounded_size);
+                if (!iobuf_arena)
+                        goto unlock;
+
+                iobuf = __iobuf_get (iobuf_arena, rounded_size);
+                if (!iobuf)
+                        goto unlock;
+
+                __iobuf_ref (iobuf);
+         }
+unlock:
+        pthread_mutex_unlock (&iobuf_pool->mutex);
+
+        return iobuf;
+}
 
 struct iobuf *
 iobuf_get (struct iobuf_pool *iobuf_pool)
 {
-        struct iobuf       *iobuf = NULL;
-        struct iobuf_arena *iobuf_arena = NULL;
+        struct iobuf       *iobuf        = NULL;
+        struct iobuf_arena *iobuf_arena  = NULL;
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf_pool, out);
 
         pthread_mutex_lock (&iobuf_pool->mutex);
         {
                 /* most eligible arena for picking an iobuf */
-                iobuf_arena = __iobuf_select_arena (iobuf_pool);
+                iobuf_arena = __iobuf_select_arena (iobuf_pool,
+                                                    iobuf_pool->default_page_size);
                 if (!iobuf_arena) {
                         gf_log (THIS->name, GF_LOG_WARNING, "arena not found");
                         goto unlock;
                 }
 
-                iobuf = __iobuf_get (iobuf_arena);
+                iobuf = __iobuf_get (iobuf_arena,
+                                     iobuf_pool->default_page_size);
                 if (!iobuf) {
                         gf_log (THIS->name, GF_LOG_WARNING, "iobuf not found");
                         goto unlock;
@@ -434,20 +519,30 @@ out:
         return iobuf;
 }
 
-
 void
 __iobuf_put (struct iobuf *iobuf, struct iobuf_arena *iobuf_arena)
 {
         struct iobuf_pool *iobuf_pool = NULL;
+        int                index      = 0;
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf_arena, out);
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf, out);
 
         iobuf_pool = iobuf_arena->iobuf_pool;
 
+        index = log_base2 (iobuf_arena->page_size);
+        if (index > GF_VARIABLE_IOBUF_COUNT) {
+                gf_log ("iobuf", GF_LOG_DEBUG, "size of iobuf being returned to"
+                        " pool(%"GF_PRI_SIZET") is greater than max supported "
+                        "size(%llu) arena = %p",
+                        iobuf_arena->page_size, 1ULL << GF_VARIABLE_IOBUF_COUNT,
+                        iobuf_arena);
+                return;
+        }
+
         if (iobuf_arena->passive_cnt == 0) {
                 list_del (&iobuf_arena->list);
-                list_add_tail (&iobuf_arena->list, &iobuf_pool->arenas.list);
+                list_add_tail (&iobuf_arena->list, &iobuf_pool->arenas[index]);
         }
 
         list_del_init (&iobuf->list);
@@ -458,7 +553,7 @@ __iobuf_put (struct iobuf *iobuf, struct iobuf_arena *iobuf_arena)
 
         if (iobuf_arena->active_cnt == 0) {
                 list_del (&iobuf_arena->list);
-                list_add_tail (&iobuf_arena->list, &iobuf_pool->purge.list);
+                list_add_tail (&iobuf_arena->list, &iobuf_pool->purge[index]);
         }
 out:
         return;
@@ -703,7 +798,7 @@ iobuf_size (struct iobuf *iobuf)
                 goto out;
         }
 
-        size = iobuf->iobuf_arena->iobuf_pool->page_size;
+        size = iobuf->iobuf_arena->page_size;
 out:
         return size;
 }
@@ -789,6 +884,7 @@ iobuf_stats_dump (struct iobuf_pool *iobuf_pool)
         char               msg[1024];
         struct iobuf_arena *trav = NULL;
         int                i = 1;
+        int                j = 0;
         int                ret = -1;
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf_pool, out);
@@ -802,19 +898,22 @@ iobuf_stats_dump (struct iobuf_pool *iobuf_pool)
         }
         gf_proc_dump_add_section("iobuf.global");
         gf_proc_dump_write("iobuf.global.iobuf_pool","%p", iobuf_pool);
-        gf_proc_dump_write("iobuf.global.iobuf_pool.page_size", "%d",
-                           iobuf_pool->page_size);
+        gf_proc_dump_write("iobuf.global.iobuf_pool.default_page_size", "%d",
+                                                iobuf_pool->default_page_size);
         gf_proc_dump_write("iobuf.global.iobuf_pool.arena_size", "%d",
                            iobuf_pool->arena_size);
         gf_proc_dump_write("iobuf.global.iobuf_pool.arena_cnt", "%d",
                            iobuf_pool->arena_cnt);
 
-        list_for_each_entry (trav, &iobuf_pool->arenas.list, list) {
-                snprintf(msg, sizeof(msg), "iobuf.global.iobuf_pool.arena.%d",
-                         i);
-                gf_proc_dump_add_section(msg);
-                iobuf_arena_info_dump(trav,msg);
-                i++;
+        for (j = 0; j < GF_VARIABLE_IOBUF_COUNT; j++) {
+                list_for_each_entry (trav, &iobuf_pool->arenas[j], list) {
+                        snprintf(msg, sizeof(msg),
+                                 "iobuf.global.iobuf_pool.arena.%d", i);
+                        gf_proc_dump_add_section(msg);
+                        iobuf_arena_info_dump(trav,msg);
+                        i++;
+                }
+
         }
 
         pthread_mutex_unlock(&iobuf_pool->mutex);
diff --git a/libglusterfs/src/iobuf.h b/libglusterfs/src/iobuf.h
index fc8ff623ad4..6a5b460dcea 100644
--- a/libglusterfs/src/iobuf.h
+++ b/libglusterfs/src/iobuf.h
@@ -26,6 +26,8 @@
 #include <sys/mman.h>
 #include <sys/uio.h>
 
+#define GF_VARIABLE_IOBUF_COUNT 32
+
 /* Lets try to define the new anonymous mapping
  * flag, in case the system is still using the
  * now deprecated MAP_ANON flag.
@@ -75,6 +77,12 @@ struct iobuf_arena {
                         struct iobuf_arena *prev;
                 };
         };
+
+        size_t              page_size;  /* size of all iobufs in this arena */
+        size_t              arena_size; /* this is equal to
+                                           (iobuf_pool->arena_size / page_size)
+                                           * page_size */
+
         struct iobuf_pool  *iobuf_pool;
 
         void               *mem_base;
@@ -91,17 +99,29 @@ struct iobuf_arena {
 
 struct iobuf_pool {
         pthread_mutex_t     mutex;
-        size_t              page_size;  /* size of all iobufs in this pool */
-        size_t              arena_size; /* size of memory region in arena */
+        size_t              arena_size; /* size of memory region in
+                                           arena */
+        size_t              default_page_size; /* default size of iobuf */
 
         int                 arena_cnt;
-        struct iobuf_arena  arenas;     /* head node arena
-                                           (unused by itself) */
-        struct iobuf_arena  filled;     /* arenas without  free iobufs */
-        struct iobuf_arena  purge;      /* arenas which can be purged */
-};
-
+        struct list_head    arenas[GF_VARIABLE_IOBUF_COUNT];
+        /* array of arenas. Each element of
+           the array is a list of arenas
+           holding iobufs of particular
+           page_size
+        */
+        struct list_head    filled[GF_VARIABLE_IOBUF_COUNT];
+        /*
+          array of arenas without free iobufs
+        */
+
+        struct list_head    purge[GF_VARIABLE_IOBUF_COUNT];
+        /*
+          array of of arenas which can be
+          purged
+        */
 
+};
 
 
 struct iobuf_pool *iobuf_pool_new (size_t arena_size, size_t page_size);
@@ -113,8 +133,8 @@ void iobuf_pool_destroy (struct iobuf_pool *iobuf_pool);
 void iobuf_to_iovec(struct iobuf *iob, struct iovec *iov);
 
 #define iobuf_ptr(iob) ((iob)->ptr)
-#define iobpool_pagesize(iobpool) ((iobpool)->page_size)
-#define iobuf_pagesize(iob) (iobpool_pagesize((iob)->iobuf_arena->iobuf_pool))
+#define iobpool_default_pagesize(iobpool) ((iobpool)->default_page_size)
+#define iobuf_pagesize(iob) (iob->iobuf_arena->page_size)
 
 
 struct iobref {
@@ -134,4 +154,6 @@ size_t iobuf_size (struct iobuf *iobuf);
 size_t iobref_size (struct iobref *iobref);
 void   iobuf_stats_dump (struct iobuf_pool *iobuf_pool);
 
+struct iobuf *
+iobuf_get2 (struct iobuf_pool *iobuf_pool, size_t page_size);
 #endif /* !_IOBUF_H_ */
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 09e676179d3..ae59d074f49 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -1173,7 +1173,7 @@ rpc_clnt_record_build_record (struct rpc_clnt *clnt, int prognum, int progver,
                 goto out;
         }
 
-        pagesize = ((struct iobuf_pool *)clnt->ctx->iobuf_pool)->page_size;
+        pagesize = iobuf_pagesize (request_iob);
 
         record = iobuf_ptr (request_iob);  /* Now we have it. */
 
diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index 10198b0f35e..875f6fae45c 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -775,7 +775,7 @@ rpcsvc_callback_build_record (rpcsvc_t *rpc, int prognum, int progver,
                 goto out;
         }
 
-        pagesize = ((struct iobuf_pool *)rpc->ctx->iobuf_pool)->page_size;
+        pagesize = iobuf_pagesize (request_iob);
 
         record = iobuf_ptr (request_iob);  /* Now we have it. */
 
@@ -946,7 +946,7 @@ rpcsvc_record_build_record (rpcsvc_request_t *req, size_t payload,
 
         svc = req->svc;
         replyiob = iobuf_get (svc->ctx->iobuf_pool);
-        pagesize = iobpool_pagesize ((struct iobuf_pool *)svc->ctx->iobuf_pool);
+        pagesize = iobuf_pagesize (replyiob);
         if (!replyiob) {
                 goto err_exit;
         }
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index 9fc491bb7f1..3b6a2ce2a7c 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -3108,7 +3108,7 @@ fuse_thread_proc (void *data)
 
         iov_in[0].iov_len = sizeof (*finh) + sizeof (struct fuse_write_in);
         iov_in[1].iov_len = ((struct iobuf_pool *)this->ctx->iobuf_pool)
-                              ->page_size;
+                              ->default_page_size;
         priv->msg0_len_p = &iov_in[0].iov_len;
 
         for (;;) {
diff --git a/xlators/performance/quick-read/src/quick-read.c b/xlators/performance/quick-read/src/quick-read.c
index bd9b4741638..7e859501768 100644
--- a/xlators/performance/quick-read/src/quick-read.c
+++ b/xlators/performance/quick-read/src/quick-read.c
@@ -1183,8 +1183,8 @@ qr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                                         }
 
                                         count = (op_ret
-                                                 / iobuf_pool->page_size);
-                                        if ((op_ret % iobuf_pool->page_size)
+                                                 / iobuf_pool->default_page_size);
+                                        if ((op_ret % iobuf_pool->default_page_size)
                                             != 0) {
                                                 count++;
                                         }
@@ -1223,16 +1223,18 @@ qr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
                                                 start = offset
                                                         +
-                                                        (iobuf_pool->page_size
+                                                        (iobuf_pool->default_page_size
                                                          * i);
 
                                                 if (start > end) {
                                                         len = 0;
                                                 } else {
-                                                        len = (iobuf_pool->page_size
+                                                        len =
+                                                        (iobuf_pool->default_page_size
                                                                > (end - start))
                                                                 ? (end - start)
-                                                                : iobuf_pool->page_size;
+                                                                :
+                                                                iobuf_pool->default_page_size;
 
                                                         memcpy (iobuf->ptr,
                                                                 content->data
diff --git a/xlators/protocol/client/src/client3_1-fops.c b/xlators/protocol/client/src/client3_1-fops.c
index 4790c790b5f..61dd3c1fbfd 100644
--- a/xlators/protocol/client/src/client3_1-fops.c
+++ b/xlators/protocol/client/src/client3_1-fops.c
@@ -2560,7 +2560,7 @@ client3_1_lookup (call_frame_t *frame, xlator_t *this,
                         rsphdr = &vector[0];
                         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
                         rsphdr->iov_len
-                                = rsp_iobuf->iobuf_arena->iobuf_pool->page_size;
+                                = iobuf_pagesize (rsp_iobuf);
                         count = 1;
                         rsp_iobuf = NULL;
                         local->iobref = rsp_iobref;
@@ -3521,7 +3521,7 @@ client3_1_readv (call_frame_t *frame, xlator_t *this,
         iobref_add (rsp_iobref, rsp_iobuf);
         iobuf_unref (rsp_iobuf);
         rsp_vec.iov_base = iobuf_ptr (rsp_iobuf);
-        rsp_vec.iov_len = rsp_iobuf->iobuf_arena->iobuf_pool->page_size;
+        rsp_vec.iov_len = iobuf_pagesize (rsp_iobuf);
 
         rsp_iobuf = NULL;
 
@@ -4189,7 +4189,7 @@ client3_1_fgetxattr (call_frame_t *frame, xlator_t *this,
         iobuf_unref (rsp_iobuf);
         rsphdr = &vector[0];
         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-        rsphdr->iov_len = rsp_iobuf->iobuf_arena->iobuf_pool->page_size;
+        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);;
         count = 1;
         rsp_iobuf = NULL;
         local->iobref = rsp_iobref;
@@ -4294,7 +4294,7 @@ client3_1_getxattr (call_frame_t *frame, xlator_t *this,
         iobuf_unref (rsp_iobuf);
         rsphdr = &vector[0];
         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-        rsphdr->iov_len = rsp_iobuf->iobuf_arena->iobuf_pool->page_size;
+        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
         count = 1;
         rsp_iobuf = NULL;
         local->iobref = rsp_iobref;
@@ -4416,7 +4416,7 @@ client3_1_xattrop (call_frame_t *frame, xlator_t *this,
         iobuf_unref (rsp_iobuf);
         rsphdr = &vector[0];
         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-        rsphdr->iov_len = rsp_iobuf->iobuf_arena->iobuf_pool->page_size;
+        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
         count = 1;
         rsp_iobuf = NULL;
         local->iobref = rsp_iobref;
@@ -4557,7 +4557,7 @@ client3_1_fxattrop (call_frame_t *frame, xlator_t *this,
         iobuf_unref (rsp_iobuf);
         rsphdr = &vector[0];
         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-        rsphdr->iov_len = rsp_iobuf->iobuf_arena->iobuf_pool->page_size;
+        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
         count = 1;
         rsp_iobuf = NULL;
         local->iobref = rsp_iobref;
@@ -5173,7 +5173,7 @@ client3_1_readdir (call_frame_t *frame, xlator_t *this,
                 rsphdr = &vector[0];
                 rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
                 rsphdr->iov_len
-                        = rsp_iobuf->iobuf_arena->iobuf_pool->page_size;
+                        = iobuf_pagesize (rsp_iobuf);
                 count = 1;
                 rsp_iobuf = NULL;
                 local->iobref = rsp_iobref;
@@ -5291,7 +5291,7 @@ client3_1_readdirp (call_frame_t *frame, xlator_t *this,
                 rsphdr = &vector[0];
                 rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
                 rsphdr->iov_len
-                        = rsp_iobuf->iobuf_arena->iobuf_pool->page_size;
+                        = iobuf_pagesize (rsp_iobuf);
                 count = 1;
                 rsp_iobuf = NULL;
                 local->iobref = rsp_iobref;