libglusterfs: Implement API that provides page-aligned iobufs

One of the consumers of a page aligned buffer would be posix's
readv fop on O_DIRECT fds. Today the way it works is by getting
a page-aligned buffer through calloc, pread()ing into this buffer
and then copying its contents into a newly created iobuf's ptr.
This results in an extra memcpy() which can be avoided if we could
implement an api that would return an iobuf whose ptr is
page-aligned. That way the iobuf->ptr can be directly passed to
sys_pread() as a parameter by posix translator.

Change-Id: I385139bc6ee309fc501034b3af8f7814fab8cd65
BUG: 1343838
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-on: http://review.gluster.org/14672
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Zhou Zhengping <johnzzpcrystal@gmail.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>

2 files changed, 39 insertions, 0 deletions

diff --git a/libglusterfs/src/iobuf.c b/libglusterfs/src/iobuf.c
index d1eb0acaf5e..17cd68fc206 100644
--- a/libglusterfs/src/iobuf.c
+++ b/libglusterfs/src/iobuf.c
@@ -679,6 +679,36 @@ unlock:
 }
 
 struct iobuf *
+iobuf_get_page_aligned (struct iobuf_pool *iobuf_pool, size_t page_size,
+                        size_t align_size)
+{
+        size_t              req_size     = 0;
+        struct iobuf       *iobuf        = NULL;
+
+        req_size = page_size;
+
+        if (req_size == 0) {
+                req_size = iobuf_pool->default_page_size;
+        }
+
+        iobuf = iobuf_get2 (iobuf_pool, req_size + align_size);
+        /* If std allocation was used, then free_ptr will be non-NULL. In this
+         * case, we do not want to modify the original free_ptr.
+         * On the other hand, if the buf was gotten through the available
+         * arenas, then we use iobuf->free_ptr to store the original
+         * pointer to the offset into the mmap'd block of memory and in turn
+         * reuse iobuf->ptr to hold the page-aligned address. And finally, in
+         * iobuf_put(), we copy iobuf->free_ptr into iobuf->ptr - back to where
+         * it was originally when __iobuf_get() returned this iobuf.
+         */
+        if (!iobuf->free_ptr)
+                iobuf->free_ptr = iobuf->ptr;
+        iobuf->ptr = GF_ALIGN_BUF (iobuf->ptr, align_size);
+
+        return iobuf;
+}
+
+struct iobuf *
 iobuf_get (struct iobuf_pool *iobuf_pool)
 {
         struct iobuf       *iobuf        = NULL;
@@ -745,6 +775,11 @@ __iobuf_put (struct iobuf *iobuf, struct iobuf_arena *iobuf_arena)
         list_del_init (&iobuf->list);
         iobuf_arena->active_cnt--;
 
+        if (iobuf->free_ptr) {
+                iobuf->ptr = iobuf->free_ptr;
+                iobuf->free_ptr = NULL;
+        }
+
         list_add (&iobuf->list, &iobuf_arena->passive.list);
         iobuf_arena->passive_cnt++;
 
diff --git a/libglusterfs/src/iobuf.h b/libglusterfs/src/iobuf.h
index 7e5cfe37a28..1578ceb84dd 100644
--- a/libglusterfs/src/iobuf.h
+++ b/libglusterfs/src/iobuf.h
@@ -169,4 +169,8 @@ void   iobuf_stats_dump (struct iobuf_pool *iobuf_pool);
 
 struct iobuf *
 iobuf_get2 (struct iobuf_pool *iobuf_pool, size_t page_size);
+
+struct iobuf *
+iobuf_get_page_aligned (struct iobuf_pool *iobuf_pool, size_t page_size,
+                        size_t align_size);
 #endif /* !_IOBUF_H_ */