Changed rbthash_table_init() to take a mem-pool argument.

Changes in libglusterfs/rbthash:

rbthash_table_init() now takes a mem-pool argument.
The mem-pool argument would be mutually exclusive to expected_entries.
If expected_entries is provided, mem-pool would be ignored and vice-versa.

Changes in io-cache:
1) Moved rbthash creation to readv.
2) rbthash makes use of 1 rbt instead of 4096
3) A global mem-pool is being used in place of a mem-pool per rbt.

Signed-off-by: Vijay Bellur <vijay@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 335 (Io-cache optimization)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=335

4 files changed, 66 insertions, 20 deletions

diff --git a/xlators/performance/io-cache/src/io-cache.c b/xlators/performance/io-cache/src/io-cache.c
index 83948ff0e3f..3fb4c850efd 100644
--- a/xlators/performance/io-cache/src/io-cache.c
+++ b/xlators/performance/io-cache/src/io-cache.c
@@ -891,6 +891,13 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	ioc_inode_t  *ioc_inode = NULL;
 	ioc_local_t  *local = NULL;
 	uint32_t     weight = 0;
+        ioc_table_t  *table = NULL;
+        uint32_t     num_pages = 0;
+        int32_t      op_errno = -1;
+
+        if (!this) {
+                goto out;
+        }
 
 	inode_ctx_get (fd->inode, this, &tmp_ioc_inode);
 	ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
@@ -903,6 +910,47 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
 		return 0;
 	}
 
+
+        table = this->private;
+
+        if (!table) {
+                gf_log (this->name, GF_LOG_ERROR, "table is null");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+
+        ioc_table_lock (table);
+        if (!table->mem_pool) {
+
+                num_pages = (table->cache_size / table->page_size)
+                        + ((table->cache_size % table->page_size) ? 1 : 0);
+
+                table->mem_pool
+                        =  mem_pool_new (rbthash_entry_t, num_pages);
+
+                if (!table->mem_pool) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to allocate mem_pool");
+                        op_errno = ENOMEM;
+                        ioc_table_unlock (table);
+                        goto out;
+                }
+        }
+        ioc_table_unlock (table);
+
+        if (!ioc_inode->cache.page_table) {
+                ioc_inode->cache.page_table
+                        = rbthash_table_init (IOC_PAGE_TABLE_BUCKET_COUNT,
+                                              ioc_hashfn, NULL, 0,
+                                              table->mem_pool);
+
+                if (ioc_inode->cache.page_table == NULL) {
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+        }
+
 	if (!fd_ctx_get (fd, this, NULL)) {
 		/* disable caching for this fd, go ahead with normal readv */
 		STACK_WIND (frame, ioc_readv_disabled_cbk,
@@ -915,9 +963,8 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	local = (ioc_local_t *) CALLOC (1, sizeof (ioc_local_t));
         if (local == NULL) {
                 gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM, NULL, 0, NULL,
-                                     NULL);
-                return 0;
+                op_errno = ENOMEM;
+                goto out;
         }
 
 	INIT_LIST_HEAD (&local->fill_list);
@@ -943,8 +990,11 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	ioc_table_unlock (ioc_inode->table);
 
 	ioc_dispatch_requests (frame, ioc_inode, fd, offset, size);
-  
 	return 0;
+
+out:
+        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        return 0;
 }
 
 /*
diff --git a/xlators/performance/io-cache/src/io-cache.h b/xlators/performance/io-cache/src/io-cache.h
index de6cb3527ee..44f621f8078 100644
--- a/xlators/performance/io-cache/src/io-cache.h
+++ b/xlators/performance/io-cache/src/io-cache.h
@@ -41,7 +41,7 @@
 
 #define IOC_PAGE_SIZE    (1024 * 128)   /* 128KB */
 #define IOC_CACHE_SIZE   (32 * 1024 * 1024)
-#define IOC_PAGE_TABLE_BUCKET_COUNT 4096
+#define IOC_PAGE_TABLE_BUCKET_COUNT 1
 
 struct ioc_table;
 struct ioc_local;
@@ -169,6 +169,7 @@ struct ioc_table {
 	uint32_t         inode_count;
 	int32_t          cache_timeout;
 	int32_t          max_pri;
+        struct mem_pool  *mem_pool;
 };
 
 typedef struct ioc_table ioc_table_t;
@@ -332,4 +333,6 @@ ioc_prune (ioc_table_t *table);
 int32_t
 ioc_need_prune (ioc_table_t *table);
 
+inline uint32_t
+ioc_hashfn (void *data, int len);
 #endif /* __IO_CACHE_H */
diff --git a/xlators/performance/io-cache/src/ioc-inode.c b/xlators/performance/io-cache/src/ioc-inode.c
index 11da863123f..9ac5469f5e3 100644
--- a/xlators/performance/io-cache/src/ioc-inode.c
+++ b/xlators/performance/io-cache/src/ioc-inode.c
@@ -177,16 +177,6 @@ ioc_inode_update (ioc_table_t *table, inode_t *inode, uint32_t weight)
         no_of_pages = (table->cache_size / table->page_size)
                 + ((table->cache_size % table->page_size) ? 1 : 0);
 
-	/* initialize the list for pages */
-        ioc_inode->cache.page_table = rbthash_table_init (IOC_PAGE_TABLE_BUCKET_COUNT,
-                                                          ioc_hashfn, NULL,
-                                                          no_of_pages);
-        if (ioc_inode->cache.page_table == NULL) {
-                FREE (ioc_inode);
-                ioc_inode = NULL;
-                goto out;
-        }
-
 	INIT_LIST_HEAD (&ioc_inode->cache.page_lru);
 
 	ioc_table_lock (table);
diff --git a/xlators/performance/stat-prefetch/src/stat-prefetch.c b/xlators/performance/stat-prefetch/src/stat-prefetch.c
index 29d915c1a3d..566dd18094f 100644
--- a/xlators/performance/stat-prefetch/src/stat-prefetch.c
+++ b/xlators/performance/stat-prefetch/src/stat-prefetch.c
@@ -171,10 +171,12 @@ sp_cache_init (void)
 
         cache = CALLOC (1, sizeof (*cache));
         if (cache) {
-                cache->table = rbthash_table_init (GF_SP_CACHE_BUCKETS,
-                                                   sp_hashfn,
-                                                   free,
-                                                   GF_SP_CACHE_ENTRIES_EXPECTED);
+                cache->table =
+                        rbthash_table_init (GF_SP_CACHE_BUCKETS,
+                                            sp_hashfn,
+                                            free,
+                                            GF_SP_CACHE_ENTRIES_EXPECTED,
+                                            NULL);
                 if (cache->table == NULL) {
                         FREE (cache);
                         cache = NULL;
@@ -216,7 +218,8 @@ sp_cache_remove_entry (sp_cache_t *cache, char *name, char remove_all)
                         cache->table = rbthash_table_init (GF_SP_CACHE_BUCKETS,
                                                            sp_hashfn,
                                                            free,
-                                                           GF_SP_CACHE_ENTRIES_EXPECTED);
+                                                           GF_SP_CACHE_ENTRIES_EXPECTED,
+                                                           NULL);
                         if (cache->table == NULL) {
                                 cache->table = table;
                         } else {