Changed rbthash_table_init() to take a mem-pool argument.

Changes in libglusterfs/rbthash:

rbthash_table_init() now takes a mem-pool argument.
The mem-pool argument would be mutually exclusive to expected_entries.
If expected_entries is provided, mem-pool would be ignored and vice-versa.

Changes in io-cache:
1) Moved rbthash creation to readv.
2) rbthash makes use of 1 rbt instead of 4096
3) A global mem-pool is being used in place of a mem-pool per rbt.

Signed-off-by: Vijay Bellur <vijay@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 335 (Io-cache optimization)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=335

6 files changed, 115 insertions, 31 deletions

diff --git a/libglusterfs/src/rbthash.c b/libglusterfs/src/rbthash.c
index c2cbf99b4a1..63d08385883 100644
--- a/libglusterfs/src/rbthash.c
+++ b/libglusterfs/src/rbthash.c
@@ -79,9 +79,21 @@ err:
 }
 
 
+/*
+ * rbthash_table_init - Initialize a RBT based hash table
+ * @buckets - Number of buckets in the hash table
+ * @hfunc   - hashing function
+ * @dfunc   - destroyer for data in the RBT
+ * @expected_entries - Number of entries expected in RBT. Mutually exclusive
+ * with entrypool.
+ * @entrypool -  Memory pool in lieu of expected_entries.
+ */
+
 rbthash_table_t *
-rbthash_table_init (int buckets, rbt_hasher_t hfunc, rbt_data_destroyer_t dfunc,
-                    unsigned long expected_entries)
+rbthash_table_init (int buckets, rbt_hasher_t hfunc,
+                    rbt_data_destroyer_t dfunc,
+                    unsigned long expected_entries,
+                    struct mem_pool *entrypool)
 {
         rbthash_table_t         *newtab = NULL;
         int                     ret = -1;
@@ -91,6 +103,19 @@ rbthash_table_init (int buckets, rbt_hasher_t hfunc, rbt_data_destroyer_t dfunc,
                 return NULL;
         }
 
+        if (!entrypool && !expected_entries) {
+                gf_log (GF_RBTHASH, GF_LOG_ERROR,
+                        "Both mem-pool and expected entries not provided");
+                return NULL;
+        }
+
+        if (entrypool && expected_entries) {
+                gf_log (GF_RBTHASH, GF_LOG_ERROR,
+                        "Both mem-pool and expected entries are provided");
+                return NULL;
+        }
+
+
         newtab = CALLOC (1, sizeof (*newtab));
         if (!newtab)
                 return NULL;
@@ -101,10 +126,17 @@ rbthash_table_init (int buckets, rbt_hasher_t hfunc, rbt_data_destroyer_t dfunc,
                 goto free_newtab;
         }
 
-        newtab->entrypool =  mem_pool_new (rbthash_entry_t, expected_entries);
-        if (!newtab->entrypool) {
-                gf_log (GF_RBTHASH, GF_LOG_ERROR,"Failed to allocate mem-pool");
-                goto free_buckets;
+        if (expected_entries) {
+                newtab->entrypool =
+                        mem_pool_new (rbthash_entry_t, expected_entries);
+                if (!newtab->entrypool) {
+                        gf_log (GF_RBTHASH, GF_LOG_ERROR,
+                                "Failed to allocate mem-pool");
+                        goto free_buckets;
+                }
+                newtab->pool_alloced = _gf_true;
+        } else {
+                newtab->entrypool = entrypool;
         }
 
         LOCK_INIT (&newtab->tablelock);
@@ -113,13 +145,16 @@ rbthash_table_init (int buckets, rbt_hasher_t hfunc, rbt_data_destroyer_t dfunc,
 
         if (ret == -1) {
                 gf_log (GF_RBTHASH, GF_LOG_ERROR, "Failed to init buckets");
-                mem_pool_destroy (newtab->entrypool);
-        } else
+                if (newtab->pool_alloced)
+                        mem_pool_destroy (newtab->entrypool);
+        } else {
                 gf_log (GF_RBTHASH, GF_LOG_TRACE, "Inited hash table: buckets:"
                         " %d", buckets);
+        }
 
         newtab->hashfunc = hfunc;
         newtab->dfunc = dfunc;
+
 free_buckets:
         if (ret == -1)
                 FREE (newtab->buckets);
@@ -133,7 +168,6 @@ free_newtab:
         return newtab;
 }
 
-
 rbthash_entry_t *
 rbthash_init_entry (rbthash_table_t *tbl, void *data, void *key, int keylen)
 {
@@ -381,7 +415,8 @@ rbthash_table_destroy (rbthash_table_t *tbl)
                 return;
 
         rbthash_table_destroy_buckets (tbl);
-        mem_pool_destroy (tbl->entrypool);
+        if (tbl->pool_alloced)
+                mem_pool_destroy (tbl->entrypool);
 
         FREE (tbl->buckets);
         FREE (tbl);
diff --git a/libglusterfs/src/rbthash.h b/libglusterfs/src/rbthash.h
index f2200f8957c..535dccad926 100644
--- a/libglusterfs/src/rbthash.h
+++ b/libglusterfs/src/rbthash.h
@@ -23,6 +23,7 @@
 #include "locking.h"
 #include "mem-pool.h"
 #include "logging.h"
+#include "common-utils.h"
 
 #include <pthread.h>
 
@@ -52,11 +53,13 @@ typedef struct rbthash_table {
         struct rbthash_bucket   *buckets;
         rbt_hasher_t            hashfunc;
         rbt_data_destroyer_t    dfunc;
+        gf_boolean_t            pool_alloced;
 } rbthash_table_t;
 
 extern rbthash_table_t *
 rbthash_table_init (int buckets, rbt_hasher_t hfunc,
-                    rbt_data_destroyer_t dfunc, unsigned long expected_entries);
+                    rbt_data_destroyer_t dfunc, unsigned long expected_entries,
+                    struct mem_pool *entrypool);
 
 extern int
 rbthash_insert (rbthash_table_t *tbl, void *data, void *key, int keylen);
diff --git a/xlators/performance/io-cache/src/io-cache.c b/xlators/performance/io-cache/src/io-cache.c
index 83948ff0e3f..3fb4c850efd 100644
--- a/xlators/performance/io-cache/src/io-cache.c
+++ b/xlators/performance/io-cache/src/io-cache.c
@@ -891,6 +891,13 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	ioc_inode_t  *ioc_inode = NULL;
 	ioc_local_t  *local = NULL;
 	uint32_t     weight = 0;
+        ioc_table_t  *table = NULL;
+        uint32_t     num_pages = 0;
+        int32_t      op_errno = -1;
+
+        if (!this) {
+                goto out;
+        }
 
 	inode_ctx_get (fd->inode, this, &tmp_ioc_inode);
 	ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
@@ -903,6 +910,47 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
 		return 0;
 	}
 
+
+        table = this->private;
+
+        if (!table) {
+                gf_log (this->name, GF_LOG_ERROR, "table is null");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+
+        ioc_table_lock (table);
+        if (!table->mem_pool) {
+
+                num_pages = (table->cache_size / table->page_size)
+                        + ((table->cache_size % table->page_size) ? 1 : 0);
+
+                table->mem_pool
+                        =  mem_pool_new (rbthash_entry_t, num_pages);
+
+                if (!table->mem_pool) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to allocate mem_pool");
+                        op_errno = ENOMEM;
+                        ioc_table_unlock (table);
+                        goto out;
+                }
+        }
+        ioc_table_unlock (table);
+
+        if (!ioc_inode->cache.page_table) {
+                ioc_inode->cache.page_table
+                        = rbthash_table_init (IOC_PAGE_TABLE_BUCKET_COUNT,
+                                              ioc_hashfn, NULL, 0,
+                                              table->mem_pool);
+
+                if (ioc_inode->cache.page_table == NULL) {
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+        }
+
 	if (!fd_ctx_get (fd, this, NULL)) {
 		/* disable caching for this fd, go ahead with normal readv */
 		STACK_WIND (frame, ioc_readv_disabled_cbk,
@@ -915,9 +963,8 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	local = (ioc_local_t *) CALLOC (1, sizeof (ioc_local_t));
         if (local == NULL) {
                 gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM, NULL, 0, NULL,
-                                     NULL);
-                return 0;
+                op_errno = ENOMEM;
+                goto out;
         }
 
 	INIT_LIST_HEAD (&local->fill_list);
@@ -943,8 +990,11 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	ioc_table_unlock (ioc_inode->table);
 
 	ioc_dispatch_requests (frame, ioc_inode, fd, offset, size);
-  
 	return 0;
+
+out:
+        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        return 0;
 }
 
 /*
diff --git a/xlators/performance/io-cache/src/io-cache.h b/xlators/performance/io-cache/src/io-cache.h
index de6cb3527ee..44f621f8078 100644
--- a/xlators/performance/io-cache/src/io-cache.h
+++ b/xlators/performance/io-cache/src/io-cache.h
@@ -41,7 +41,7 @@
 
 #define IOC_PAGE_SIZE    (1024 * 128)   /* 128KB */
 #define IOC_CACHE_SIZE   (32 * 1024 * 1024)
-#define IOC_PAGE_TABLE_BUCKET_COUNT 4096
+#define IOC_PAGE_TABLE_BUCKET_COUNT 1
 
 struct ioc_table;
 struct ioc_local;
@@ -169,6 +169,7 @@ struct ioc_table {
 	uint32_t         inode_count;
 	int32_t          cache_timeout;
 	int32_t          max_pri;
+        struct mem_pool  *mem_pool;
 };
 
 typedef struct ioc_table ioc_table_t;
@@ -332,4 +333,6 @@ ioc_prune (ioc_table_t *table);
 int32_t
 ioc_need_prune (ioc_table_t *table);
 
+inline uint32_t
+ioc_hashfn (void *data, int len);
 #endif /* __IO_CACHE_H */
diff --git a/xlators/performance/io-cache/src/ioc-inode.c b/xlators/performance/io-cache/src/ioc-inode.c
index 11da863123f..9ac5469f5e3 100644
--- a/xlators/performance/io-cache/src/ioc-inode.c
+++ b/xlators/performance/io-cache/src/ioc-inode.c
@@ -177,16 +177,6 @@ ioc_inode_update (ioc_table_t *table, inode_t *inode, uint32_t weight)
         no_of_pages = (table->cache_size / table->page_size)
                 + ((table->cache_size % table->page_size) ? 1 : 0);
 
-	/* initialize the list for pages */
-        ioc_inode->cache.page_table = rbthash_table_init (IOC_PAGE_TABLE_BUCKET_COUNT,
-                                                          ioc_hashfn, NULL,
-                                                          no_of_pages);
-        if (ioc_inode->cache.page_table == NULL) {
-                FREE (ioc_inode);
-                ioc_inode = NULL;
-                goto out;
-        }
-
 	INIT_LIST_HEAD (&ioc_inode->cache.page_lru);
 
 	ioc_table_lock (table);
diff --git a/xlators/performance/stat-prefetch/src/stat-prefetch.c b/xlators/performance/stat-prefetch/src/stat-prefetch.c
index 29d915c1a3d..566dd18094f 100644
--- a/xlators/performance/stat-prefetch/src/stat-prefetch.c
+++ b/xlators/performance/stat-prefetch/src/stat-prefetch.c
@@ -171,10 +171,12 @@ sp_cache_init (void)
 
         cache = CALLOC (1, sizeof (*cache));
         if (cache) {
-                cache->table = rbthash_table_init (GF_SP_CACHE_BUCKETS,
-                                                   sp_hashfn,
-                                                   free,
-                                                   GF_SP_CACHE_ENTRIES_EXPECTED);
+                cache->table =
+                        rbthash_table_init (GF_SP_CACHE_BUCKETS,
+                                            sp_hashfn,
+                                            free,
+                                            GF_SP_CACHE_ENTRIES_EXPECTED,
+                                            NULL);
                 if (cache->table == NULL) {
                         FREE (cache);
                         cache = NULL;
@@ -216,7 +218,8 @@ sp_cache_remove_entry (sp_cache_t *cache, char *name, char remove_all)
                         cache->table = rbthash_table_init (GF_SP_CACHE_BUCKETS,
                                                            sp_hashfn,
                                                            free,
-                                                           GF_SP_CACHE_ENTRIES_EXPECTED);
+                                                           GF_SP_CACHE_ENTRIES_EXPECTED,
+                                                           NULL);
                         if (cache->table == NULL) {
                                 cache->table = table;
                         } else {