mem-pool: Make mem-pool ptr avialable in ptr

The header of the ptr returned from mem-pool will now store the
mem-pool ptr it belongs to. mem_put will now take only the pointer
to be freed.

Also, changing MALLOC call to GF_CALLOC in mem_get when we run out
of entries in mem-pool. This also will have the header information
saved.

Change-Id: I3de182663a7f5b49c9e9425e9531775b70bdff67
BUG: 3390
Reviewed-on: http://review.gluster.com/205
Reviewed-by: Amar Tumballi <amar@gluster.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

7 files changed, 44 insertions, 24 deletions

diff --git a/libglusterfs/src/call-stub.c b/libglusterfs/src/call-stub.c
index fab368a6ddd..67543c64a13 100644
--- a/libglusterfs/src/call-stub.c
+++ b/libglusterfs/src/call-stub.c
@@ -3835,7 +3835,7 @@ call_stub_destroy (call_stub_t *stub)
         }
 
         stub->stub_mem_pool = NULL;
-        mem_put (tmp_pool, stub);
+        mem_put (stub);
 out:
         tmp_pool = NULL;
 
diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index 72df5e1d395..d6c4806a11a 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -428,7 +428,7 @@ fd_destroy (fd_t *fd)
         GF_FREE (fd->_ctx);
         inode_unref (fd->inode);
         fd->inode = (inode_t *)0xaaaaaaaa;
-        mem_put (tmp_pool,fd);
+        mem_put (fd);
         tmp_pool = NULL;
 out:
         return;
@@ -500,7 +500,7 @@ fd_create (inode_t *inode, pid_t pid)
         fd->_ctx = GF_CALLOC (1, (sizeof (struct _fd_ctx) * fd->xl_count),
                               gf_common_mt_fd_ctx);
         if (!fd->_ctx) {
-                mem_put (inode->table->fd_mem_pool, fd);
+                mem_put (fd);
                 fd = NULL;
                 goto out;
         }
diff --git a/libglusterfs/src/inode.c b/libglusterfs/src/inode.c
index a3709cdb34d..1b43203113f 100644
--- a/libglusterfs/src/inode.c
+++ b/libglusterfs/src/inode.c
@@ -154,7 +154,7 @@ __dentry_unset (dentry_t *dentry)
                 dentry->parent = NULL;
         }
 
-        mem_put (tmp_pool, dentry);
+        mem_put (dentry);
         tmp_pool = NULL;
 
 }
@@ -336,7 +336,7 @@ __inode_destroy (inode_t *inode)
 noctx:
         LOCK_DESTROY (&inode->lock);
         //  memset (inode, 0xb, sizeof (*inode));
-        mem_put (tmp_pool, inode);
+        mem_put (inode);
         tmp_pool = NULL;
 
 }
@@ -501,7 +501,7 @@ __dentry_create (inode_t *inode, inode_t *parent, const char *name)
 
         newd->name = gf_strdup (name);
         if (newd->name == NULL) {
-                mem_put (parent->table->dentry_pool, newd);
+                mem_put (newd);
                 newd = NULL;
                 goto out;
         }
@@ -547,7 +547,7 @@ __inode_create (inode_table_t *table)
 
         if (newi->_ctx == NULL) {
                 LOCK_DESTROY (&newi->lock);
-                mem_put (table->inode_pool, newi);
+                mem_put (newi);
                 newi = NULL;
                 goto out;
         }
diff --git a/libglusterfs/src/mem-pool.c b/libglusterfs/src/mem-pool.c
index 5fdd5e5f2d1..f3dfc2149e4 100644
--- a/libglusterfs/src/mem-pool.c
+++ b/libglusterfs/src/mem-pool.c
@@ -24,10 +24,12 @@
 #include <stdarg.h>
 
 #define GF_MEM_POOL_LIST_BOUNDARY        (sizeof(struct list_head))
-#define GF_MEM_POOL_PAD_BOUNDARY         (GF_MEM_POOL_LIST_BOUNDARY + sizeof(int))
+#define GF_MEM_POOL_PTR                  (sizeof(struct mem_pool*))
+#define GF_MEM_POOL_PAD_BOUNDARY         (GF_MEM_POOL_LIST_BOUNDARY  + GF_MEM_POOL_PTR + sizeof(int))
 #define mem_pool_chunkhead2ptr(head)     ((head) + GF_MEM_POOL_PAD_BOUNDARY)
 #define mem_pool_ptr2chunkhead(ptr)      ((ptr) - GF_MEM_POOL_PAD_BOUNDARY)
 #define is_mem_chunk_in_use(ptr)         (*ptr == 1)
+#define mem_pool_from_ptr(ptr)           ((ptr) + GF_MEM_POOL_LIST_BOUNDARY)
 
 #define GF_MEM_HEADER_SIZE  (4 + sizeof (size_t) + sizeof (xlator_t *) + 4 + 8)
 #define GF_MEM_TRAILER_SIZE 8
@@ -379,6 +381,7 @@ mem_get (struct mem_pool *mem_pool)
         struct list_head *list = NULL;
         void             *ptr = NULL;
         int             *in_use = NULL;
+        struct mem_pool **pool_ptr = NULL;
 
         if (!mem_pool) {
                 gf_log ("mem-pool", GF_LOG_ERROR, "invalid argument");
@@ -395,7 +398,8 @@ mem_get (struct mem_pool *mem_pool)
                         mem_pool->cold_count--;
 
                         ptr = list;
-                        in_use = (ptr + GF_MEM_POOL_LIST_BOUNDARY);
+                        in_use = (ptr + GF_MEM_POOL_LIST_BOUNDARY +
+                                  GF_MEM_POOL_PTR);
                         *in_use = 1;
 
                         goto fwd_addr_out;
@@ -421,17 +425,20 @@ mem_get (struct mem_pool *mem_pool)
                  * because it is too much work knowing that a better slab
                  * allocator is coming RSN.
                  */
-                ptr = MALLOC (mem_pool->real_sizeof_type);
+                ptr = GF_CALLOC (1, mem_pool->padded_sizeof_type,
+                                 gf_common_mt_mem_pool);
+                gf_log_callingfn ("mem-pool", GF_LOG_DEBUG, "Mem pool is full. "
+                                  "Callocing mem");
 
                 /* Memory coming from the heap need not be transformed from a
                  * chunkhead to a usable pointer since it is not coming from
                  * the pool.
                  */
-                goto unlocked_out;
         }
 fwd_addr_out:
+        pool_ptr = mem_pool_from_ptr (ptr);
+        *pool_ptr = (struct mem_pool *)mem_pool;
         ptr = mem_pool_chunkhead2ptr (ptr);
-unlocked_out:
         UNLOCK (&mem_pool->lock);
 
         return ptr;
@@ -458,25 +465,39 @@ __is_member (struct mem_pool *pool, void *ptr)
 
 
 void
-mem_put (struct mem_pool *pool, void *ptr)
+mem_put (void *ptr)
 {
         struct list_head *list = NULL;
         int    *in_use = NULL;
         void   *head = NULL;
+        struct mem_pool **tmp = NULL;
+        struct mem_pool *pool = NULL;
 
-        if (!pool || !ptr) {
+        if (!ptr) {
                 gf_log ("mem-pool", GF_LOG_ERROR, "invalid argument");
                 return;
         }
 
+        list = head = mem_pool_ptr2chunkhead (ptr);
+        tmp = mem_pool_from_ptr (head);
+        if (!tmp) {
+                gf_log ("mem-pool", GF_LOG_ERROR, "ptr header is corrupted");
+                return;
+        }
+
+        pool = *tmp;
+        if (!pool) {
+                gf_log ("mem-pool", GF_LOG_ERROR, "mem-pool ptr is NULL");
+                return;
+        }
         LOCK (&pool->lock);
         {
 
                 switch (__is_member (pool, ptr))
                 {
                 case 1:
-                        list = head = mem_pool_ptr2chunkhead (ptr);
-                        in_use = (head + GF_MEM_POOL_LIST_BOUNDARY);
+                        in_use = (head + GF_MEM_POOL_LIST_BOUNDARY +
+                                  GF_MEM_POOL_PTR);
                         if (!is_mem_chunk_in_use(in_use)) {
                                 gf_log_callingfn ("mem-pool", GF_LOG_CRITICAL,
                                                   "mem_put called on freed ptr %p of mem "
@@ -506,7 +527,7 @@ mem_put (struct mem_pool *pool, void *ptr)
                          * not have enough info to distinguish between the two
                          * situations.
                          */
-                        FREE (ptr);
+                        GF_FREE (list);
                         break;
                 default:
                         /* log error */
@@ -516,7 +537,6 @@ mem_put (struct mem_pool *pool, void *ptr)
         UNLOCK (&pool->lock);
 }
 
-
 void
 mem_pool_destroy (struct mem_pool *pool)
 {
diff --git a/libglusterfs/src/mem-pool.h b/libglusterfs/src/mem-pool.h
index 7092ece30d2..98454c5be6f 100644
--- a/libglusterfs/src/mem-pool.h
+++ b/libglusterfs/src/mem-pool.h
@@ -152,7 +152,7 @@ mem_pool_new_fn (unsigned long sizeof_type, unsigned long count);
 
 #define mem_pool_new(type,count) mem_pool_new_fn (sizeof(type), count)
 
-void mem_put (struct mem_pool *pool, void *ptr);
+void mem_put (void *ptr);
 void *mem_get (struct mem_pool *pool);
 void *mem_get0 (struct mem_pool *pool);
 
diff --git a/libglusterfs/src/rbthash.c b/libglusterfs/src/rbthash.c
index c134c3a0012..beda672d0f7 100644
--- a/libglusterfs/src/rbthash.c
+++ b/libglusterfs/src/rbthash.c
@@ -201,7 +201,7 @@ rbthash_init_entry (rbthash_table_t *tbl, void *data, void *key, int keylen)
         ret = 0;
 free_entry:
         if (ret == -1) {
-                mem_put (tbl->entrypool, entry);
+                mem_put (entry);
                 entry = NULL;
         }
 
@@ -230,7 +230,7 @@ rbthash_deinit_entry (rbthash_table_t *tbl, rbthash_entry_t *entry)
                 }
                 UNLOCK (&tbl->tablelock);
 
-                mem_put (tbl->entrypool, entry);
+                mem_put (entry);
         }
 
         return;
@@ -398,7 +398,7 @@ rbthash_remove (rbthash_table_t *tbl, void *key, int keylen)
         }
         UNLOCK (&tbl->tablelock);
 
-        mem_put (tbl->entrypool, entry);
+        mem_put (entry);
 
         return dataref;
 }
diff --git a/libglusterfs/src/stack.h b/libglusterfs/src/stack.h
index 87c723112c6..455a360be23 100644
--- a/libglusterfs/src/stack.h
+++ b/libglusterfs/src/stack.h
@@ -149,7 +149,7 @@ FRAME_DESTROY (call_frame_t *frame)
         }
 
         LOCK_DESTROY (&frame->lock);
-        mem_put (frame->root->pool->frame_mem_pool, frame);
+        mem_put (frame);
 
         if (local)
                 GF_FREE (local);
@@ -178,7 +178,7 @@ STACK_DESTROY (call_stack_t *stack)
         while (stack->frames.next) {
                 FRAME_DESTROY (stack->frames.next);
         }
-        mem_put (stack->pool->stack_mem_pool, stack);
+        mem_put (stack);
 
         if (local)
                 GF_FREE (local);