mem-pool: track glusterfs_ctx_t in struct mem_pool

In order to generate statedumps per glusterfs_ctx_t, it is needed to
place all the memory pools in a structure that the context can reach.
The 'struct mem_pool' has been extended with a 'list_head owner' that is
linked with the glusterfs_ctx_t->mempool_list.

All callers of mem_pool_new() have been updated to pass the current
glusterfs_ctx_t along. This context is needed to add the new memory pool
to the list and for grabbing the ctx->lock while updating the
glusterfs_ctx_t->mempool_list.

Updates: #307
Change-Id: Ia9384424d8d1630ef3efc9d5d523bf739c356c6e
Signed-off-by: Niels de Vos <ndevos@redhat.com>
Reviewed-on: https://review.gluster.org/18075
Smoke: Gluster Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>

10 files changed, 38 insertions, 18 deletions

diff --git a/libglusterfs/src/ctx.c b/libglusterfs/src/ctx.c
index 94c56acdea5..90480d09774 100644
--- a/libglusterfs/src/ctx.c
+++ b/libglusterfs/src/ctx.c
@@ -31,9 +31,7 @@ glusterfs_ctx_new ()
         ctx->mem_acct_enable = gf_global_mem_acct_enable_get();
 
         INIT_LIST_HEAD (&ctx->graphs);
-#if defined(OLD_MEM_POOLS)
         INIT_LIST_HEAD (&ctx->mempool_list);
-#endif
         INIT_LIST_HEAD (&ctx->volfile_list);
 
 	ctx->daemon_pipe[0] = -1;
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 8a55d724c6e..37ce5d91a1c 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -502,11 +502,9 @@ struct _glusterfs_ctx {
         int                process_mode; /*mode in which process is runninng*/
         struct syncenv    *env;          /* The env pointer to the synctasks */
 
-#if defined(OLD_MEM_POOLS)
         struct list_head   mempool_list; /* used to keep a global list of
                                             mempools, used to log details of
                                             mempool in statedump */
-#endif
         char              *statedump_path;
 
         struct mem_pool   *dict_pool;
diff --git a/libglusterfs/src/mem-pool.c b/libglusterfs/src/mem-pool.c
index 8bd9779ce81..66b45c0d669 100644
--- a/libglusterfs/src/mem-pool.c
+++ b/libglusterfs/src/mem-pool.c
@@ -663,7 +663,7 @@ void mem_pools_fini (void) {}
 #endif
 
 struct mem_pool *
-mem_pool_new_fn (unsigned long sizeof_type,
+mem_pool_new_fn (glusterfs_ctx_t *ctx, unsigned long sizeof_type,
                  unsigned long count, char *name)
 {
         unsigned int            i;
@@ -693,10 +693,18 @@ mem_pool_new_fn (unsigned long sizeof_type,
         if (!new)
                 return NULL;
 
+        new->ctx = ctx;
         new->sizeof_type = sizeof_type;
         new->count = count;
         new->name = name;
         new->pool = pool;
+        INIT_LIST_HEAD (&new->owner);
+
+        LOCK (&ctx->lock);
+        {
+                list_add (&new->owner, &ctx->mempool_list);
+        }
+        UNLOCK (&ctx->lock);
 
         return new;
 }
@@ -867,6 +875,14 @@ mem_put (void *ptr)
 void
 mem_pool_destroy (struct mem_pool *pool)
 {
+        /* remove this pool from the owner (glusterfs_ctx_t) */
+        LOCK (&pool->ctx->lock);
+        {
+                list_del (&pool->owner);
+        }
+        UNLOCK (&pool->ctx->lock);
+
+        /* free this pool, but keep the mem_pool_shared */
         GF_FREE (pool);
 
         /*
diff --git a/libglusterfs/src/mem-pool.h b/libglusterfs/src/mem-pool.h
index cc7df0b77c8..ecf2abe9245 100644
--- a/libglusterfs/src/mem-pool.h
+++ b/libglusterfs/src/mem-pool.h
@@ -16,6 +16,7 @@
 #include "atomic.h"
 #include "logging.h"
 #include "mem-types.h"
+#include "glusterfs.h" /* for glusterfs_ctx_t */
 #include <stdlib.h>
 #include <inttypes.h>
 #include <string.h>
@@ -206,11 +207,15 @@ out:
 /* kind of 'header' for the actual mem_pool_shared structure, this might make
  * it possible to dump some more details in a statedump */
 struct mem_pool {
+        /* object size, without pooled_obj_hdr_t */
         unsigned long           sizeof_type;
         unsigned long           count;
         char                    *name;
 
-        struct mem_pool_shared  *pool;
+        struct list_head        owner;  /* glusterfs_ctx_t->mempool_list */
+        glusterfs_ctx_t         *ctx;   /* take ctx->lock when updating owner */
+
+        struct mem_pool_shared  *pool;  /* the initial pool that was returned */
 };
 
 typedef struct pooled_obj_hdr {
@@ -275,9 +280,14 @@ void mem_pools_init_late (void);   /* start the pool_sweeper thread */
 void mem_pools_fini (void);        /* cleanup memory pools */
 
 struct mem_pool *
-mem_pool_new_fn (unsigned long sizeof_type, unsigned long count, char *name);
+mem_pool_new_fn (glusterfs_ctx_t *ctx, unsigned long sizeof_type, unsigned long
+                 count, char *name);
+
+#define mem_pool_new(type,count) \
+        mem_pool_new_fn (THIS->ctx, sizeof(type), count, #type)
 
-#define mem_pool_new(type,count) mem_pool_new_fn (sizeof(type), count, #type)
+#define mem_pool_new_ctx(ctx,type,count) \
+        mem_pool_new_fn (ctx, sizeof(type), count, #type)
 
 void mem_put (void *ptr);
 void *mem_get (struct mem_pool *pool);
diff --git a/libglusterfs/src/rbthash.c b/libglusterfs/src/rbthash.c
index 52d8a15fd2c..06fc7ee35c8 100644
--- a/libglusterfs/src/rbthash.c
+++ b/libglusterfs/src/rbthash.c
@@ -83,7 +83,7 @@ err:
  */
 
 rbthash_table_t *
-rbthash_table_init (int buckets, rbt_hasher_t hfunc,
+rbthash_table_init (glusterfs_ctx_t *ctx, int buckets, rbt_hasher_t hfunc,
                     rbt_data_destroyer_t dfunc,
                     unsigned long expected_entries,
                     struct mem_pool *entrypool)
@@ -123,7 +123,8 @@ rbthash_table_init (int buckets, rbt_hasher_t hfunc,
 
         if (expected_entries) {
                 newtab->entrypool =
-                        mem_pool_new (rbthash_entry_t, expected_entries);
+                        mem_pool_new_ctx (ctx, rbthash_entry_t,
+                                          expected_entries);
                 if (!newtab->entrypool) {
                         goto free_buckets;
                 }
diff --git a/libglusterfs/src/rbthash.h b/libglusterfs/src/rbthash.h
index b093ce9982d..949b88a4380 100644
--- a/libglusterfs/src/rbthash.h
+++ b/libglusterfs/src/rbthash.h
@@ -52,7 +52,7 @@ typedef struct rbthash_table {
 } rbthash_table_t;
 
 extern rbthash_table_t *
-rbthash_table_init (int buckets, rbt_hasher_t hfunc,
+rbthash_table_init (glusterfs_ctx_t *ctx, int buckets, rbt_hasher_t hfunc,
                     rbt_data_destroyer_t dfunc, unsigned long expected_entries,
                     struct mem_pool *entrypool);
 
diff --git a/libglusterfs/src/statedump.c b/libglusterfs/src/statedump.c
index a2000b158e6..67c46998d63 100644
--- a/libglusterfs/src/statedump.c
+++ b/libglusterfs/src/statedump.c
@@ -399,14 +399,10 @@ gf_proc_dump_mempool_info (glusterfs_ctx_t *ctx)
         LOCK (&ctx->lock);
         {
                 list_for_each_entry (pool, &ctx->mempool_list, owner) {
-                        int64_t active = GF_ATOMIC_GET (pool->active);
-
                         gf_proc_dump_write ("-----", "-----");
                         gf_proc_dump_write ("pool-name", "%s", pool->name);
-                        gf_proc_dump_write ("active-count", "%"GF_PRI_ATOMIC, active);
                         gf_proc_dump_write ("sizeof-type", "%d", pool->sizeof_type);
                         gf_proc_dump_write ("padded-sizeof", "%lu", 1 << pool->pool->power_of_two);
-                        gf_proc_dump_write ("size", "%lu", (1 << pool->pool->power_of_two) * active);
                         gf_proc_dump_write ("shared-pool", "%p", pool->pool);
                 }
         }
diff --git a/xlators/cluster/ec/src/ec-method.c b/xlators/cluster/ec/src/ec-method.c
index e0dd8e7f1a0..a2dd2bd3731 100644
--- a/xlators/cluster/ec/src/ec-method.c
+++ b/xlators/cluster/ec/src/ec-method.c
@@ -310,7 +310,7 @@ ec_method_init(xlator_t *xl, ec_matrix_list_t *list, uint32_t columns,
     INIT_LIST_HEAD(&list->lru);
     int32_t err;
 
-    list->pool = mem_pool_new_fn(sizeof(ec_matrix_t) +
+    list->pool = mem_pool_new_fn(xl->ctx, sizeof(ec_matrix_t) +
                                  sizeof(ec_matrix_row_t) * columns +
                                  sizeof(uint32_t) * columns * columns,
                                  128, "ec_matrix_t");
diff --git a/xlators/nfs/server/src/nfs3.c b/xlators/nfs/server/src/nfs3.c
index 040d31604bf..b053eb3250e 100644
--- a/xlators/nfs/server/src/nfs3.c
+++ b/xlators/nfs/server/src/nfs3.c
@@ -5786,7 +5786,8 @@ nfs3_init_state (xlator_t *nfsx)
 
         localpool = nfs->memfactor * GF_NFS_CONCURRENT_OPS_MULT;
         gf_msg_trace (GF_NFS3, 0, "local pool: %d", localpool);
-        nfs3->localpool = mem_pool_new (nfs3_call_state_t, localpool);
+        nfs3->localpool = mem_pool_new_ctx (nfsx->ctx, nfs3_call_state_t,
+                                            localpool);
         if (!nfs3->localpool) {
                 gf_msg (GF_NFS3, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
                         "local mempool creation failed");
diff --git a/xlators/performance/io-cache/src/io-cache.c b/xlators/performance/io-cache/src/io-cache.c
index 896394231f9..3e2e96b3fef 100644
--- a/xlators/performance/io-cache/src/io-cache.c
+++ b/xlators/performance/io-cache/src/io-cache.c
@@ -1146,7 +1146,7 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 if (!ioc_inode->cache.page_table) {
                         ioc_inode->cache.page_table
                                 = rbthash_table_init
-                                (IOC_PAGE_TABLE_BUCKET_COUNT,
+                                (this->ctx, IOC_PAGE_TABLE_BUCKET_COUNT,
                                  ioc_hashfn, NULL, 0,
                                  table->mem_pool);