mempool: add more counters to understand the usage scenarios properly

current design of mempool is to fallback to standard calloc/free if
all the buffers in pool are exhausted. Understanding more about those
numbers will help us to tune mempool parameters properly over time.

Change-Id: I2c94373186f7c6a486caff2611c2d9df2c37db3c
Signed-off-by: Amar Tumballi <amarts@redhat.com>
BUG: 797730
Reviewed-on: http://review.gluster.com/2804
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Kaushal M <kaushal@redhat.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

5 files changed, 66 insertions, 7 deletions

diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 995526abbea..34b37cfb4a7 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -4058,6 +4058,8 @@ cli_print_volume_status_mempool (dict_t *dict, char *prefix)
         uint64_t        paddedsizeof = 0;
         uint64_t        alloccount = 0;
         int32_t         maxalloc = 0;
+        uint64_t        pool_misses = 0;
+        int32_t         maxstdalloc = 0;
         char            key[1024] = {0,};
         int             i = 0;
 
@@ -4071,10 +4073,12 @@ cli_print_volume_status_mempool (dict_t *dict, char *prefix)
                 goto out;
 
         cli_out ("Mempool Stats\n-------------");
-        cli_out ("%-30s %9s %9s %12s %10s %8s", "Name", "HotCount","ColdCount",
-                 "PaddedSizeof", "AllocCount", "MaxAlloc");
-        cli_out ("%-30s %9s %9s %12s %10s %8s", "----", "--------", "---------",
-                 "------------", "----------", "--------");
+        cli_out ("%-30s %9s %9s %12s %10s %8s %8s %12s", "Name", "HotCount",
+                 "ColdCount", "PaddedSizeof", "AllocCount", "MaxAlloc",
+                 "Misses", "Max-StdAlloc");
+        cli_out ("%-30s %9s %9s %12s %10s %8s %8s %12s", "----", "--------",
+                 "---------", "------------", "----------",
+                 "--------", "--------", "------------");
 
         for (i = 0; i < mempool_count; i++) {
                 memset (key, 0, sizeof (key));
@@ -4114,9 +4118,21 @@ cli_print_volume_status_mempool (dict_t *dict, char *prefix)
                 if (ret)
                         goto out;
 
-                cli_out ("%-30s %9d %9d %12"PRIu64" %10"PRIu64" %8d", name,
-                         hotcount, coldcount, paddedsizeof, alloccount,
-                         maxalloc);
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.max-stdalloc", prefix, i);
+                ret = dict_get_int32 (dict, key, &maxstdalloc);
+                if (ret)
+                        goto out;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.pool-misses", prefix, i);
+                ret = dict_get_uint64 (dict, key, &pool_misses);
+                if (ret)
+                        goto out;
+
+                cli_out ("%-30s %9d %9d %12"PRIu64" %10"PRIu64" %8d %8"PRIu64
+                         " %12d", name, hotcount, coldcount, paddedsizeof,
+                         alloccount, maxalloc, pool_misses, maxstdalloc);
         }
 
 out:
diff --git a/cli/src/cli-xml-output.c b/cli/src/cli-xml-output.c
index 3e081277bfc..6a22b249b7c 100644
--- a/cli/src/cli-xml-output.c
+++ b/cli/src/cli-xml-output.c
@@ -441,6 +441,27 @@ cli_xml_output_vol_status_mempool (xmlTextWriterPtr writer, dict_t *dict,
                                                        "%d", maxalloc);
                 XML_RET_CHECK_AND_GOTO (ret, out);
 
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.pool-misses", prefix, i);
+                ret = dict_get_uint64 (dict, key, &alloccount);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"poolMisses",
+                                                       "%"PRIu64, alloccount);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.max-stdalloc", prefix, i);
+                ret = dict_get_int32 (dict, key, &maxalloc);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"maxStdAlloc",
+                                                       "%d", maxalloc);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+
                 /* </pool> */
                 ret = xmlTextWriterEndElement (writer);
                 XML_RET_CHECK_AND_GOTO (ret, out);
diff --git a/libglusterfs/src/mem-pool.c b/libglusterfs/src/mem-pool.c
index 2662dc70ab5..0cfd8bd712a 100644
--- a/libglusterfs/src/mem-pool.c
+++ b/libglusterfs/src/mem-pool.c
@@ -451,6 +451,10 @@ mem_get (struct mem_pool *mem_pool)
                  * because it is too much work knowing that a better slab
                  * allocator is coming RSN.
                  */
+                mem_pool->pool_misses++;
+                mem_pool->curr_stdalloc++;
+                if (mem_pool->max_stdalloc < mem_pool->curr_stdalloc)
+                        mem_pool->max_stdalloc = mem_pool->curr_stdalloc;
                 ptr = GF_CALLOC (1, mem_pool->padded_sizeof_type,
                                  gf_common_mt_mem_pool);
                 gf_log_callingfn ("mem-pool", GF_LOG_DEBUG, "Mem pool is full. "
@@ -553,6 +557,7 @@ mem_put (void *ptr)
                          * not have enough info to distinguish between the two
                          * situations.
                          */
+                        pool->curr_stdalloc--;
                         GF_FREE (list);
                         break;
                 default:
diff --git a/libglusterfs/src/mem-pool.h b/libglusterfs/src/mem-pool.h
index b9255eae930..2b1cba0ce74 100644
--- a/libglusterfs/src/mem-pool.h
+++ b/libglusterfs/src/mem-pool.h
@@ -147,7 +147,10 @@ struct mem_pool {
         void             *pool_end;
         int               real_sizeof_type;
         uint64_t          alloc_count;
+        uint64_t          pool_misses;
         int               max_alloc;
+        int               curr_stdalloc;
+        int               max_stdalloc;
         char             *name;
         struct list_head  global_list;
 };
diff --git a/libglusterfs/src/statedump.c b/libglusterfs/src/statedump.c
index dbbccdb44c3..9b15c5ba50d 100644
--- a/libglusterfs/src/statedump.c
+++ b/libglusterfs/src/statedump.c
@@ -305,6 +305,9 @@ gf_proc_dump_mempool_info (glusterfs_ctx_t *ctx)
                                     pool->padded_sizeof_type);
                 gf_proc_dump_write ("alloc-count", "%"PRIu64, pool->alloc_count);
                 gf_proc_dump_write ("max-alloc", "%d", pool->max_alloc);
+
+                gf_proc_dump_write ("pool-misses", "%"PRIu64, pool->pool_misses);
+                gf_proc_dump_write ("max-stdalloc", "%d", pool->max_stdalloc);
         }
 }
 
@@ -356,6 +359,17 @@ gf_proc_dump_mempool_info_to_dict (glusterfs_ctx_t *ctx, dict_t *dict)
                 if (ret)
                         return;
 
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "pool%d.max-stdalloc", count);
+                ret = dict_set_int32 (dict, key, pool->max_stdalloc);
+                if (ret)
+                        return;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "pool%d.pool-misses", count);
+                ret = dict_set_uint64 (dict, key, pool->pool_misses);
+                if (ret)
+                        return;
                 count++;
         }
         ret = dict_set_int32 (dict, "mempool-count", count);