mempool: adjustments in pool sizes

* while creating 'rpc_clnt', the caller knows what would be the ideal
  load on it, so an extra argument to set some pool sizes

* while creating 'rpcsvc', the caller knows what would be the ideal
  load of it, so an extra argument to set request pool size

* cli memory footprint is reduced

Change-Id: Ie245216525b450e3373ef55b654b4cd30741347f
Signed-off-by: Amar Tumballi <amarts@redhat.com>
BUG: 765336
Reviewed-on: http://review.gluster.com/2784
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

22 files changed, 27 insertions, 26 deletions

diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index b73400a7d..da0e56ffc 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -354,7 +354,7 @@ init (xlator_t *this)
         }
 
         /* keep more local here as we may need them for self-heal etc */
-        this->local_pool = mem_pool_new (afr_local_t, 4096);
+        this->local_pool = mem_pool_new (afr_local_t, 512);
         if (!this->local_pool) {
                 ret = -1;
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
index cec8066e2..e795c38e0 100644
--- a/xlators/cluster/afr/src/pump.c
+++ b/xlators/cluster/afr/src/pump.c
@@ -2526,7 +2526,7 @@ init (xlator_t *this)
         }
 
         /* keep more local here as we may need them for self-heal etc */
-        this->local_pool = mem_pool_new (afr_local_t, 4096);
+        this->local_pool = mem_pool_new (afr_local_t, 128);
         if (!this->local_pool) {
                 ret = -1;
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c
index eb55fd46c..4502a751b 100644
--- a/xlators/cluster/dht/src/dht.c
+++ b/xlators/cluster/dht/src/dht.c
@@ -454,7 +454,7 @@ init (xlator_t *this)
                 goto err;
         }
 
-        this->local_pool = mem_pool_new (dht_local_t, 1024);
+        this->local_pool = mem_pool_new (dht_local_t, 512);
         if (!this->local_pool) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to create local_t's memory pool");
diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c
index 63778afcf..951fe4e04 100644
--- a/xlators/cluster/dht/src/nufa.c
+++ b/xlators/cluster/dht/src/nufa.c
@@ -626,7 +626,7 @@ init (xlator_t *this)
                 goto err;
         }
 
-        this->local_pool = mem_pool_new (dht_local_t, 1024);
+        this->local_pool = mem_pool_new (dht_local_t, 128);
         if (!this->local_pool) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to create local_t's memory pool");
diff --git a/xlators/cluster/dht/src/switch.c b/xlators/cluster/dht/src/switch.c
index 4b5545ffe..1451e011e 100644
--- a/xlators/cluster/dht/src/switch.c
+++ b/xlators/cluster/dht/src/switch.c
@@ -933,7 +933,7 @@ init (xlator_t *this)
                 goto err;
         }
 
-        this->local_pool = mem_pool_new (dht_local_t, 1024);
+        this->local_pool = mem_pool_new (dht_local_t, 128);
         if (!this->local_pool) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to create local_t's memory pool");
diff --git a/xlators/cluster/stripe/src/stripe.c b/xlators/cluster/stripe/src/stripe.c
index c5ea891a7..62a28d71f 100644
--- a/xlators/cluster/stripe/src/stripe.c
+++ b/xlators/cluster/stripe/src/stripe.c
@@ -4433,7 +4433,7 @@ init (xlator_t *this)
         /* notify related */
         priv->nodes_down = priv->child_count;
 
-        this->local_pool = mem_pool_new (stripe_local_t, 1024);
+        this->local_pool = mem_pool_new (stripe_local_t, 128);
         if (!this->local_pool) {
                 ret = -1;
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c
index b3ea23dfc..a1a56ab20 100644
--- a/xlators/features/locks/src/posix.c
+++ b/xlators/features/locks/src/posix.c
@@ -2047,7 +2047,7 @@ init (xlator_t *this)
                 }
         }
 
-        this->local_pool = mem_pool_new (pl_local_t, 1024);
+        this->local_pool = mem_pool_new (pl_local_t, 32);
         if (!this->local_pool) {
                 ret = -1;
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
index 6e384f0b9..7f9610066 100644
--- a/xlators/features/marker/src/marker.c
+++ b/xlators/features/marker/src/marker.c
@@ -2504,7 +2504,7 @@ init (xlator_t *this)
                 }
         }
 
-        this->local_pool = mem_pool_new (marker_local_t, 1024);
+        this->local_pool = mem_pool_new (marker_local_t, 128);
         if (!this->local_pool) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to create local_t's memory pool");
diff --git a/xlators/features/quiesce/src/quiesce.h b/xlators/features/quiesce/src/quiesce.h
index 1adb70ebe..08f87e2ae 100644
--- a/xlators/features/quiesce/src/quiesce.h
+++ b/xlators/features/quiesce/src/quiesce.h
@@ -25,7 +25,7 @@
 #include "xlator.h"
 #include "timer.h"
 
-#define GF_FOPS_EXPECTED_IN_PARALLEL 4096
+#define GF_FOPS_EXPECTED_IN_PARALLEL 512
 
 typedef struct {
         gf_timer_t       *timer;
diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c
index 6b7db70cf..d819ea784 100644
--- a/xlators/features/quota/src/quota.c
+++ b/xlators/features/quota/src/quota.c
@@ -3000,7 +3000,7 @@ init (xlator_t *this)
 
         GF_OPTION_INIT ("timeout", priv->timeout, int64, err);
 
-        this->local_pool = mem_pool_new (quota_local_t, 1024);
+        this->local_pool = mem_pool_new (quota_local_t, 64);
         if (!this->local_pool) {
                 ret = -1;
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/features/trash/src/trash.c b/xlators/features/trash/src/trash.c
index 0aec4bc3d..15d9a429f 100644
--- a/xlators/features/trash/src/trash.c
+++ b/xlators/features/trash/src/trash.c
@@ -1518,7 +1518,7 @@ init (xlator_t *this)
                         _priv->max_trash_file_size);
         }
 
-        this->local_pool = mem_pool_new (trash_local_t, 1024);
+        this->local_pool = mem_pool_new (trash_local_t, 64);
         if (!this->local_pool) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to create local_t's memory pool");
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 79439535f..2e5825e85 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -2053,8 +2053,8 @@ glusterd_rpc_create (struct rpc_clnt **rpc,
 
         GF_ASSERT (options);
 
-        new_rpc = rpc_clnt_new (options, this->ctx, this->name);
-
+        /* TODO: is 32 enough? or more ? */
+        new_rpc = rpc_clnt_new (options, this->ctx, this->name, 16);
         if (!new_rpc)
                 goto out;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index ce044cf35..f5ec65976 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -860,7 +860,7 @@ init (xlator_t *this)
         ret = glusterd_rpcsvc_options_build (this->options);
         if (ret)
                 goto out;
-        rpc = rpcsvc_init (this, this->ctx, this->options);
+        rpc = rpcsvc_init (this, this->ctx, this->options, 64);
         if (rpc == NULL) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to init rpc");
diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c
index 67efe5307..ac6daafb7 100644
--- a/xlators/nfs/server/src/nfs.c
+++ b/xlators/nfs/server/src/nfs.c
@@ -692,7 +692,7 @@ nfs_init_state (xlator_t *this)
                 }
         }
 
-        nfs->rpcsvc =  rpcsvc_init (this, this->ctx, this->options);
+        nfs->rpcsvc =  rpcsvc_init (this, this->ctx, this->options, 0);
         if (!nfs->rpcsvc) {
                 ret = -1;
                 gf_log (GF_NFS, GF_LOG_ERROR, "RPC service init failed");
diff --git a/xlators/nfs/server/src/nlm4.c b/xlators/nfs/server/src/nlm4.c
index 5df6719a6..5145dc965 100644
--- a/xlators/nfs/server/src/nlm4.c
+++ b/xlators/nfs/server/src/nlm4.c
@@ -885,7 +885,8 @@ nlm4_establish_callback (void *csarg)
                 goto err;
         }
 
-        rpc_clnt = rpc_clnt_new (options, cs->nfsx->ctx, "NLM-client");
+        /* TODO: is 32 frames in transit enough ? */
+        rpc_clnt = rpc_clnt_new (options, cs->nfsx->ctx, "NLM-client", 32);
         if (rpc_clnt == NULL) {
                 gf_log (GF_NLM, GF_LOG_ERROR, "rpc_clnt NULL");
                 goto err;
diff --git a/xlators/performance/io-cache/src/io-cache.c b/xlators/performance/io-cache/src/io-cache.c
index d8fa5e991..bea22268b 100644
--- a/xlators/performance/io-cache/src/io-cache.c
+++ b/xlators/performance/io-cache/src/io-cache.c
@@ -1747,7 +1747,7 @@ init (xlator_t *this)
         for (index = 0; index < (table->max_pri); index++)
                 INIT_LIST_HEAD (&table->inode_lru[index]);
 
-        this->local_pool = mem_pool_new (ioc_local_t, 1024);
+        this->local_pool = mem_pool_new (ioc_local_t, 64);
         if (!this->local_pool) {
                 ret = -1;
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/performance/quick-read/src/quick-read.c b/xlators/performance/quick-read/src/quick-read.c
index b1b260f55..a64e63de4 100644
--- a/xlators/performance/quick-read/src/quick-read.c
+++ b/xlators/performance/quick-read/src/quick-read.c
@@ -3552,7 +3552,7 @@ init (xlator_t *this)
                 INIT_LIST_HEAD (&priv->table.lru[i]);
         }
 
-        this->local_pool = mem_pool_new (qr_local_t, 1024);
+        this->local_pool = mem_pool_new (qr_local_t, 64);
         if (!this->local_pool) {
                 ret = -1;
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/performance/read-ahead/src/read-ahead.c b/xlators/performance/read-ahead/src/read-ahead.c
index 72c7e6aa2..1a4dda97a 100644
--- a/xlators/performance/read-ahead/src/read-ahead.c
+++ b/xlators/performance/read-ahead/src/read-ahead.c
@@ -1062,7 +1062,7 @@ init (xlator_t *this)
 
         pthread_mutex_init (&conf->conf_lock, NULL);
 
-        this->local_pool = mem_pool_new (ra_local_t, 1024);
+        this->local_pool = mem_pool_new (ra_local_t, 64);
         if (!this->local_pool) {
                 ret = -1;
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/performance/write-behind/src/write-behind.c b/xlators/performance/write-behind/src/write-behind.c
index 7cae5405d..cc0569dd4 100644
--- a/xlators/performance/write-behind/src/write-behind.c
+++ b/xlators/performance/write-behind/src/write-behind.c
@@ -2955,7 +2955,7 @@ init (xlator_t *this)
         GF_OPTION_INIT ("enable-trickling-writes", conf->enable_trickling_writes,
                         bool, out);
 
-        this->local_pool = mem_pool_new (wb_local_t, 1024);
+        this->local_pool = mem_pool_new (wb_local_t, 64);
         if (!this->local_pool) {
                 ret = -1;
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 1386a2a1b..9a4945227 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -2194,7 +2194,7 @@ client_init_rpc (xlator_t *this)
                 goto out;
         }
 
-        conf->rpc = rpc_clnt_new (this->options, this->ctx, this->name);
+        conf->rpc = rpc_clnt_new (this->options, this->ctx, this->name, 0);
         if (!conf->rpc) {
                 gf_log (this->name, GF_LOG_ERROR, "failed to initialize RPC");
                 goto out;
@@ -2371,7 +2371,7 @@ init (xlator_t *this)
                 goto out;
         }
 
-        this->local_pool = mem_pool_new (clnt_local_t, 1024);
+        this->local_pool = mem_pool_new (clnt_local_t, 64);
         if (!this->local_pool) {
                 ret = -1;
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 9de1082dc..11824b27f 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -931,7 +931,7 @@ server_build_config (xlator_t *this, server_conf_t *conf)
         ret = dict_get_int32 (this->options, "inode-lru-limit",
                               &conf->inode_lru_limit);
         if (ret < 0) {
-                conf->inode_lru_limit = 1024;
+                conf->inode_lru_limit = 16384;
         }
 
         conf->verify_volfile = 1;
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index b45b77baa..0f9195f9e 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -904,8 +904,7 @@ init (xlator_t *this)
         }
 
         /* RPC related */
-        //conf->rpc = rpc_svc_init (&conf->rpc_conf);
-        conf->rpc = rpcsvc_init (this, this->ctx, this->options);
+        conf->rpc = rpcsvc_init (this, this->ctx, this->options, 0);
         if (conf->rpc == NULL) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "creation of rpcsvc failed");
@@ -1078,7 +1077,8 @@ struct volume_options options[] = {
         { .key   = {"inode-lru-limit"},
           .type  = GF_OPTION_TYPE_INT,
           .min   = 0,
-          .max   = (1 * GF_UNIT_MB)
+          .max   = (1 * GF_UNIT_MB),
+          .default_value = "16384",
         },
         { .key   = {"verify-volfile-checksum"},
           .type  = GF_OPTION_TYPE_BOOL