mempool: adjustments in pool sizes

* while creating 'rpc_clnt', the caller knows what would be the ideal
  load on it, so an extra argument to set some pool sizes

* while creating 'rpcsvc', the caller knows what would be the ideal
  load of it, so an extra argument to set request pool size

* cli memory footprint is reduced

Change-Id: Ie245216525b450e3373ef55b654b4cd30741347f
Signed-off-by: Amar Tumballi <amarts@redhat.com>
BUG: 765336
Reviewed-on: http://review.gluster.com/2784
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

4 files changed, 18 insertions, 13 deletions

diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index b2b20ea31..61d28bcac 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -23,7 +23,7 @@
 #include "config.h"
 #endif
 
-#define RPC_CLNT_DEFAULT_REQUEST_COUNT 4096
+#define RPC_CLNT_DEFAULT_REQUEST_COUNT 512
 
 #include "rpc-clnt.h"
 #include "byte-order.h"
@@ -1018,8 +1018,8 @@ out:
 }
 
 struct rpc_clnt *
-rpc_clnt_new (dict_t *options,
-              glusterfs_ctx_t *ctx, char *name)
+rpc_clnt_new (dict_t *options, glusterfs_ctx_t *ctx, char *name,
+              uint32_t reqpool_size)
 {
         int                    ret  = -1;
         struct rpc_clnt       *rpc  = NULL;
@@ -1032,8 +1032,10 @@ rpc_clnt_new (dict_t *options,
         pthread_mutex_init (&rpc->lock, NULL);
         rpc->ctx = ctx;
 
-        rpc->reqpool = mem_pool_new (struct rpc_req,
-                                     RPC_CLNT_DEFAULT_REQUEST_COUNT);
+        if (!reqpool_size)
+                reqpool_size = RPC_CLNT_DEFAULT_REQUEST_COUNT;
+
+        rpc->reqpool = mem_pool_new (struct rpc_req, reqpool_size);
         if (rpc->reqpool == NULL) {
                 pthread_mutex_destroy (&rpc->lock);
                 GF_FREE (rpc);
@@ -1042,7 +1044,7 @@ rpc_clnt_new (dict_t *options,
         }
 
         rpc->saved_frames_pool = mem_pool_new (struct saved_frame,
-                                              RPC_CLNT_DEFAULT_REQUEST_COUNT);
+                                               reqpool_size);
         if (rpc->saved_frames_pool == NULL) {
                 pthread_mutex_destroy (&rpc->lock);
                 mem_pool_destroy (rpc->reqpool);
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index 4fce08546..17e7e1271 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -192,7 +192,7 @@ typedef struct rpc_clnt {
 
 
 struct rpc_clnt *rpc_clnt_new (dict_t *options, glusterfs_ctx_t *ctx,
-                               char *name);
+                               char *name, uint32_t reqpool_size);
 
 int rpc_clnt_start (struct rpc_clnt *rpc);
 
diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index ca6a6ca4c..3592e02b9 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -1804,10 +1804,11 @@ out:
 /* The global RPC service initializer.
  */
 rpcsvc_t *
-rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options)
+rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options,
+             uint32_t poolcount)
 {
         rpcsvc_t          *svc              = NULL;
-        int                ret              = -1, poolcount = 0;
+        int                ret              = -1;
 
         if ((!ctx) || (!options))
                 return NULL;
@@ -1828,7 +1829,8 @@ rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options)
                 goto free_svc;
         }
 
-        poolcount   = RPCSVC_POOLCOUNT_MULT * svc->memfactor;
+        if (!poolcount)
+                poolcount = RPCSVC_POOLCOUNT_MULT * svc->memfactor;
 
         gf_log (GF_RPCSVC, GF_LOG_TRACE, "rx pool: %d", poolcount);
         svc->rxpool = mem_pool_new (rpcsvc_request_t, poolcount);
diff --git a/rpc/rpc-lib/src/rpcsvc.h b/rpc/rpc-lib/src/rpcsvc.h
index 5b297d8cf..6a6d23062 100644
--- a/rpc/rpc-lib/src/rpcsvc.h
+++ b/rpc/rpc-lib/src/rpcsvc.h
@@ -52,9 +52,9 @@
 
 #define RPCSVC_FRAGHDR_SIZE  4       /* 4-byte RPC fragment header size */
 #define RPCSVC_DEFAULT_LISTEN_PORT      GF_DEFAULT_BASE_PORT
-#define RPCSVC_DEFAULT_MEMFACTOR        15
+#define RPCSVC_DEFAULT_MEMFACTOR        8
 #define RPCSVC_EVENTPOOL_SIZE_MULT      1024
-#define RPCSVC_POOLCOUNT_MULT           35
+#define RPCSVC_POOLCOUNT_MULT           64
 #define RPCSVC_CONN_READ        (128 * GF_UNIT_KB)
 #define RPCSVC_PAGE_SIZE        (128 * GF_UNIT_KB)
 
@@ -420,7 +420,8 @@ rpcsvc_register_portmap_enabled (rpcsvc_t *svc);
  * Called in main.
  */
 extern rpcsvc_t *
-rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options);
+rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options,
+             uint32_t poolcount);
 
 int
 rpcsvc_register_notify (rpcsvc_t *svc, rpcsvc_notify_t notify, void *mydata);