diff options
31 files changed, 61 insertions, 56 deletions
diff --git a/cli/src/cli.c b/cli/src/cli.c index 70204cabe..46a445cf7 100644 --- a/cli/src/cli.c +++ b/cli/src/cli.c @@ -164,19 +164,18 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx) if (!pool) return -1; - /* frame_mem_pool size 112 * 16k */ - pool->frame_mem_pool = mem_pool_new (call_frame_t, 16384); - + /* frame_mem_pool size 112 * 64 */ + pool->frame_mem_pool = mem_pool_new (call_frame_t, 32); if (!pool->frame_mem_pool) return -1; - /* stack_mem_pool size 256 * 8k */ - pool->stack_mem_pool = mem_pool_new (call_stack_t, 8192); + /* stack_mem_pool size 256 * 128 */ + pool->stack_mem_pool = mem_pool_new (call_stack_t, 16); if (!pool->stack_mem_pool) return -1; - ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024); + ctx->stub_mem_pool = mem_pool_new (call_stub_t, 16); if (!ctx->stub_mem_pool) return -1; @@ -495,7 +494,7 @@ cli_rpc_init (struct cli_state *state) if (ret) goto out; - rpc = rpc_clnt_new (options, this->ctx, this->name); + rpc = rpc_clnt_new (options, this->ctx, this->name, 16); if (!rpc) goto out; diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c index e41653faf..4220baf98 100644 --- a/glusterfsd/src/glusterfsd-mgmt.c +++ b/glusterfsd/src/glusterfsd-mgmt.c @@ -1475,7 +1475,7 @@ glusterfs_listener_init (glusterfs_ctx_t *ctx) if (ret) goto out; - rpc = rpcsvc_init (THIS, ctx, options); + rpc = rpcsvc_init (THIS, ctx, options, 8); if (rpc == NULL) { goto out; } @@ -1528,7 +1528,7 @@ glusterfs_mgmt_init (glusterfs_ctx_t *ctx) if (ret) goto out; - rpc = rpc_clnt_new (options, THIS->ctx, THIS->name); + rpc = rpc_clnt_new (options, THIS->ctx, THIS->name, 8); if (!rpc) { ret = -1; gf_log (THIS->name, GF_LOG_WARNING, "failed to create rpc clnt"); diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c index c233a551d..573aca561 100644 --- a/glusterfsd/src/glusterfsd.c +++ b/glusterfsd/src/glusterfsd.c @@ -1009,17 +1009,15 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx) return -1; } - /* frame_mem_pool size 112 * 16k */ - pool->frame_mem_pool = mem_pool_new (call_frame_t, 16384); - + /* frame_mem_pool size 112 * 4k */ + pool->frame_mem_pool = mem_pool_new (call_frame_t, 4096); if (!pool->frame_mem_pool) { gf_log ("", GF_LOG_CRITICAL, "ERROR: glusterfs frame pool creation failed"); return -1; } - /* stack_mem_pool size 256 * 8k */ - pool->stack_mem_pool = mem_pool_new (call_stack_t, 8192); - + /* stack_mem_pool size 256 * 1024 */ + pool->stack_mem_pool = mem_pool_new (call_stack_t, 1024); if (!pool->stack_mem_pool) { gf_log ("", GF_LOG_CRITICAL, "ERROR: glusterfs stack pool creation failed"); diff --git a/libglusterfs/src/inode.c b/libglusterfs/src/inode.c index 4685ec8d2..67752da71 100644 --- a/libglusterfs/src/inode.c +++ b/libglusterfs/src/inode.c @@ -1254,7 +1254,9 @@ inode_table_new (size_t lru_limit, xlator_t *xl) if (!new->name_hash) goto out; - new->fd_mem_pool = mem_pool_new (fd_t, 16384); + /* if number of fd open in one process is more than this, + we may hit perf issues */ + new->fd_mem_pool = mem_pool_new (fd_t, 1024); if (!new->fd_mem_pool) goto out; diff --git a/libglusterfs/src/inode.h b/libglusterfs/src/inode.h index 7dda0401d..5dced8006 100644 --- a/libglusterfs/src/inode.h +++ b/libglusterfs/src/inode.h @@ -28,7 +28,7 @@ #include <stdint.h> #include <sys/types.h> -#define DEFAULT_INODE_MEMPOOL_ENTRIES 16384 +#define DEFAULT_INODE_MEMPOOL_ENTRIES 32 * 1024 struct _inode_table; typedef struct _inode_table inode_table_t; diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c index b2b20ea31..61d28bcac 100644 --- a/rpc/rpc-lib/src/rpc-clnt.c +++ b/rpc/rpc-lib/src/rpc-clnt.c @@ -23,7 +23,7 @@ #include "config.h" #endif -#define RPC_CLNT_DEFAULT_REQUEST_COUNT 4096 +#define RPC_CLNT_DEFAULT_REQUEST_COUNT 512 #include "rpc-clnt.h" #include "byte-order.h" @@ -1018,8 +1018,8 @@ out: } struct rpc_clnt * -rpc_clnt_new (dict_t *options, - glusterfs_ctx_t *ctx, char *name) +rpc_clnt_new (dict_t *options, glusterfs_ctx_t *ctx, char *name, + uint32_t reqpool_size) { int ret = -1; struct rpc_clnt *rpc = NULL; @@ -1032,8 +1032,10 @@ rpc_clnt_new (dict_t *options, pthread_mutex_init (&rpc->lock, NULL); rpc->ctx = ctx; - rpc->reqpool = mem_pool_new (struct rpc_req, - RPC_CLNT_DEFAULT_REQUEST_COUNT); + if (!reqpool_size) + reqpool_size = RPC_CLNT_DEFAULT_REQUEST_COUNT; + + rpc->reqpool = mem_pool_new (struct rpc_req, reqpool_size); if (rpc->reqpool == NULL) { pthread_mutex_destroy (&rpc->lock); GF_FREE (rpc); @@ -1042,7 +1044,7 @@ rpc_clnt_new (dict_t *options, } rpc->saved_frames_pool = mem_pool_new (struct saved_frame, - RPC_CLNT_DEFAULT_REQUEST_COUNT); + reqpool_size); if (rpc->saved_frames_pool == NULL) { pthread_mutex_destroy (&rpc->lock); mem_pool_destroy (rpc->reqpool); diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h index 4fce08546..17e7e1271 100644 --- a/rpc/rpc-lib/src/rpc-clnt.h +++ b/rpc/rpc-lib/src/rpc-clnt.h @@ -192,7 +192,7 @@ typedef struct rpc_clnt { struct rpc_clnt *rpc_clnt_new (dict_t *options, glusterfs_ctx_t *ctx, - char *name); + char *name, uint32_t reqpool_size); int rpc_clnt_start (struct rpc_clnt *rpc); diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c index ca6a6ca4c..3592e02b9 100644 --- a/rpc/rpc-lib/src/rpcsvc.c +++ b/rpc/rpc-lib/src/rpcsvc.c @@ -1804,10 +1804,11 @@ out: /* The global RPC service initializer. */ rpcsvc_t * -rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options) +rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options, + uint32_t poolcount) { rpcsvc_t *svc = NULL; - int ret = -1, poolcount = 0; + int ret = -1; if ((!ctx) || (!options)) return NULL; @@ -1828,7 +1829,8 @@ rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options) goto free_svc; } - poolcount = RPCSVC_POOLCOUNT_MULT * svc->memfactor; + if (!poolcount) + poolcount = RPCSVC_POOLCOUNT_MULT * svc->memfactor; gf_log (GF_RPCSVC, GF_LOG_TRACE, "rx pool: %d", poolcount); svc->rxpool = mem_pool_new (rpcsvc_request_t, poolcount); diff --git a/rpc/rpc-lib/src/rpcsvc.h b/rpc/rpc-lib/src/rpcsvc.h index 5b297d8cf..6a6d23062 100644 --- a/rpc/rpc-lib/src/rpcsvc.h +++ b/rpc/rpc-lib/src/rpcsvc.h @@ -52,9 +52,9 @@ #define RPCSVC_FRAGHDR_SIZE 4 /* 4-byte RPC fragment header size */ #define RPCSVC_DEFAULT_LISTEN_PORT GF_DEFAULT_BASE_PORT -#define RPCSVC_DEFAULT_MEMFACTOR 15 +#define RPCSVC_DEFAULT_MEMFACTOR 8 #define RPCSVC_EVENTPOOL_SIZE_MULT 1024 -#define RPCSVC_POOLCOUNT_MULT 35 +#define RPCSVC_POOLCOUNT_MULT 64 #define RPCSVC_CONN_READ (128 * GF_UNIT_KB) #define RPCSVC_PAGE_SIZE (128 * GF_UNIT_KB) @@ -420,7 +420,8 @@ rpcsvc_register_portmap_enabled (rpcsvc_t *svc); * Called in main. */ extern rpcsvc_t * -rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options); +rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options, + uint32_t poolcount); int rpcsvc_register_notify (rpcsvc_t *svc, rpcsvc_notify_t notify, void *mydata); diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c index b73400a7d..da0e56ffc 100644 --- a/xlators/cluster/afr/src/afr.c +++ b/xlators/cluster/afr/src/afr.c @@ -354,7 +354,7 @@ init (xlator_t *this) } /* keep more local here as we may need them for self-heal etc */ - this->local_pool = mem_pool_new (afr_local_t, 4096); + this->local_pool = mem_pool_new (afr_local_t, 512); if (!this->local_pool) { ret = -1; gf_log (this->name, GF_LOG_ERROR, diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c index cec8066e2..e795c38e0 100644 --- a/xlators/cluster/afr/src/pump.c +++ b/xlators/cluster/afr/src/pump.c @@ -2526,7 +2526,7 @@ init (xlator_t *this) } /* keep more local here as we may need them for self-heal etc */ - this->local_pool = mem_pool_new (afr_local_t, 4096); + this->local_pool = mem_pool_new (afr_local_t, 128); if (!this->local_pool) { ret = -1; gf_log (this->name, GF_LOG_ERROR, diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c index eb55fd46c..4502a751b 100644 --- a/xlators/cluster/dht/src/dht.c +++ b/xlators/cluster/dht/src/dht.c @@ -454,7 +454,7 @@ init (xlator_t *this) goto err; } - this->local_pool = mem_pool_new (dht_local_t, 1024); + this->local_pool = mem_pool_new (dht_local_t, 512); if (!this->local_pool) { gf_log (this->name, GF_LOG_ERROR, "failed to create local_t's memory pool"); diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c index 63778afcf..951fe4e04 100644 --- a/xlators/cluster/dht/src/nufa.c +++ b/xlators/cluster/dht/src/nufa.c @@ -626,7 +626,7 @@ init (xlator_t *this) goto err; } - this->local_pool = mem_pool_new (dht_local_t, 1024); + this->local_pool = mem_pool_new (dht_local_t, 128); if (!this->local_pool) { gf_log (this->name, GF_LOG_ERROR, "failed to create local_t's memory pool"); diff --git a/xlators/cluster/dht/src/switch.c b/xlators/cluster/dht/src/switch.c index 4b5545ffe..1451e011e 100644 --- a/xlators/cluster/dht/src/switch.c +++ b/xlators/cluster/dht/src/switch.c @@ -933,7 +933,7 @@ init (xlator_t *this) goto err; } - this->local_pool = mem_pool_new (dht_local_t, 1024); + this->local_pool = mem_pool_new (dht_local_t, 128); if (!this->local_pool) { gf_log (this->name, GF_LOG_ERROR, "failed to create local_t's memory pool"); diff --git a/xlators/cluster/stripe/src/stripe.c b/xlators/cluster/stripe/src/stripe.c index c5ea891a7..62a28d71f 100644 --- a/xlators/cluster/stripe/src/stripe.c +++ b/xlators/cluster/stripe/src/stripe.c @@ -4433,7 +4433,7 @@ init (xlator_t *this) /* notify related */ priv->nodes_down = priv->child_count; - this->local_pool = mem_pool_new (stripe_local_t, 1024); + this->local_pool = mem_pool_new (stripe_local_t, 128); if (!this->local_pool) { ret = -1; gf_log (this->name, GF_LOG_ERROR, diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c index b3ea23dfc..a1a56ab20 100644 --- a/xlators/features/locks/src/posix.c +++ b/xlators/features/locks/src/posix.c @@ -2047,7 +2047,7 @@ init (xlator_t *this) } } - this->local_pool = mem_pool_new (pl_local_t, 1024); + this->local_pool = mem_pool_new (pl_local_t, 32); if (!this->local_pool) { ret = -1; gf_log (this->name, GF_LOG_ERROR, diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c index 6e384f0b9..7f9610066 100644 --- a/xlators/features/marker/src/marker.c +++ b/xlators/features/marker/src/marker.c @@ -2504,7 +2504,7 @@ init (xlator_t *this) } } - this->local_pool = mem_pool_new (marker_local_t, 1024); + this->local_pool = mem_pool_new (marker_local_t, 128); if (!this->local_pool) { gf_log (this->name, GF_LOG_ERROR, "failed to create local_t's memory pool"); diff --git a/xlators/features/quiesce/src/quiesce.h b/xlators/features/quiesce/src/quiesce.h index 1adb70ebe..08f87e2ae 100644 --- a/xlators/features/quiesce/src/quiesce.h +++ b/xlators/features/quiesce/src/quiesce.h @@ -25,7 +25,7 @@ #include "xlator.h" #include "timer.h" -#define GF_FOPS_EXPECTED_IN_PARALLEL 4096 +#define GF_FOPS_EXPECTED_IN_PARALLEL 512 typedef struct { gf_timer_t *timer; diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c index 6b7db70cf..d819ea784 100644 --- a/xlators/features/quota/src/quota.c +++ b/xlators/features/quota/src/quota.c @@ -3000,7 +3000,7 @@ init (xlator_t *this) GF_OPTION_INIT ("timeout", priv->timeout, int64, err); - this->local_pool = mem_pool_new (quota_local_t, 1024); + this->local_pool = mem_pool_new (quota_local_t, 64); if (!this->local_pool) { ret = -1; gf_log (this->name, GF_LOG_ERROR, diff --git a/xlators/features/trash/src/trash.c b/xlators/features/trash/src/trash.c index 0aec4bc3d..15d9a429f 100644 --- a/xlators/features/trash/src/trash.c +++ b/xlators/features/trash/src/trash.c @@ -1518,7 +1518,7 @@ init (xlator_t *this) _priv->max_trash_file_size); } - this->local_pool = mem_pool_new (trash_local_t, 1024); + this->local_pool = mem_pool_new (trash_local_t, 64); if (!this->local_pool) { gf_log (this->name, GF_LOG_ERROR, "failed to create local_t's memory pool"); diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c index 79439535f..2e5825e85 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-handler.c @@ -2053,8 +2053,8 @@ glusterd_rpc_create (struct rpc_clnt **rpc, GF_ASSERT (options); - new_rpc = rpc_clnt_new (options, this->ctx, this->name); - + /* TODO: is 32 enough? or more ? */ + new_rpc = rpc_clnt_new (options, this->ctx, this->name, 16); if (!new_rpc) goto out; diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c index ce044cf35..f5ec65976 100644 --- a/xlators/mgmt/glusterd/src/glusterd.c +++ b/xlators/mgmt/glusterd/src/glusterd.c @@ -860,7 +860,7 @@ init (xlator_t *this) ret = glusterd_rpcsvc_options_build (this->options); if (ret) goto out; - rpc = rpcsvc_init (this, this->ctx, this->options); + rpc = rpcsvc_init (this, this->ctx, this->options, 64); if (rpc == NULL) { gf_log (this->name, GF_LOG_ERROR, "failed to init rpc"); diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c index 67efe5307..ac6daafb7 100644 --- a/xlators/nfs/server/src/nfs.c +++ b/xlators/nfs/server/src/nfs.c @@ -692,7 +692,7 @@ nfs_init_state (xlator_t *this) } } - nfs->rpcsvc = rpcsvc_init (this, this->ctx, this->options); + nfs->rpcsvc = rpcsvc_init (this, this->ctx, this->options, 0); if (!nfs->rpcsvc) { ret = -1; gf_log (GF_NFS, GF_LOG_ERROR, "RPC service init failed"); diff --git a/xlators/nfs/server/src/nlm4.c b/xlators/nfs/server/src/nlm4.c index 5df6719a6..5145dc965 100644 --- a/xlators/nfs/server/src/nlm4.c +++ b/xlators/nfs/server/src/nlm4.c @@ -885,7 +885,8 @@ nlm4_establish_callback (void *csarg) goto err; } - rpc_clnt = rpc_clnt_new (options, cs->nfsx->ctx, "NLM-client"); + /* TODO: is 32 frames in transit enough ? */ + rpc_clnt = rpc_clnt_new (options, cs->nfsx->ctx, "NLM-client", 32); if (rpc_clnt == NULL) { gf_log (GF_NLM, GF_LOG_ERROR, "rpc_clnt NULL"); goto err; diff --git a/xlators/performance/io-cache/src/io-cache.c b/xlators/performance/io-cache/src/io-cache.c index d8fa5e991..bea22268b 100644 --- a/xlators/performance/io-cache/src/io-cache.c +++ b/xlators/performance/io-cache/src/io-cache.c @@ -1747,7 +1747,7 @@ init (xlator_t *this) for (index = 0; index < (table->max_pri); index++) INIT_LIST_HEAD (&table->inode_lru[index]); - this->local_pool = mem_pool_new (ioc_local_t, 1024); + this->local_pool = mem_pool_new (ioc_local_t, 64); if (!this->local_pool) { ret = -1; gf_log (this->name, GF_LOG_ERROR, diff --git a/xlators/performance/quick-read/src/quick-read.c b/xlators/performance/quick-read/src/quick-read.c index b1b260f55..a64e63de4 100644 --- a/xlators/performance/quick-read/src/quick-read.c +++ b/xlators/performance/quick-read/src/quick-read.c @@ -3552,7 +3552,7 @@ init (xlator_t *this) INIT_LIST_HEAD (&priv->table.lru[i]); } - this->local_pool = mem_pool_new (qr_local_t, 1024); + this->local_pool = mem_pool_new (qr_local_t, 64); if (!this->local_pool) { ret = -1; gf_log (this->name, GF_LOG_ERROR, diff --git a/xlators/performance/read-ahead/src/read-ahead.c b/xlators/performance/read-ahead/src/read-ahead.c index 72c7e6aa2..1a4dda97a 100644 --- a/xlators/performance/read-ahead/src/read-ahead.c +++ b/xlators/performance/read-ahead/src/read-ahead.c @@ -1062,7 +1062,7 @@ init (xlator_t *this) pthread_mutex_init (&conf->conf_lock, NULL); - this->local_pool = mem_pool_new (ra_local_t, 1024); + this->local_pool = mem_pool_new (ra_local_t, 64); if (!this->local_pool) { ret = -1; gf_log (this->name, GF_LOG_ERROR, diff --git a/xlators/performance/write-behind/src/write-behind.c b/xlators/performance/write-behind/src/write-behind.c index 7cae5405d..cc0569dd4 100644 --- a/xlators/performance/write-behind/src/write-behind.c +++ b/xlators/performance/write-behind/src/write-behind.c @@ -2955,7 +2955,7 @@ init (xlator_t *this) GF_OPTION_INIT ("enable-trickling-writes", conf->enable_trickling_writes, bool, out); - this->local_pool = mem_pool_new (wb_local_t, 1024); + this->local_pool = mem_pool_new (wb_local_t, 64); if (!this->local_pool) { ret = -1; gf_log (this->name, GF_LOG_ERROR, diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c index 1386a2a1b..9a4945227 100644 --- a/xlators/protocol/client/src/client.c +++ b/xlators/protocol/client/src/client.c @@ -2194,7 +2194,7 @@ client_init_rpc (xlator_t *this) goto out; } - conf->rpc = rpc_clnt_new (this->options, this->ctx, this->name); + conf->rpc = rpc_clnt_new (this->options, this->ctx, this->name, 0); if (!conf->rpc) { gf_log (this->name, GF_LOG_ERROR, "failed to initialize RPC"); goto out; @@ -2371,7 +2371,7 @@ init (xlator_t *this) goto out; } - this->local_pool = mem_pool_new (clnt_local_t, 1024); + this->local_pool = mem_pool_new (clnt_local_t, 64); if (!this->local_pool) { ret = -1; gf_log (this->name, GF_LOG_ERROR, diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c index 9de1082dc..11824b27f 100644 --- a/xlators/protocol/server/src/server-helpers.c +++ b/xlators/protocol/server/src/server-helpers.c @@ -931,7 +931,7 @@ server_build_config (xlator_t *this, server_conf_t *conf) ret = dict_get_int32 (this->options, "inode-lru-limit", &conf->inode_lru_limit); if (ret < 0) { - conf->inode_lru_limit = 1024; + conf->inode_lru_limit = 16384; } conf->verify_volfile = 1; diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c index b45b77baa..0f9195f9e 100644 --- a/xlators/protocol/server/src/server.c +++ b/xlators/protocol/server/src/server.c @@ -904,8 +904,7 @@ init (xlator_t *this) } /* RPC related */ - //conf->rpc = rpc_svc_init (&conf->rpc_conf); - conf->rpc = rpcsvc_init (this, this->ctx, this->options); + conf->rpc = rpcsvc_init (this, this->ctx, this->options, 0); if (conf->rpc == NULL) { gf_log (this->name, GF_LOG_WARNING, "creation of rpcsvc failed"); @@ -1078,7 +1077,8 @@ struct volume_options options[] = { { .key = {"inode-lru-limit"}, .type = GF_OPTION_TYPE_INT, .min = 0, - .max = (1 * GF_UNIT_MB) + .max = (1 * GF_UNIT_MB), + .default_value = "16384", }, { .key = {"verify-volfile-checksum"}, .type = GF_OPTION_TYPE_BOOL |