features/shard: Make lru limit of inode list configurable

Currently this lru limit is hard-coded to 16384. This patch makes it
configurable to make it easier to hit the lru limit and enable testing
of different cases that arise when the limit is reached.

The option is features.shard-lru-limit. It is by design allowed to
be configured only in init() but not in reconfigure(). This is to avoid
all the complexity associated with eviction of least recently used shards
when the list is shrunk.

Change-Id: Ifdcc2099f634314fafe8444e2d676e192e89e295
updates: bz#1605056
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>

2 files changed, 21 insertions, 4 deletions

diff --git a/xlators/features/shard/src/shard.c b/xlators/features/shard/src/shard.c
index bbec113781f..2e76720d176 100644
--- a/xlators/features/shard/src/shard.c
+++ b/xlators/features/shard/src/shard.c
@@ -668,7 +668,7 @@ __shard_update_shards_inode_list (inode_t *linked_inode, xlator_t *this,
         shard_inode_ctx_get (linked_inode, this, &ctx);
 
         if (list_empty (&ctx->ilist)) {
-                if (priv->inode_count + 1 <= SHARD_MAX_INODES) {
+                if (priv->inode_count + 1 <= priv->lru_limit) {
                 /* If this inode was linked here for the first time (indicated
                  * by empty list), and if there is still space in the priv list,
                  * add this ctx to the tail of the list.
@@ -6688,6 +6688,8 @@ init (xlator_t *this)
 
         GF_OPTION_INIT ("shard-deletion-rate", priv->deletion_rate, uint32, out);
 
+        GF_OPTION_INIT ("shard-lru-limit", priv->lru_limit, uint64, out);
+
         this->local_pool = mem_pool_new (shard_local_t, 128);
         if (!this->local_pool) {
                 ret = -1;
@@ -6807,7 +6809,7 @@ shard_priv_dump (xlator_t *this)
         gf_proc_dump_write ("shard-block-size", "%s", str);
         gf_proc_dump_write ("inode-count", "%d", priv->inode_count);
         gf_proc_dump_write ("ilist_head", "%p", &priv->ilist_head);
-        gf_proc_dump_write ("lru-max-limit", "%d", SHARD_MAX_INODES);
+        gf_proc_dump_write ("lru-max-limit", "%d", priv->lru_limit);
 
         GF_FREE (str);
 
@@ -6884,5 +6886,21 @@ struct volume_options options[] = {
            .max = INT_MAX,
            .description = "The number of shards to send deletes on at a time",
         },
+        {  .key = {"shard-lru-limit"},
+           .type = GF_OPTION_TYPE_INT,
+           .op_version = {GD_OP_VERSION_4_2_0},
+           .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT,
+           .tags = {"shard"},
+           .default_value = "16384",
+           .min = 20,
+           .max = INT_MAX,
+           .description = "The number of resolved shard inodes to keep in "
+                          "memory. A higher number means shards that are "
+                          "resolved will remain in memory longer, avoiding "
+                          "frequent lookups on them when they participate in "
+                          "file operations. The option also has a bearing on "
+                          "amount of memory consumed by these inodes and their "
+                          "internal metadata",
+        },
         { .key = {NULL} },
 };
diff --git a/xlators/features/shard/src/shard.h b/xlators/features/shard/src/shard.h
index 5de098a7a44..ac3813c8c50 100644
--- a/xlators/features/shard/src/shard.h
+++ b/xlators/features/shard/src/shard.h
@@ -23,8 +23,6 @@
 #define SHARD_MAX_BLOCK_SIZE  (4 * GF_UNIT_TB)
 #define SHARD_XATTR_PREFIX "trusted.glusterfs.shard."
 #define GF_XATTR_SHARD_BLOCK_SIZE "trusted.glusterfs.shard.block-size"
-#define SHARD_INODE_LRU_LIMIT 4096
-#define SHARD_MAX_INODES 16384
 /**
  *  Bit masks for the valid flag, which is used while updating ctx
 **/
@@ -216,6 +214,7 @@ typedef struct shard_priv {
         struct list_head ilist_head;
         uint32_t deletion_rate;
         shard_first_lookup_state_t first_lookup;
+        uint64_t lru_limit;
 } shard_priv_t;
 
 typedef struct {