diff options
author | Amar Tumballi <amarts@redhat.com> | 2018-10-16 16:31:49 +0530 |
---|---|---|
committer | Amar Tumballi <amarts@redhat.com> | 2018-12-14 17:34:28 +0000 |
commit | d49b41e817d592c1904b6f01716df6546dad3ebe (patch) | |
tree | 12546ad6da0534683cb7bed5b829500ecdfafe93 /libglusterfs/src/glusterfs | |
parent | fc74ef85e0780e0a265275df00e4d0d4a2d05eab (diff) |
fuse: add --lru-limit option
The inode LRU mechanism is moot in fuse xlator (ie. there is no
limit for the LRU list), as fuse inodes are referenced from
kernel context, and thus they can only be dropped on request of
the kernel. This might results in a high number of passive
inodes which are useless for the glusterfs client, causing a
significant memory overhead.
This change tries to remedy this by extending the LRU semantics
and allowing to set a finite limit on the fuse inode LRU.
A brief history of problem:
When gluster's inode table was designed, fuse didn't have any
'invalidate' method, which means, userspace application could
never ask kernel to send a 'forget()' fop, instead had to wait
for kernel to send it based on kernel's parameters. Inode table
remembers the number of times kernel has cached the inode based
on the 'nlookup' parameter. And 'nlookup' field is not used by
no other entry points (like server-protocol, gfapi etc).
Hence the inode_table of fuse module always has to have lru-limit
as '0', which means no limit. GlusterFS always had to keep all
inodes in memory as kernel would have had a reference to it.
Again, the reason for this is, kernel's glusterfs inode reference
was pointer of 'inode_t' structure in glusterfs. As it is a
pointer, we could never free it (to prevent segfault, or memory
corruption).
Solution:
In the inode table, handle the prune case of inodes with 'nlookup'
differently, and call a 'invalidator' method, which in this case is
fuse_invalidate(), and it sends the request to kernel for getting
the forget request.
When the kernel sends the forget, it means, it has dropped all
the reference to the inode, and it will send the forget with the
'nlookup' parameter too. We just need to make sure to reduce the
'nlookup' value we have when we get forget. That automatically
cause the relevant prune to happen.
Credits: Csaba Henk, Xavier Hernandez, Raghavendra Gowdappa, Nithya B
fixes: bz#1560969
Change-Id: Ifee0737b23b12b1426c224ec5b8f591f487d83a2
Signed-off-by: Amar Tumballi <amarts@redhat.com>
Diffstat (limited to 'libglusterfs/src/glusterfs')
-rw-r--r-- | libglusterfs/src/glusterfs/glusterfs.h | 1 | ||||
-rw-r--r-- | libglusterfs/src/glusterfs/inode.h | 17 |
2 files changed, 17 insertions, 1 deletions
diff --git a/libglusterfs/src/glusterfs/glusterfs.h b/libglusterfs/src/glusterfs/glusterfs.h index 908a0ce774f..9f14f2f5440 100644 --- a/libglusterfs/src/glusterfs/glusterfs.h +++ b/libglusterfs/src/glusterfs/glusterfs.h @@ -524,6 +524,7 @@ struct _cmd_args { pid_t client_pid; int client_pid_set; unsigned uid_map_root; + int32_t lru_limit; int background_qlen; int congestion_threshold; char *fuse_mountopts; diff --git a/libglusterfs/src/glusterfs/inode.h b/libglusterfs/src/glusterfs/inode.h index 5934373ec5b..52efdd85ccc 100644 --- a/libglusterfs/src/glusterfs/inode.h +++ b/libglusterfs/src/glusterfs/inode.h @@ -54,6 +54,13 @@ struct _inode_table { struct mem_pool *dentry_pool; /* memory pool for dentrys */ struct mem_pool *fd_mem_pool; /* memory pool for fd_t */ int ctxcount; /* number of slots in inode->ctx */ + + /* This is required for 'invalidation' when 'nlookup' would be used, + specially in case of fuse-bridge */ + int32_t (*invalidator_fn)(xlator_t *, inode_t *); + xlator_t *invalidator_xl; + struct list_head invalidate; /* inodes which are in invalidation queue */ + uint32_t invalidate_size; /* count of inodes in invalidation list */ }; struct _dentry { @@ -100,6 +107,7 @@ struct _inode { struct list_head list; /* active/lru/purge */ struct _inode_ctx *_ctx; /* replacement for dict_t *(inode->ctx) */ + bool invalidate_sent; /* Set it if invalidator_fn is called for inode */ }; #define UUID0_STR "00000000-0000-0000-0000-000000000000" @@ -107,7 +115,12 @@ struct _inode { #define GFID_STR_PFX_LEN (sizeof(GFID_STR_PFX) - 1) inode_table_t * -inode_table_new(size_t lru_limit, xlator_t *xl); +inode_table_new(uint32_t lru_limit, xlator_t *xl); + +inode_table_t * +inode_table_with_invalidator(uint32_t lru_limit, xlator_t *xl, + int32_t (*invalidator_fn)(xlator_t *, inode_t *), + xlator_t *invalidator_xl); void inode_table_destroy_all(glusterfs_ctx_t *ctx); @@ -139,6 +152,8 @@ inode_lookup(inode_t *inode); int inode_forget(inode_t *inode, uint64_t nlookup); +int +inode_forget_with_unref(inode_t *inode, uint64_t nlookup); int inode_ref_reduce_by_n(inode_t *inode, uint64_t nref); |