mount/fuse: expose auto-invalidation as a mount option

Auto invalidation is necessary when same (meta)data is shared/access
across multiple mounts. However, if (meta)data is not shared, all
relevant I/O goes through the cache of single mount and hence is
coherent with (meta)data on bricks always. So, fuse-auto-invalidation
can be disabled for this case which gives a huge performance boost for
workloads that write data and then immediately read the data they just
wrote.

From glusterfs --help,

<snip>
      --auto-invalidation[=BOOL]   controls whether fuse-kernel can
                             auto-invalidate attribute, dentry and page-cache.
                             Disable this only if same files/directories are
                             not accessed across two different mounts
                             concurrently [default: "on"]
</snip>

Details on how disabling auto-invalidation helped to reduce pgbench
init times can be found at [1]. Time taken for pgbench init of scale
8000 was 8340s. That will be an improvement of 86% (59280s vs 8340s)
with auto-invalidations turned off along with other
optimizations. Just disabling auto-invalidation contributed 56%
improvement by reducing the total time taken by 33260s.

[1] https://www.spinics.net/lists/gluster-devel/msg25907.html

Change-Id: I0ed730dba9064bd9c576ad1800170a21e100e1ce
Signed-off-by: Raghavendra Gowdappa <rgowdapp@redhat.com>
updates: bz#1664934

3 files changed, 33 insertions, 10 deletions

diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index ff5831234b1..c3945d7a13c 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -5067,6 +5067,9 @@ fuse_init(xlator_t *this, fuse_in_header_t *finh, void *msg,
         /* If user did not explicitly set --fopen-keep-cache[=off],
            then check if kernel support FUSE_AUTO_INVAL_DATA and ...
         */
+
+        priv->fopen_keep_cache = 1;
+
 #if FUSE_KERNEL_MINOR_VERSION >= 20
         if (fini->flags & FUSE_AUTO_INVAL_DATA) {
             /* ... enable fopen_keep_cache mode if supported.
@@ -5075,25 +5078,26 @@ fuse_init(xlator_t *this, fuse_in_header_t *finh, void *msg,
                    "Detected "
                    "support for FUSE_AUTO_INVAL_DATA. Enabling "
                    "fopen_keep_cache automatically.");
-            fino.flags |= FUSE_AUTO_INVAL_DATA;
-            priv->fopen_keep_cache = 1;
+
+            if (priv->fuse_auto_inval)
+                fino.flags |= FUSE_AUTO_INVAL_DATA;
         } else
 #endif
         {
-
-            gf_log("glusterfs-fuse", GF_LOG_DEBUG,
-                   "No support "
-                   "for FUSE_AUTO_INVAL_DATA. Disabling "
-                   "fopen_keep_cache.");
-            /* ... else disable. */
-            priv->fopen_keep_cache = 0;
+            if (priv->fuse_auto_inval) {
+                gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+                       "No support for FUSE_AUTO_INVAL_DATA. Disabling "
+                       "fopen_keep_cache.");
+                /* ... else disable. */
+                priv->fopen_keep_cache = 0;
+            }
         }
     } else if (priv->fopen_keep_cache == 1) {
         /* If user explicitly set --fopen-keep-cache[=on],
            then enable FUSE_AUTO_INVAL_DATA if possible.
         */
 #if FUSE_KERNEL_MINOR_VERSION >= 20
-        if (fini->flags & FUSE_AUTO_INVAL_DATA) {
+        if (priv->fuse_auto_inval && (fini->flags & FUSE_AUTO_INVAL_DATA)) {
             gf_log("glusterfs-fuse", GF_LOG_DEBUG,
                    "fopen_keep_cache "
                    "is explicitly set. Enabling FUSE_AUTO_INVAL_DATA");
@@ -6507,6 +6511,8 @@ init(xlator_t *this_xl)
     GF_OPTION_INIT("reader-thread-count", priv->reader_thread_count, uint32,
                    cleanup_exit);
 
+    GF_OPTION_INIT("auto-invalidation", priv->fuse_auto_inval, bool,
+                   cleanup_exit);
     GF_OPTION_INIT(ZR_ENTRY_TIMEOUT_OPT, priv->entry_timeout, double,
                    cleanup_exit);
 
@@ -6944,6 +6950,15 @@ struct volume_options options[] = {
         .description = "makes glusterfs invalidate kernel inodes after "
                        "reaching this limit (0 means 'unlimited')",
     },
+    {
+        .key = {"auto-invalidation"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "true",
+        .description = "controls whether fuse-kernel can auto-invalidate "
+                       "attribute, dentry and page-cache. Disable this only "
+                       "if same files/directories are not accessed across "
+                       "two different mounts concurrently",
+    },
     {.key = {NULL}},
 };
 
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index b892113eb79..697bd8848e1 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -187,6 +187,7 @@ struct fuse_private {
     pthread_mutex_t interrupt_mutex;
 
     gf_boolean_t flush_handle_interrupt;
+    gf_boolean_t fuse_auto_inval;
 
     /* LRU Limit, if not set, default is 128k for now */
     uint32_t lru_limit;
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index d09a7cd663e..868a57f7ede 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -229,6 +229,10 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --reader-thread-count=$reader_thread_count");
     fi
 
+    if [ -n "$auto-invalidation" ]; then
+        cmd_line=$(echo "$cmd_line --auto-invalidation=$fuse_auto_invalidation");
+    fi
+
     if [ -n "$volume_name" ]; then
         cmd_line=$(echo "$cmd_line --volume-name=$volume_name");
     fi
@@ -529,6 +533,9 @@ with_options()
         "reader-thread-count")
             reader_thread_count=$value
             ;;
+        "auto-invalidation")
+            fuse_auto_invalidation=$value
+	    ;;
         "no-root-squash")
             if [ $value = "yes" ] ||
                 [ $value = "on" ] ||