gfapi: do not cache upcalls if the application is not interested

When the volume option 'features.cache-invalidation' is enabled, upcall
events are sent from the brick process to the client. Even if the client
is not interested in upcall events itself, md-cache or other xlators may
benefit from them.

By adding a new 'cache_upcalls' boolean in the 'struct glfs', we can
enable the caching of upcalls when the application called
glfs_h_poll_upcall(). NFS-Ganesha sets up a thread for handling upcalls
in the initialization phase, and calls glfs_h_poll_upcall() before any
NFS-client accesses the NFS-export.

In the future there will be a more flexible registration API for
enabling certain kind of upcall events. Until that is available, this
should work just fine.

Verificatio of this change is not trivial within our current regression
test framework. The bug report contains a description on how to reliably
reproduce the problem with the glusterfs-coreutils.

Change-Id: I818595c92db50e6e48f7bfe287ee05103a4a30a2
BUG: 1368842
Signed-off-by: Niels de Vos <ndevos@redhat.com>
Reviewed-on: http://review.gluster.org/15191
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Poornima G <pgurusid@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: soumya k <skoduri@redhat.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>

5 files changed, 21 insertions, 5 deletions

diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index 18bac8195d0..9c31502d725 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -4325,7 +4325,8 @@ priv_glfs_process_upcall_event (struct glfs *fs, void *data)
         {
                 ctx = fs->ctx;
 
-                if (ctx->cleanup_started) {
+                /* if we're not interested in upcalls (anymore), skip them */
+                if (ctx->cleanup_started || !fs->cache_upcalls) {
                         pthread_mutex_unlock (&fs->mutex);
                         goto out;
                 }
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
index 7623b29f0cb..47bdbcbec52 100644
--- a/api/src/glfs-handleops.c
+++ b/api/src/glfs-handleops.c
@@ -2026,6 +2026,10 @@ pub_glfs_h_poll_upcall (struct glfs *fs, struct glfs_callback_arg *up_arg)
                 }
 
                 fs->pin_refcnt++;
+
+                /* once we call this function, the applications seems to be
+                 * interested in events, enable caching them */
+                fs->cache_upcalls = _gf_true;
         }
         pthread_mutex_unlock (&fs->mutex);
 
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index dc4c3f7c18e..471fa5fffc1 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -210,6 +210,7 @@ struct glfs {
 
 	gf_boolean_t        migration_in_progress;
 
+        gf_boolean_t        cache_upcalls; /* add upcalls to the upcall_list? */
         struct list_head    upcall_list;
         pthread_mutex_t     upcall_list_mutex; /* mutex for upcall entry list */
 
diff --git a/tests/basic/gfapi/bug1291259.c b/tests/basic/gfapi/bug1291259.c
index 98e3b13c63d..9fbe02ba34d 100644
--- a/tests/basic/gfapi/bug1291259.c
+++ b/tests/basic/gfapi/bug1291259.c
@@ -51,8 +51,6 @@ main (int argc, char *argv[])
         unsigned char   globjhdl[GFAPI_HANDLE_LENGTH];
         unsigned char   globjhdl2[GFAPI_HANDLE_LENGTH];
 
-        cbk.reason = 0;
-
         fprintf (stderr, "Starting libgfapi_fini\n");
         if (argc != 4) {
                 fprintf (stderr, "Invalid argument\n");
@@ -79,6 +77,13 @@ main (int argc, char *argv[])
         ret = glfs_init (fs);
         LOG_ERR("glfs_init", ret);
 
+        /* This does not block, but enables caching of events. Real
+         * applications like NFS-Ganesha run this in a thread before activity
+         * on the fs (through this instance) happens. */
+        ret = glfs_h_poll_upcall(fs, &cbk);
+        LOG_ERR ("glfs_h_poll_upcall", ret);
+        cbk.reason = 0;
+
         fs2 = glfs_new (volname);
         if (!fs) {
                 fprintf (stderr, "glfs_new: returned NULL\n");
diff --git a/tests/basic/gfapi/upcall-cache-invalidate.c b/tests/basic/gfapi/upcall-cache-invalidate.c
index ea1b5c4a88b..44e186955e7 100644
--- a/tests/basic/gfapi/upcall-cache-invalidate.c
+++ b/tests/basic/gfapi/upcall-cache-invalidate.c
@@ -43,8 +43,6 @@ main (int argc, char *argv[])
         char      *hostname = NULL;
         struct glfs_callback_inode_arg *in_arg = NULL;
 
-        cbk.reason = 0;
-
         if (argc != 4) {
                 fprintf (stderr, "Invalid argument\n");
                 exit(1);
@@ -69,6 +67,13 @@ main (int argc, char *argv[])
         ret = glfs_init (fs);
         LOG_ERR("glfs_init", ret);
 
+        /* This does not block, but enables caching of events. Real
+         * applications like NFS-Ganesha run this in a thread before activity
+         * on the fs (through this instance) happens. */
+        ret = glfs_h_poll_upcall(fs_tmp, &cbk);
+        LOG_ERR ("glfs_h_poll_upcall", ret);
+        cbk.reason = 0;
+
         fs2 = glfs_new (volname);
         if (!fs2) {
                 fprintf (stderr, "glfs_new fs2: returned NULL\n");