gfapi: APIs to store and process upcall notifications received

In case of any upcall cbk events received by the protocol/client,
gfapi will be notified which queues them up in a list (<gfapi_cbk_upcall>).

Applicatons are responsible to provide APIs to process & notify them in case
of any such upcall events queued.

Added a new API which will be used by Ganesha to repeatedly poll for any
such upcall event notified (<glfs_h_poll_upcall>).

A new test-file has been added to test the cache_invalidation upcall events.

Below link has a writeup which explains the code changes done -
        URL: https://soumyakoduri.wordpress.com/2015/02/25/glusterfs-understanding-upcall-infrastructure-and-cache-invalidation-support/

Change-Id: Iafc6880000c865fd4da22d0cfc388ec135b5a1c5
BUG: 1200262
Signed-off-by: Soumya Koduri <skoduri@redhat.com>
Reviewed-on: http://review.gluster.org/9536
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>

14 files changed, 654 insertions, 4 deletions

diff --git a/api/src/gfapi.aliases b/api/src/gfapi.aliases
index 2ab7d443eb5..6dfc1089d86 100644
--- a/api/src/gfapi.aliases
+++ b/api/src/gfapi.aliases
@@ -126,8 +126,9 @@ _pub_glfs_get_volfile _glfs_get_volfile$GFAPI_3.6.0
 _pub_glfs_h_access _glfs_h_access$GFAPI_3.6.0
 
 _pub_glfs_ipc _glfs_ipc$GFAPI_3.7.0
+_pub_glfs_h_poll_upcall _glfs_h_poll_upcall$GFAPI_3.7.0
 
 _priv_glfs_free_from_ctx _glfs_free_from_ctx$GFAPI_PRIVATE_3.7.0
 _priv_glfs_new_from_ctx _glfs_new_from_ctx$GFAPI_PRIVATE_3.7.0
 _priv_glfs_resolve _glfs_resolve$GFAPI_PRIVATE_3.7.0
-
+_priv_glfs_process_upcall_event _glfs_process_upcall_event$GFAPI_PRIVATE_3.7.0
diff --git a/api/src/gfapi.map b/api/src/gfapi.map
index 39202e1883f..4721efdff80 100644
--- a/api/src/gfapi.map
+++ b/api/src/gfapi.map
@@ -148,6 +148,7 @@ GFAPI_3.6.0 {
 GFAPI_3.7.0 {
 	global:
 		glfs_ipc;
+                glfs_h_poll_upcall;
 } GFAPI_3.6.0;
 
 GFAPI_PRIVATE_3.7.0 {
@@ -155,5 +156,5 @@ GFAPI_PRIVATE_3.7.0 {
 		glfs_free_from_ctx;
 		glfs_new_from_ctx;
                 glfs_resolve;
+                glfs_process_upcall_event;
 } GFAPI_3.7.0;
-
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index 182317fa41a..f0c769def29 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -16,6 +16,7 @@
 #include "glfs.h"
 #include "compat-errno.h"
 #include <limits.h>
+#include "glusterfs3.h"
 
 #ifdef NAME_MAX
 #define GF_NAME_MAX NAME_MAX
@@ -3495,3 +3496,87 @@ out:
 
 GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0);
 
+/*
+ * This routine is called in case of any notification received
+ * from the server. All the upcall events are queued up in a list
+ * to be read by the applications.
+ *
+ * XXX: Applications may register a cbk function for each 'fs'
+ * which then needs to be called by this routine incase of any
+ * event received. The cbk fn is responsible for notifying the
+ * applications the way it desires for each event queued (for eg.,
+ * can raise a signal or broadcast a cond variable etc.)
+ */
+void
+priv_glfs_process_upcall_event (struct glfs *fs, void *data)
+{
+        int                ret             = -1;
+        inode_t            *inode          = NULL;
+        uuid_t             gfid;
+        upcall_entry       *u_list         = NULL;
+        glusterfs_ctx_t    *ctx            = NULL;
+        struct gf_upcall   *upcall_data    = NULL;
+        struct glfs_object *object         = NULL;
+
+        gf_log (THIS->name, GF_LOG_DEBUG,
+                "Upcall gfapi callback is called");
+
+        if (!fs || !data)
+                goto out;
+
+        /* Unlike in I/O path, "glfs_fini" would not have freed
+         * 'fs' by the time we take lock as it waits for all epoll
+         * threads to exit including this
+         */
+        pthread_mutex_lock (&fs->mutex);
+        {
+                ctx = fs->ctx;
+
+                if (ctx->cleanup_started) {
+                        pthread_mutex_unlock (&fs->mutex);
+                        goto out;
+                }
+
+                fs->pin_refcnt++;
+        }
+        pthread_mutex_unlock (&fs->mutex);
+
+        upcall_data = (struct gf_upcall *)data;
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Upcall gfapi gfid = %s"
+                "ret = %d", (char *)(upcall_data->gfid), ret);
+
+        memcpy(gfid, (char *)(upcall_data->gfid), 16);
+        u_list = GF_CALLOC (1, sizeof(*u_list),
+                            glfs_mt_upcall_entry_t);
+
+        if (!u_list) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Upcall entry allocation"
+                        "failed.");
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&u_list->upcall_list);
+
+        uuid_copy (u_list->gfid, gfid);
+        u_list->event_type = upcall_data->event_type;
+        u_list->flags = (uint32_t)(upcall_data->flags);
+        u_list->expire_time_attr = upcall_data->expire_time_attr;
+
+        pthread_mutex_lock (&fs->upcall_list_mutex);
+        {
+                list_add_tail (&u_list->upcall_list,
+                               &fs->upcall_list);
+        }
+        pthread_mutex_unlock (&fs->upcall_list_mutex);
+
+        pthread_mutex_lock (&fs->mutex);
+        {
+                fs->pin_refcnt--;
+        }
+        pthread_mutex_unlock (&fs->mutex);
+out:
+        return;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0);
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
index 631af01d97b..037315a518d 100644
--- a/api/src/glfs-handleops.c
+++ b/api/src/glfs-handleops.c
@@ -1594,3 +1594,132 @@ out:
 
 GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2);
 
+/*
+ * This API is used to poll for upcall events stored in the
+ * upcall list. Current users of this API is NFS-Ganesha.
+ * Incase of any event received, it will be mapped appropriately
+ * into 'callback_arg' along with the handle to be passed to
+ * NFS-Ganesha.
+ *
+ * Application is responsible for allocating and passing the
+ * references of all the pointers except for "glhandle".
+ * After processing the event, it needs to free "glhandle"
+ *
+ * TODO: there should be a glfs api to destroy these handles,
+ * maybe "glfs_destroy_object" to free the object.
+ *
+ * Also similar to I/Os, the application should ideally stop polling
+ * before calling glfs_fini(..). Hence making an assumption that
+ * 'fs' & ctx structures cannot be freed while in this routine.
+ */
+int
+pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
+{
+        struct glfs_object  *handle   = NULL;
+        uuid_t              gfid;
+        upcall_entry        *u_list   = NULL;
+        upcall_entry        *tmp      = NULL;
+        xlator_t            *subvol   = NULL;
+        int                 found     = 0;
+        int                 reason    = 0;
+        glusterfs_ctx_t     *ctx      = NULL;
+        int                 ret       = -1;
+
+        if (!fs || !up_arg) {
+                errno = EINVAL;
+                goto err;
+        }
+
+        __glfs_entry_fs (fs);
+
+        /* get the active volume */
+        subvol = glfs_active_subvol (fs);
+
+        if (!subvol) {
+                errno = EIO;
+                goto err;
+        }
+
+        up_arg->handle = NULL;
+
+        /* Ideally applications should stop polling before calling
+         * 'glfs_fini'. Yet cross check if cleanup has started
+         */
+        pthread_mutex_lock (&fs->mutex);
+        {
+                ctx = fs->ctx;
+
+                if (ctx->cleanup_started) {
+                        pthread_mutex_unlock (&fs->mutex);
+                        goto out;
+                }
+
+                fs->pin_refcnt++;
+        }
+        pthread_mutex_unlock (&fs->mutex);
+
+        pthread_mutex_lock (&fs->upcall_list_mutex);
+        {
+                list_for_each_entry_safe (u_list, tmp,
+                                          &fs->upcall_list,
+                                          upcall_list) {
+                        uuid_copy (gfid, u_list->gfid);
+                        found = 1;
+                        break;
+                }
+        }
+        /* No other thread can delete this entry. So unlock it */
+        pthread_mutex_unlock (&fs->upcall_list_mutex);
+
+        if (found) {
+                handle = glfs_h_create_from_handle (fs, gfid,
+                                                    GFAPI_HANDLE_LENGTH,
+                                                    &up_arg->buf);
+
+                if (!handle) {
+                        errno = ENOMEM;
+                        goto out;
+                }
+
+                switch (u_list->event_type) {
+                case CACHE_INVALIDATION:
+                        if (u_list->flags & (~(INODE_UPDATE_FLAGS))) {
+                                /* Invalidate CACHE */
+                                reason = INODE_INVALIDATE;
+                                gf_log (subvol->name, GF_LOG_DEBUG,
+                                        "Reason - INODE_INVALIDATION");
+                        } else {
+                                reason = INODE_UPDATE;
+                                gf_log (subvol->name, GF_LOG_DEBUG,
+                                        "Reason - INODE_UPDATE");
+                        }
+                        break;
+                default:
+                        break;
+                }
+
+                up_arg->handle = handle;
+                up_arg->reason = reason;
+                up_arg->flags = u_list->flags;
+                up_arg->expire_time_attr = u_list->expire_time_attr;
+
+                list_del_init (&u_list->upcall_list);
+                GF_FREE (u_list);
+        }
+
+        ret = 0;
+
+out:
+        pthread_mutex_lock (&fs->mutex);
+        {
+                fs->pin_refcnt--;
+        }
+        pthread_mutex_unlock (&fs->mutex);
+
+        glfs_subvol_done (fs, subvol);
+
+err:
+        return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_poll_upcall, 3.7.0);
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
index 12bae6d6e86..c88f134b001 100644
--- a/api/src/glfs-handles.h
+++ b/api/src/glfs-handles.h
@@ -60,6 +60,22 @@
  * glfs_h_create_from_handle */
 #define GFAPI_HANDLE_LENGTH 16
 
+/* These flags should be in sync to the ones defined in upcall.h */
+#define UP_NLINK   0x00000001   /* update nlink */
+#define UP_MODE    0x00000002   /* update mode and ctime */
+#define UP_OWN     0x00000004   /* update mode,uid,gid and ctime */
+#define UP_SIZE    0x00000008   /* update fsize */
+#define UP_TIMES   0x00000010   /* update all times */
+#define UP_ATIME   0x00000020   /* update atime only */
+#define UP_PERM    0x00000040   /* update fields needed for
+                                   permission checking */
+#define UP_RENAME  0x00000080   /* this is a rename op -
+                                   delete the cache entry */
+
+#define INODE_UPDATE_FLAGS (UP_NLINK | UP_MODE | \
+                            UP_OWN | UP_SIZE | \
+                            UP_TIMES | UP_ATIME)
+
 /* Portability non glibc c++ build systems */
 #ifndef __THROW
 # if defined __cplusplus
@@ -82,6 +98,36 @@ __BEGIN_DECLS
 struct glfs_object;
 typedef struct glfs_object glfs_object_t;
 
+/*
+ * Applications (currently NFS-Ganesha) can make use of this
+ * structure to read upcall notifications sent by server.
+ *
+ * They are responsible for allocating and passing the references
+ * of all the pointers except for "handle".
+ *
+ * After processing the event, they need to free "handle"
+ * TODO: there should be a glfs api to destroy these handles,
+ * maybe "glfs_destroy_object" to free the object.
+ */
+struct callback_arg {
+        struct glfs             *fs; /* glfs object */
+        int                     reason;  /* Upcall event type */
+        struct glfs_object      *handle; /* Handle which need to be acted upon */
+        int                     flags; /* Cache UPDATE/INVALIDATE flags */
+        struct stat             buf; /* Latest stat of this entry */
+        unsigned int            expire_time_attr; /* the amount of time for which
+                                                   * the application need to cache
+                                                   * this entry
+                                                   */
+};
+
+/* reason list in callback_arg */
+enum callback_type {
+        CBK_EVENT_NULL,
+        INODE_INVALIDATE,
+        INODE_UPDATE,
+};
+
 /* Handle based operations */
 /* Operations that generate handles */
 struct glfs_object *glfs_h_lookupat (struct glfs *fs,
@@ -188,6 +234,43 @@ int
 glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask) __THROW
         GFAPI_PUBLIC(glfs_h_access, 3.6.0);
 
+/*
+  SYNOPSIS
+
+  glfs_h_poll_upcall: Poll for upcall events given a 'glfs' object.
+
+  DESCRIPTION
+
+  This API is used to poll for upcall events stored in the
+  upcall list. Current users of this API is NFS-Ganesha.
+  Incase of any event received, it will be mapped appropriately
+  into 'callback_arg' along with the handle('glfs_object') to be
+  passed to NFS-Ganesha.
+
+  In case of success, applications need to check the value of
+  cbk->handle to be NON NULL before processing the upcall
+  events.
+
+  PARAMETERS
+
+  @fs: glfs object to poll the upcall events for
+  @cbk: Structure to store upcall events as desired by the application.
+        Application is responsible for allocating and passing the
+        references of all the pointers of this structure except for
+        "handle". In case of any events received, it needs to free
+        "handle"
+
+  RETURN VALUES
+
+  0   : Success.
+  -1  : Error condition, mostly due to out of memory.
+
+*/
+
+int
+glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *cbk) __THROW
+        GFAPI_PUBLIC(glfs_h_poll_upcall, 3.7.0);
+
 __END_DECLS
 
 #endif /* !_GLFS_HANDLES_H */
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index b704c558722..2c0dfe8074e 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -108,6 +108,25 @@
 
 struct glfs;
 
+/* This enum should be in sync with
+ * 'upcall_event_type' declared in
+ * 'xlators/features/upcall/src/upcall.h'
+ */
+enum upcall_event_type_t {
+        EVENT_NULL,
+        CACHE_INVALIDATION,
+};
+typedef enum upcall_event_type_t upcall_event_type;
+
+struct _upcall_entry_t {
+        struct list_head  upcall_list;
+        uuid_t            gfid;
+        upcall_event_type event_type;
+        uint32_t          flags;
+        uint32_t          expire_time_attr;
+};
+typedef struct _upcall_entry_t upcall_entry;
+
 typedef int (*glfs_init_cbk) (struct glfs *fs, int ret);
 
 struct glfs {
@@ -140,6 +159,11 @@ struct glfs {
 	struct list_head    openfds;
 
 	gf_boolean_t        migration_in_progress;
+
+        struct list_head    upcall_list;
+        pthread_mutex_t     upcall_list_mutex; /* mutex for upcall entry list */
+
+        uint32_t            pin_refcnt;
 };
 
 struct glfs_fd {
@@ -182,6 +206,9 @@ fd_t *__glfs_migrate_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd
 
 int glfs_first_lookup (xlator_t *subvol);
 
+void glfs_process_upcall_event (struct glfs *fs, void *data);
+        GFAPI_PRIVATE(glfs_process_upcall_event, 3.7.0);
+
 static inline void
 __glfs_entry_fs (struct glfs *fs)
 {
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
index 0e54719d72c..dfce2f9b78c 100644
--- a/api/src/glfs-master.c
+++ b/api/src/glfs-master.c
@@ -113,6 +113,9 @@ notify (xlator_t *this, int event, void *data, ...)
 		break;
 	case GF_EVENT_CHILD_CONNECTING:
 		break;
+        case GF_EVENT_UPCALL:
+                glfs_process_upcall_event (fs, data);
+                break;
 	default:
 		gf_log (this->name, GF_LOG_DEBUG,
 			"got notify event %d", event);
diff --git a/api/src/glfs-mem-types.h b/api/src/glfs-mem-types.h
index 0a2d4a7df22..c1883f089fd 100644
--- a/api/src/glfs-mem-types.h
+++ b/api/src/glfs-mem-types.h
@@ -25,6 +25,7 @@ enum glfs_mem_types_ {
         glfs_mt_server_cmdline_t,
 	glfs_mt_glfs_object_t,
 	glfs_mt_readdirbuf_t,
+        glfs_mt_upcall_entry_t,
 	glfs_mt_end
 
 };
diff --git a/api/src/glfs.c b/api/src/glfs.c
index f23481bbb4c..02a8984f450 100644
--- a/api/src/glfs.c
+++ b/api/src/glfs.c
@@ -600,6 +600,11 @@ pub_glfs_new (const char *volname)
 
 	INIT_LIST_HEAD (&fs->openfds);
 
+        INIT_LIST_HEAD (&fs->upcall_list);
+        pthread_mutex_init (&fs->upcall_list_mutex, NULL);
+
+        fs->pin_refcnt = 0;
+
 	return fs;
 }
 
@@ -626,6 +631,11 @@ priv_glfs_new_from_ctx (glusterfs_ctx_t *ctx)
 
         INIT_LIST_HEAD (&fs->openfds);
 
+        INIT_LIST_HEAD (&fs->upcall_list);
+        pthread_mutex_init (&fs->upcall_list_mutex, NULL);
+
+        fs->pin_refcnt = 0;
+
         return fs;
 }
 
@@ -635,9 +645,20 @@ GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_new_from_ctx, 3.7.0);
 void
 priv_glfs_free_from_ctx (struct glfs *fs)
 {
+        upcall_entry       *u_list   = NULL;
+        upcall_entry       *tmp      = NULL;
+
         if (!fs)
                 return;
 
+        /* cleanup upcall structures */
+        list_for_each_entry_safe (u_list, tmp,
+                                  &fs->upcall_list,
+                                  upcall_list) {
+                list_del_init (&u_list->upcall_list);
+        }
+        (void) pthread_mutex_destroy (&fs->upcall_list_mutex);
+
         (void) pthread_cond_destroy (&fs->cond);
         (void) pthread_cond_destroy (&fs->child_down_cond);
 
@@ -906,6 +927,7 @@ pub_glfs_fini (struct glfs *fs)
         int                fs_init = 0;
         int                err = -1;
 
+
         if (!fs) {
                 errno = EINVAL;
                 return 0;
@@ -923,10 +945,24 @@ pub_glfs_fini (struct glfs *fs)
 
         while (countdown--) {
                 /* give some time for background frames to finish */
-                if (!call_pool->cnt)
-                        break;
+                pthread_mutex_lock (&fs->mutex);
+                {
+                        /* Do we need to increase countdown? */
+                        if ((!call_pool->cnt) && (!fs->pin_refcnt)) {
+                                gf_log ("glfs", GF_LOG_ERROR,
+                                        "call_pool_cnt - %ld,"
+                                        "pin_refcnt - %d",
+                                        call_pool->cnt, fs->pin_refcnt);
+
+                                ctx->cleanup_started = 1;
+                                pthread_mutex_unlock (&fs->mutex);
+                                break;
+                        }
+                }
+                pthread_mutex_unlock (&fs->mutex);
                 usleep (100000);
         }
+
         /* leaked frames may exist, we ignore */
 
         /*We deem glfs_fini as successful if there are no pending frames in the call
diff --git a/rpc/xdr/src/glusterfs3.h b/rpc/xdr/src/glusterfs3.h
index b3ee267b64d..c2fa15f9e79 100644
--- a/rpc/xdr/src/glusterfs3.h
+++ b/rpc/xdr/src/glusterfs3.h
@@ -279,4 +279,17 @@ gf_proto_upcall_from_upcall (gfs3_upcall_req *gf_up_req,
         gf_up_req->flags            = gf_up_data->flags;
         gf_up_req->expire_time_attr = gf_up_data->expire_time_attr;
 }
+
+static inline void
+gf_proto_upcall_to_upcall (gfs3_upcall_req *gf_up_req,
+                           struct gf_upcall *gf_up_data)
+{
+        if (!gf_up_req || !gf_up_data)
+                return;
+
+        memcpy (gf_up_data->gfid, gf_up_req->gfid, 16);
+        gf_up_data->event_type       = gf_up_req->event_type;
+        gf_up_data->flags            = gf_up_req->flags;
+        gf_up_data->expire_time_attr = gf_up_req->expire_time_attr;
+}
 #endif /* !_GLUSTERFS3_H */
diff --git a/tests/basic/gfapi/Makefile.am b/tests/basic/gfapi/Makefile.am
new file mode 100644
index 00000000000..2041112a7af
--- /dev/null
+++ b/tests/basic/gfapi/Makefile.am
@@ -0,0 +1,15 @@
+## compiles against the *system* version of libgfapi,
+## but not the libgfapi for the testcases
+
+CFLAGS   = -Wall -g $(shell pkg-config --cflags glusterfs-api)
+LDFLAGS  = $(shell pkg-config --libs glusterfs-api)
+
+BINARIES = upcall-cache-invalidate
+
+%: %.c
+
+all: $(BINARIES)
+
+clean:
+        -$(RM) $(BINARIES)
+
diff --git a/tests/basic/gfapi/upcall-cache-invalidate.c b/tests/basic/gfapi/upcall-cache-invalidate.c
new file mode 100644
index 00000000000..e91d05d4058
--- /dev/null
+++ b/tests/basic/gfapi/upcall-cache-invalidate.c
@@ -0,0 +1,188 @@
+#include <fcntl.h>
+#include <unistd.h>
+#include <time.h>
+#include <limits.h>
+#include <alloca.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <glusterfs/api/glfs.h>
+#include <glusterfs/api/glfs-handles.h>
+int gfapi = 1;
+
+#define LOG_ERR(func, ret) do { \
+        if (ret != 0) {            \
+                fprintf (stderr, "%s : returned error %d (%s)\n", \
+                         func, ret, strerror (errno)); \
+                goto out; \
+        } else { \
+                fprintf (stderr, "%s : returned %d\n", func, ret); \
+        } \
+        } while (0)
+
+int
+main (int argc, char *argv[])
+{
+        glfs_t    *fs = NULL;
+        glfs_t    *fs2 = NULL;
+        glfs_t    *fs_tmp = NULL;
+        glfs_t    *fs_tmp2 = NULL;
+        int        ret = 0, i;
+        glfs_fd_t *fd = NULL;
+        glfs_fd_t *fd2 = NULL;
+        glfs_fd_t *fd_tmp = NULL;
+        glfs_fd_t *fd_tmp2 = NULL;
+        char       readbuf[32];
+        char      *filename = "file_tmp";
+        char      *writebuf = NULL;
+        char      *vol_id  = NULL;
+        unsigned int       cnt = 1;
+        struct    callback_arg cbk;
+        char      *logfile = NULL;
+        char      *volname = NULL;
+
+        cbk.handle = NULL;
+
+        if (argc != 3) {
+                fprintf (stderr, "Invalid argument\n");
+                exit(1);
+        }
+
+        volname = argv[1];
+        logfile = argv[2];
+
+        fs = glfs_new (volname);
+        if (!fs) {
+                fprintf (stderr, "glfs_new: returned NULL\n");
+                return -1;
+        }
+
+        ret = glfs_set_volfile_server (fs, "tcp", "localhost", 24007);
+        LOG_ERR("glfs_set_volfile_server", ret);
+
+        ret = glfs_set_logging (fs, logfile, 7);
+        LOG_ERR("glfs_set_logging", ret);
+
+        ret = glfs_init (fs);
+        LOG_ERR("glfs_init", ret);
+
+        fs2 = glfs_new (volname);
+        if (!fs2) {
+                fprintf (stderr, "glfs_new fs2: returned NULL\n");
+                return 1;
+        }
+
+        ret = glfs_set_volfile_server (fs2, "tcp", "localhost", 24007);
+        LOG_ERR("glfs_set_volfile_server-fs2", ret);
+
+        ret = glfs_set_logging (fs2, logfile, 7);
+        LOG_ERR("glfs_set_logging-fs2", ret);
+
+        ret = glfs_init (fs2);
+        LOG_ERR("glfs_init-fs2", ret);
+
+        fd = glfs_creat(fs, filename, O_RDWR|O_SYNC, 0644);
+        if (fd <= 0) {
+                ret = -1;
+                LOG_ERR ("glfs_creat", ret);
+        }
+        fprintf (stderr, "glfs-create fd - %d\n", fd);
+
+        fd2 = glfs_open(fs2, filename, O_SYNC|O_RDWR|O_CREAT);
+        if (fd2 <= 0) {
+                ret = -1;
+                LOG_ERR ("glfs_open-fs2", ret);
+        }
+        fprintf (stderr, "glfs-open fd2 - %d\n", fd2);
+
+        do {
+                if (cnt%2) {
+                        fd_tmp = fd;
+                        fs_tmp = fs;
+                        fd_tmp2 = fd2;
+                        fs_tmp2 = fs2;
+                } else {
+                        fd_tmp = fd2;
+                        fs_tmp = fs2;
+                        fd_tmp2 = fd;
+                        fs_tmp2 = fs;
+                }
+
+                /* WRITE on fd_tmp */
+                writebuf = malloc(10);
+                if (writebuf) {
+                        memcpy (writebuf, "abcd", 4);
+                        ret = glfs_write (fd_tmp, writebuf, 4, 0);
+                        if (ret <= 0)   {
+                                ret = -1;
+                                LOG_ERR ("glfs_write", ret);
+                        } else {
+                                fprintf (stderr,
+                                         "glfs_write suceeded\n");
+                        }
+                        free(writebuf);
+                } else {
+                        fprintf (stderr,
+                                 "Could not allocate writebuf\n");
+                        return -1;
+                }
+
+                /* READ on fd_tmp2 */
+                ret = glfs_lseek (fd_tmp2, 0, SEEK_SET);
+                LOG_ERR ("glfs_lseek", ret);
+
+                ret = glfs_pread (fd_tmp2, readbuf, 4, 0, 0);
+
+                if (ret <= 0) {
+                        ret = -1;
+                        LOG_ERR ("glfs_pread", ret);
+                } else {
+                        fprintf (stderr, "glfs_read: %s\n", readbuf);
+                }
+
+                /* Open() fops seem to be not performed on server side until
+                 * there are I/Os on that fd
+                 */
+                if (cnt > 2) {
+                        ret = glfs_h_poll_upcall(fs_tmp, &cbk);
+                        LOG_ERR ("glfs_h_poll_upcall", ret);
+                        if (cbk.handle) {
+                                fprintf (stderr, " upcall event type - %d,"
+                                                 " flags - %d, expire_time_attr - %d\n" ,
+                                         cbk.reason, cbk.flags, cbk.expire_time_attr);
+                        } else {
+                                fprintf (stderr,
+                                         "Dint receive upcall notify event");
+                                ret = -1;
+                                goto err;
+                        }
+                }
+
+                sleep(5);
+        } while (++cnt < 5);
+
+err:
+        glfs_close(fd);
+        LOG_ERR ("glfs_close", ret);
+
+        glfs_close(fd2);
+        LOG_ERR ("glfs_close-fd2", ret);
+
+out:
+        if (fs) {
+                ret = glfs_fini(fs);
+                fprintf (stderr, "glfs_fini(fs) returned %d \n", ret);
+        }
+
+        if (fs2) {
+                ret = glfs_fini(fs2);
+                fprintf (stderr, "glfs_fini(fs2) returned %d \n", ret);
+        }
+
+        if (ret)
+                exit(1);
+        exit(0);
+}
+
+
diff --git a/tests/basic/gfapi/upcall-cache-invalidate.sh b/tests/basic/gfapi/upcall-cache-invalidate.sh
new file mode 100755
index 00000000000..20aeb1ec27a
--- /dev/null
+++ b/tests/basic/gfapi/upcall-cache-invalidate.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+cleanup;
+
+# Upcall feature is disable for now. A new xlator option
+# will be introduced to turn it on. Skipping this test
+# till then.
+
+SKIP_TESTS;
+exit 0
+
+TEST glusterd
+
+TEST $CLI volume create $V0 localhost:$B0/brick1;
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+logdir=`gluster --print-logdir`
+
+build_tester $(dirname $0)/upcall-cache-invalidate.c -lgfapi -o $(dirname $0)/upcall-cache-invalidate
+
+TEST ./$(dirname $0)/upcall-cache-invalidate $V0  $logdir/upcall-cache-invalidate.log
+
+cleanup_tester $(dirname $0)/upcall-cache-invalidate
+
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0
+
+cleanup;
diff --git a/xlators/protocol/client/src/client-callback.c b/xlators/protocol/client/src/client-callback.c
index b2707cb395b..fdfb3dc313b 100644
--- a/xlators/protocol/client/src/client-callback.c
+++ b/xlators/protocol/client/src/client-callback.c
@@ -15,6 +15,7 @@
 
 #include "client.h"
 #include "rpc-clnt.h"
+#include "defaults.h"
 
 int
 client_cbk_null (struct rpc_clnt *rpc, void *mydata, void *data)
@@ -40,10 +41,43 @@ client_cbk_ino_flush (struct rpc_clnt *rpc, void *mydata, void *data)
         return 0;
 }
 
+int
+client_cbk_upcall (struct rpc_clnt *rpc, void *mydata, void *data)
+{
+        int              ret          = -1;
+        gfs3_upcall_req  up_req;
+        struct gf_upcall upcall_data;
+        struct iovec     *iov         = NULL;
+
+        gf_log (THIS->name, GF_LOG_TRACE,
+                "Upcall callback is called");
+
+        if (!rpc || !mydata || !data)
+                goto out;
+
+        iov = (struct iovec *)data;
+        ret =  xdr_to_generic (*iov, &up_req,
+                               (xdrproc_t)xdr_gfs3_upcall_req);
+
+        if (ret < 0)
+                goto out;
+
+        gf_proto_upcall_to_upcall (&up_req, &upcall_data);
+
+        gf_log (THIS->name, GF_LOG_TRACE, "Upcall gfid = %s, ret = %d",
+                 (char *)(up_req.gfid), ret);
+
+        default_notify (THIS, GF_EVENT_UPCALL, &upcall_data);
+
+out:
+        return 0;
+}
+
 rpcclnt_cb_actor_t gluster_cbk_actors[GF_CBK_MAXVALUE] = {
         [GF_CBK_NULL]      = {"NULL",      GF_CBK_NULL,      client_cbk_null },
         [GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, client_cbk_fetchspec },
         [GF_CBK_INO_FLUSH] = {"INO_FLUSH", GF_CBK_INO_FLUSH, client_cbk_ino_flush },
+        [GF_CBK_UPCALL]    = {"UPCALL",    GF_CBK_UPCALL,    client_cbk_upcall },
 };