Upcall: Process each of the upcall events separately

As suggested during the code-review of Bug1200262, have modified
GF_CBK_UPCALL to be exlusively GF_CBK_CACHE_INVALIDATION.
Thus, for any new upcall event, a new CBK procedure will be added.

Also made changes to store upcall data separately based on the
upcall event type received.

BUG: 1200262
Change-Id: I0f5e53d6f5ece16aecb514a0a426dca40fa1c755
Signed-off-by: Soumya Koduri <skoduri@redhat.com>
Reviewed-on: http://review.gluster.org/10049
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>

15 files changed, 252 insertions, 127 deletions

diff --git a/api/src/gfapi-messages.h b/api/src/gfapi-messages.h
index d7cfd8371e6..2c59af24113 100644
--- a/api/src/gfapi-messages.h
+++ b/api/src/gfapi-messages.h
@@ -96,6 +96,7 @@
 #define API_MSG_VOLFILE_INFO                    (GLFS_GFAPI_BASE + 43)
 #define API_MSG_VOLFILE_CONNECTING              (GLFS_GFAPI_BASE + 44)
 #define API_MSG_NEW_GRAPH                       (GLFS_GFAPI_BASE + 45)
+#define API_MSG_ALLOC_FAILED                    (GLFS_GFAPI_BASE + 46)
 /*------------*/
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index de5be43f435..ae7bb087af5 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -14,6 +14,7 @@
 #include "glfs-mem-types.h"
 #include "syncop.h"
 #include "glfs.h"
+#include "gfapi-messages.h"
 #include "compat-errno.h"
 #include <limits.h>
 #include "glusterfs3.h"
@@ -26,6 +27,48 @@
 
 #define READDIRBUF_SIZE (sizeof(struct dirent) + GF_NAME_MAX + 1)
 
+/*
+ * This routine is called when an upcall event of type
+ * 'GF_UPCALL_CACHE_INVALIDATION' is received.
+ * It makes a copy of the contents of the upcall cache-invalidation
+ * data received into an entry which is stored in the upcall list
+ * maintained by gfapi.
+ */
+int
+glfs_get_upcall_cache_invalidation (struct gf_upcall *to_up_data,
+                                    struct gf_upcall *from_up_data)
+{
+
+        struct gf_upcall_cache_invalidation *ca_data = NULL;
+        struct gf_upcall_cache_invalidation *f_ca_data = NULL;
+        int                                 ret      = -1;
+
+        GF_VALIDATE_OR_GOTO (THIS->name, to_up_data, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, from_up_data, out);
+
+        f_ca_data = from_up_data->data;
+        GF_VALIDATE_OR_GOTO (THIS->name, f_ca_data, out);
+
+        ca_data = GF_CALLOC (1, sizeof(*ca_data),
+                            glfs_mt_upcall_entry_t);
+
+        if (!ca_data) {
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        API_MSG_ALLOC_FAILED,
+                        "Upcall entry allocation failed.");
+                goto out;
+        }
+
+        to_up_data->data = ca_data;
+
+        ca_data->flags = f_ca_data->flags;
+        ca_data->expire_time_attr = f_ca_data->expire_time_attr;
+
+        ret = 0;
+out:
+        return ret;
+}
+
 int
 glfs_loc_link (loc_t *loc, struct iatt *iatt)
 {
@@ -3493,7 +3536,6 @@ void
 priv_glfs_process_upcall_event (struct glfs *fs, void *data)
 {
         int                ret             = -1;
-        inode_t            *inode          = NULL;
         uuid_t             gfid;
         upcall_entry       *u_list         = NULL;
         glusterfs_ctx_t    *ctx            = NULL;
@@ -3523,12 +3565,12 @@ priv_glfs_process_upcall_event (struct glfs *fs, void *data)
         }
         pthread_mutex_unlock (&fs->mutex);
 
+
         upcall_data = (struct gf_upcall *)data;
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Upcall gfapi gfid = %s"
+        gf_log (THIS->name, GF_LOG_TRACE, "Upcall gfapi gfid = %s"
                 "ret = %d", (char *)(upcall_data->gfid), ret);
 
-        memcpy(gfid, (char *)(upcall_data->gfid), 16);
         u_list = GF_CALLOC (1, sizeof(*u_list),
                             glfs_mt_upcall_entry_t);
 
@@ -3540,10 +3582,24 @@ priv_glfs_process_upcall_event (struct glfs *fs, void *data)
 
         INIT_LIST_HEAD (&u_list->upcall_list);
 
-        gf_uuid_copy (u_list->gfid, gfid);
-        u_list->event_type = upcall_data->event_type;
-        u_list->flags = (uint32_t)(upcall_data->flags);
-        u_list->expire_time_attr = upcall_data->expire_time_attr;
+        gf_uuid_copy (u_list->upcall_data.gfid, upcall_data->gfid);
+        u_list->upcall_data.event_type = upcall_data->event_type;
+
+        switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+                ret = glfs_get_upcall_cache_invalidation (&u_list->upcall_data,
+                                                          upcall_data);
+                break;
+        default:
+                goto out;
+        }
+
+        if (ret) {
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        API_MSG_INVALID_ENTRY,
+                        "Upcall entry validation failed.");
+                goto out;
+        }
 
         pthread_mutex_lock (&fs->upcall_list_mutex);
         {
@@ -3557,7 +3613,11 @@ priv_glfs_process_upcall_event (struct glfs *fs, void *data)
                 fs->pin_refcnt--;
         }
         pthread_mutex_unlock (&fs->mutex);
+
+        ret = 0;
 out:
+        if (ret && u_list)
+                GF_FREE(u_list);
         return;
 }
 
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
index ee8ea626c51..b3f65e05fef 100644
--- a/api/src/glfs-handleops.c
+++ b/api/src/glfs-handleops.c
@@ -1641,15 +1641,17 @@ GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2);
 int
 pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
 {
-        struct glfs_object  *object   = NULL;
-        uuid_t              gfid;
-        upcall_entry        *u_list   = NULL;
-        upcall_entry        *tmp      = NULL;
-        xlator_t            *subvol   = NULL;
-        int                 found     = 0;
-        int                 reason    = 0;
-        glusterfs_ctx_t     *ctx      = NULL;
-        int                 ret       = -1;
+        struct glfs_object                  *object         = NULL;
+        uuid_t                              gfid;
+        upcall_entry                        *u_list         = NULL;
+        upcall_entry                        *tmp            = NULL;
+        xlator_t                            *subvol         = NULL;
+        int                                 found           = 0;
+        int                                 reason          = 0;
+        glusterfs_ctx_t                     *ctx            = NULL;
+        int                                 ret             = -1;
+        struct gf_upcall                    *upcall_data    = NULL;
+        struct gf_upcall_cache_invalidation *ca_data        = NULL;
 
         if (!fs || !up_arg) {
                 errno = EINVAL;
@@ -1689,7 +1691,7 @@ pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
                 list_for_each_entry_safe (u_list, tmp,
                                           &fs->upcall_list,
                                           upcall_list) {
-                        gf_uuid_copy (gfid, u_list->gfid);
+                        gf_uuid_copy (gfid, u_list->upcall_data.gfid);
                         found = 1;
                         break;
                 }
@@ -1707,12 +1709,19 @@ pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
                         goto out;
                 }
 
-                switch (u_list->event_type) {
-                case CACHE_INVALIDATION:
+                upcall_data = &u_list->upcall_data;
+
+                switch (upcall_data->event_type) {
+                case GF_UPCALL_CACHE_INVALIDATION:
                         /* XXX: Need to revisit this to support
                          * GFAPI_INODE_UPDATE if required.
                          */
+                        ca_data = upcall_data->data;
+                        GF_VALIDATE_OR_GOTO ("glfs_h_poll_upcall",
+                                             ca_data, out);
                         reason = GFAPI_INODE_INVALIDATE;
+                        up_arg->flags = ca_data->flags;
+                        up_arg->expire_time_attr = ca_data->expire_time_attr;
                         break;
                 default:
                         break;
@@ -1720,10 +1729,9 @@ pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
 
                 up_arg->object = object;
                 up_arg->reason = reason;
-                up_arg->flags = u_list->flags;
-                up_arg->expire_time_attr = u_list->expire_time_attr;
 
                 list_del_init (&u_list->upcall_list);
+                GF_FREE (u_list->upcall_data.data);
                 GF_FREE (u_list);
         }
 
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index faa53dd3225..20fbb5c3201 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -13,6 +13,7 @@
 #define _GLFS_INTERNAL_H
 
 #include "xlator.h"
+#include "glusterfs.h"
 
 #define GLFS_SYMLINK_MAX_FOLLOW 2048
 
@@ -108,22 +109,9 @@
 
 struct glfs;
 
-/* This enum should be in sync with
- * 'upcall_event_type' declared in
- * 'xlators/features/upcall/src/upcall.h'
- */
-enum upcall_event_type_t {
-        EVENT_NULL,
-        CACHE_INVALIDATION,
-};
-typedef enum upcall_event_type_t upcall_event_type;
-
 struct _upcall_entry_t {
         struct list_head  upcall_list;
-        uuid_t            gfid;
-        upcall_event_type event_type;
-        uint32_t          flags;
-        uint32_t          expire_time_attr;
+        struct gf_upcall  upcall_data;
 };
 typedef struct _upcall_entry_t upcall_entry;
 
@@ -345,5 +333,7 @@ struct glfs *glfs_new_from_ctx (glusterfs_ctx_t *ctx)
 void glfs_free_from_ctx (struct glfs *fs)
          GFAPI_PRIVATE(glfs_free_from_ctx, 3.7.0);
 
+int glfs_get_upcall_cache_invalidation (struct gf_upcall *to_up_data,
+                                        struct gf_upcall *from_up_data);
 
 #endif /* !_GLFS_INTERNAL_H */
diff --git a/api/src/glfs.c b/api/src/glfs.c
index 12ca751e34d..23377f651d1 100644
--- a/api/src/glfs.c
+++ b/api/src/glfs.c
@@ -709,6 +709,8 @@ priv_glfs_free_from_ctx (struct glfs *fs)
                                   &fs->upcall_list,
                                   upcall_list) {
                 list_del_init (&u_list->upcall_list);
+                GF_FREE (u_list->upcall_data.data);
+                GF_FREE (u_list);
         }
         (void) pthread_mutex_destroy (&fs->upcall_list_mutex);
 
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 5ca09151ae8..9926d314874 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -37,6 +37,7 @@
 #include "list.h"
 #include "logging.h"
 #include "lkowner.h"
+#include "compat-uuid.h"
 
 #define GF_YES 1
 #define GF_NO  0
@@ -370,6 +371,11 @@ typedef enum {
 } gf_xattrop_flags_t;
 
 
+typedef enum {
+        GF_UPCALL_EVENT_NULL,
+        GF_UPCALL_CACHE_INVALIDATION,
+} gf_upcall_event_t;
+
 #define GF_SET_IF_NOT_PRESENT 0x1 /* default behaviour */
 #define GF_SET_OVERWRITE      0x2 /* Overwrite with the buf given */
 #define GF_SET_DIR_ONLY       0x4
@@ -617,9 +623,13 @@ struct gf_flock {
 };
 
 struct gf_upcall {
-        char  *client_uid;
-	char  gfid[16];
-        u_int event_type;
+        char   *client_uid;
+        uuid_t gfid;
+        u_int  event_type;
+        void   *data;
+};
+
+struct gf_upcall_cache_invalidation {
         u_int flags;
         u_int expire_time_attr;
 };
diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h
index 60697b8fa66..3b964331892 100644
--- a/rpc/rpc-lib/src/protocol-common.h
+++ b/rpc/rpc-lib/src/protocol-common.h
@@ -132,9 +132,7 @@ enum gf_cbk_procnum {
         GF_CBK_INO_FLUSH,
         GF_CBK_EVENT_NOTIFY,
         GF_CBK_GET_SNAPS,
-        /* XXX: Have separate events for each
-         * UPCALL event - BZ 1200268) */
-        GF_CBK_UPCALL,
+        GF_CBK_CACHE_INVALIDATION,
         GF_CBK_MAXVALUE,
 };
 
diff --git a/rpc/xdr/src/glusterfs3-xdr.x b/rpc/xdr/src/glusterfs3-xdr.x
index 0136aec2b08..f0cd4cbc953 100644
--- a/rpc/xdr/src/glusterfs3-xdr.x
+++ b/rpc/xdr/src/glusterfs3-xdr.x
@@ -44,12 +44,13 @@ struct gf_iatt {
 };
 
 
-struct gfs3_upcall_req {
+struct gfs3_cbk_cache_invalidation_req {
         opaque         gfid[16];
         unsigned int   event_type; /* Upcall event type */
         unsigned int   flags;  /* or mask of events incase of inotify */
         unsigned int   expire_time_attr; /* the amount of time which client
                                           * can cache this entry */
+        opaque   xdata<>; /* Extra data */
 };
 
 struct gfs3_stat_req {
diff --git a/rpc/xdr/src/glusterfs3.h b/rpc/xdr/src/glusterfs3.h
index c2fa15f9e79..2acdfc756c4 100644
--- a/rpc/xdr/src/glusterfs3.h
+++ b/rpc/xdr/src/glusterfs3.h
@@ -268,28 +268,50 @@ gf_stat_from_iatt (struct gf_iatt *gf_stat, struct iatt *iatt)
 }
 
 static inline void
-gf_proto_upcall_from_upcall (gfs3_upcall_req *gf_up_req,
-                             struct gf_upcall *gf_up_data)
+gf_proto_cache_invalidation_from_upcall (gfs3_cbk_cache_invalidation_req *gf_c_req,
+                                         struct gf_upcall *gf_up_data)
 {
-        if (!gf_up_req || !gf_up_data)
-                return;
+        struct gf_upcall_cache_invalidation *gf_c_data = NULL;
+        int    is_cache_inval                          = 0;
+
+        GF_VALIDATE_OR_GOTO(THIS->name, gf_c_req, out);
+        GF_VALIDATE_OR_GOTO(THIS->name, gf_up_data, out);
+
+        is_cache_inval = ((gf_up_data->event_type ==
+                          GF_UPCALL_CACHE_INVALIDATION) ? 1 : 0);
+        GF_VALIDATE_OR_GOTO(THIS->name, is_cache_inval, out);
+
+        gf_c_data = (struct gf_upcall_cache_invalidation *)gf_up_data->data;
+        GF_VALIDATE_OR_GOTO(THIS->name, gf_c_data, out);
+
+        memcpy (gf_c_req->gfid, gf_up_data->gfid, 16);
+        gf_c_req->event_type       = gf_up_data->event_type;
+        gf_c_req->flags            = gf_c_data->flags;
+        gf_c_req->expire_time_attr = gf_c_data->expire_time_attr;
 
-        memcpy (gf_up_req->gfid, gf_up_data->gfid, 16);
-        gf_up_req->event_type       = gf_up_data->event_type;
-        gf_up_req->flags            = gf_up_data->flags;
-        gf_up_req->expire_time_attr = gf_up_data->expire_time_attr;
+out:
+        return;
 }
 
 static inline void
-gf_proto_upcall_to_upcall (gfs3_upcall_req *gf_up_req,
-                           struct gf_upcall *gf_up_data)
+gf_proto_cache_invalidation_to_upcall (gfs3_cbk_cache_invalidation_req *gf_c_req,
+                                       struct gf_upcall *gf_up_data)
 {
-        if (!gf_up_req || !gf_up_data)
-                return;
+        struct gf_upcall_cache_invalidation *gf_c_data = NULL;
+
+        GF_VALIDATE_OR_GOTO(THIS->name, gf_c_req, out);
+        GF_VALIDATE_OR_GOTO(THIS->name, gf_up_data, out);
+
+        gf_c_data = (struct gf_upcall_cache_invalidation *)gf_up_data->data;
+        GF_VALIDATE_OR_GOTO(THIS->name, gf_c_data, out);
+
+        memcpy (gf_up_data->gfid, gf_c_req->gfid, 16);
+        gf_up_data->event_type       = gf_c_req->event_type;
+
+        gf_c_data->flags            = gf_c_req->flags;
+        gf_c_data->expire_time_attr = gf_c_req->expire_time_attr;
 
-        memcpy (gf_up_data->gfid, gf_up_req->gfid, 16);
-        gf_up_data->event_type       = gf_up_req->event_type;
-        gf_up_data->flags            = gf_up_req->flags;
-        gf_up_data->expire_time_attr = gf_up_req->expire_time_attr;
+out:
+        return;
 }
 #endif /* !_GLUSTERFS3_H */
diff --git a/xlators/features/upcall/src/upcall-internal.c b/xlators/features/upcall/src/upcall-internal.c
index 41548afb930..a7f0fd991cf 100644
--- a/xlators/features/upcall/src/upcall-internal.c
+++ b/xlators/features/upcall/src/upcall-internal.c
@@ -423,7 +423,7 @@ upcall_client_cache_invalidate (xlator_t *this, uuid_t gfid,
                 /* Send notify call */
                 gf_uuid_copy(n_event_data.gfid, gfid);
                 n_event_data.client_entry = up_client_entry;
-                n_event_data.event_type = CACHE_INVALIDATION;
+                n_event_data.event_type = GF_UPCALL_CACHE_INVALIDATION;
                 n_event_data.invalidate_flags = flags;
 
                 /* Need to send inode flags */
diff --git a/xlators/features/upcall/src/upcall.c b/xlators/features/upcall/src/upcall.c
index 93861e7955f..21f8e0ba339 100644
--- a/xlators/features/upcall/src/upcall.c
+++ b/xlators/features/upcall/src/upcall.c
@@ -887,6 +887,7 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         notify_event_data_t *notify_event    = NULL;
         struct gf_upcall    up_req           = {0,};
         upcall_client_t     *up_client_entry = NULL;
+        struct gf_upcall_cache_invalidation  ca_req = {0,};
 
         switch (event) {
         case GF_EVENT_UPCALL:
@@ -903,16 +904,17 @@ notify (xlator_t *this, int32_t event, void *data, ...)
 
                 up_req.client_uid = up_client_entry->client_uid;
 
-                memcpy (up_req.gfid, notify_event->gfid, 16);
+                gf_uuid_copy (up_req.gfid, notify_event->gfid);
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Sending notify to the client- %s, gfid - %s",
                         up_client_entry->client_uid, up_req.gfid);
 
                 switch (notify_event->event_type) {
-                case CACHE_INVALIDATION:
-                        GF_ASSERT (notify_event->extra);
-                        up_req.flags = notify_event->invalidate_flags;
-                        up_req.expire_time_attr = up_client_entry->expire_time_attr;
+                case GF_UPCALL_CACHE_INVALIDATION:
+                        ca_req.flags = notify_event->invalidate_flags;
+                        ca_req.expire_time_attr =
+                                        up_client_entry->expire_time_attr;
+                        up_req.data = &ca_req;
                         break;
                 default:
                         goto out;
diff --git a/xlators/features/upcall/src/upcall.h b/xlators/features/upcall/src/upcall.h
index 7cbec22bbf2..3994c2648f4 100644
--- a/xlators/features/upcall/src/upcall.h
+++ b/xlators/features/upcall/src/upcall.h
@@ -59,12 +59,6 @@ struct _upcall_private_t {
 };
 typedef struct _upcall_private_t upcall_private_t;
 
-enum _upcall_event_type_t {
-        EVENT_NULL,
-        CACHE_INVALIDATION,
-};
-typedef enum _upcall_event_type_t upcall_event_type_t;
-
 struct _upcall_client_t {
         struct list_head client_list;
         /* strdup to store client_uid, strdup. Free it explicitly */
@@ -86,7 +80,7 @@ typedef struct _upcall_inode_ctx_t upcall_inode_ctx_t;
 struct _notify_event_data {
         uuid_t gfid;
         upcall_client_t *client_entry;
-        upcall_event_type_t event_type;
+        gf_upcall_event_t event_type;
         uint32_t invalidate_flags;
         /* any extra data needed, like inode flags
          * to be invalidated incase of cache invalidation,
diff --git a/xlators/protocol/client/src/client-callback.c b/xlators/protocol/client/src/client-callback.c
index fdfb3dc313b..722db7d0f5f 100644
--- a/xlators/protocol/client/src/client-callback.c
+++ b/xlators/protocol/client/src/client-callback.c
@@ -42,30 +42,36 @@ client_cbk_ino_flush (struct rpc_clnt *rpc, void *mydata, void *data)
 }
 
 int
-client_cbk_upcall (struct rpc_clnt *rpc, void *mydata, void *data)
+client_cbk_cache_invalidation (struct rpc_clnt *rpc, void *mydata, void *data)
 {
-        int              ret          = -1;
-        gfs3_upcall_req  up_req;
-        struct gf_upcall upcall_data;
-        struct iovec     *iov         = NULL;
+        int              ret                        = -1;
+        struct iovec     *iov                       = NULL;
+        struct gf_upcall upcall_data                = {0,};
+        uuid_t           gfid;
+        struct gf_upcall_cache_invalidation ca_data = {0,};
+        gfs3_cbk_cache_invalidation_req     ca_req  = {{0,},};
 
-        gf_log (THIS->name, GF_LOG_TRACE,
-                "Upcall callback is called");
+        gf_log (THIS->name, GF_LOG_TRACE, "Upcall callback is called");
 
         if (!rpc || !mydata || !data)
                 goto out;
 
         iov = (struct iovec *)data;
-        ret =  xdr_to_generic (*iov, &up_req,
-                               (xdrproc_t)xdr_gfs3_upcall_req);
+        ret =  xdr_to_generic (*iov, &ca_req,
+                               (xdrproc_t)xdr_gfs3_cbk_cache_invalidation_req);
 
-        if (ret < 0)
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "XDR decode of cache_invalidation failed.");
                 goto out;
+        }
 
-        gf_proto_upcall_to_upcall (&up_req, &upcall_data);
+        upcall_data.data = &ca_data;
+        gf_proto_cache_invalidation_to_upcall (&ca_req, &upcall_data);
 
+        memcpy (gfid, ca_req.gfid, 16);
         gf_log (THIS->name, GF_LOG_TRACE, "Upcall gfid = %s, ret = %d",
-                 (char *)(up_req.gfid), ret);
+                uuid_utoa (gfid), ret);
 
         default_notify (THIS, GF_EVENT_UPCALL, &upcall_data);
 
@@ -77,7 +83,9 @@ rpcclnt_cb_actor_t gluster_cbk_actors[GF_CBK_MAXVALUE] = {
         [GF_CBK_NULL]      = {"NULL",      GF_CBK_NULL,      client_cbk_null },
         [GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, client_cbk_fetchspec },
         [GF_CBK_INO_FLUSH] = {"INO_FLUSH", GF_CBK_INO_FLUSH, client_cbk_ino_flush },
-        [GF_CBK_UPCALL]    = {"UPCALL",    GF_CBK_UPCALL,    client_cbk_upcall },
+        [GF_CBK_CACHE_INVALIDATION] = {"CACHE_INVALIDATION",
+                                       GF_CBK_CACHE_INVALIDATION,
+                                       client_cbk_cache_invalidation },
 };
 
 
diff --git a/xlators/protocol/server/src/server-helpers.h b/xlators/protocol/server/src/server-helpers.h
index e7238d9ca6a..73b01b197e0 100644
--- a/xlators/protocol/server/src/server-helpers.h
+++ b/xlators/protocol/server/src/server-helpers.h
@@ -57,4 +57,5 @@ int auth_set_username_passwd (dict_t *input_params, dict_t *config_params,
                               struct _client_t *client);
 
 server_ctx_t *server_ctx_get (client_t *client, xlator_t *xlator);
+int server_process_event_upcall (xlator_t *this, void *data);
 #endif /* !_SERVER_HELPERS_H */
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index 470c57dac41..bdb9bae9cb6 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -1114,73 +1114,101 @@ fini (xlator_t *this)
 }
 
 int
+server_process_event_upcall (xlator_t *this, void *data)
+{
+        int              ret          = -1;
+        server_conf_t    *conf        = NULL;
+        client_t         *client      = NULL;
+        char             *client_uid  = NULL;
+        struct gf_upcall *upcall_data = NULL;
+        void             *up_req      = NULL;
+        rpc_transport_t  *xprt        = NULL;
+        enum gf_cbk_procnum cbk_procnum          = GF_CBK_NULL;
+        gfs3_cbk_cache_invalidation_req gf_c_req = {{0,},};
+        xdrproc_t        xdrproc;
+
+        GF_VALIDATE_OR_GOTO(this->name, data, out);
+
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+        upcall_data = (struct gf_upcall *)data;
+
+        client_uid = upcall_data->client_uid;
+
+        GF_VALIDATE_OR_GOTO(this->name, client_uid, out);
+
+        switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+                gf_proto_cache_invalidation_from_upcall (&gf_c_req,
+                                                         upcall_data);
+
+                up_req = &gf_c_req;
+                cbk_procnum = GF_CBK_CACHE_INVALIDATION;
+                xdrproc = (xdrproc_t)xdr_gfs3_cbk_cache_invalidation_req;
+                break;
+        default:
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Received invalid upcall event(%d)",
+                        upcall_data->event_type);
+                goto out;
+        }
+
+        pthread_mutex_lock (&conf->mutex);
+        {
+                list_for_each_entry (xprt, &conf->xprt_list, list) {
+                        client = xprt->xl_private;
+
+                        if (strcmp(client->client_uid, client_uid))
+                                continue;
+
+                        rpcsvc_request_submit(conf->rpc, xprt,
+                                              &server_cbk_prog,
+                                              cbk_procnum,
+                                              up_req,
+                                              this->ctx,
+                                              xdrproc);
+                        break;
+                }
+        }
+        pthread_mutex_unlock (&conf->mutex);
+        ret = 0;
+out:
+        return ret;
+}
+
+int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
-        int              ret          = 0;
+        int              ret          = -1;
         int32_t          val          = 0;
         dict_t           *dict        = NULL;
         dict_t           *output      = NULL;
         va_list          ap;
-        client_t         *client      = NULL;
-        char             *client_uid  = NULL;
-        struct gf_upcall *upcall_data = NULL;
-        gfs3_upcall_req  up_req;
-        server_conf_t    *conf        = NULL;
-        rpc_transport_t  *xprt        = NULL;
 
         dict = data;
         va_start (ap, data);
         output = va_arg (ap, dict_t*);
         va_end (ap);
 
-        conf = this->private;
-        if (!conf)
-                return 0;
-
         switch (event) {
         case GF_EVENT_UPCALL:
         {
-                if (!data) {
-                        ret = -1;
-                        goto out;
-                }
-
-                upcall_data = (struct gf_upcall *)data;
+                GF_VALIDATE_OR_GOTO(this->name, data, out);
 
-                client_uid = upcall_data->client_uid;
-
-                if (!client_uid) {
-                        ret = -1;
+                ret = server_process_event_upcall (this, data);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "server_process_event_upcall failed");
                         goto out;
                 }
-
-                gf_proto_upcall_from_upcall (&up_req, upcall_data);
-
-                pthread_mutex_lock (&conf->mutex);
-                {
-                        list_for_each_entry (xprt, &conf->xprt_list, list) {
-                                client = xprt->xl_private;
-
-                                if (strcmp(client->client_uid, client_uid))
-                                        continue;
-
-                                rpcsvc_request_submit(
-                                        conf->rpc, xprt,
-                                        &server_cbk_prog,
-                                        GF_CBK_UPCALL,
-                                        &up_req,
-                                        this->ctx,
-                                        (xdrproc_t)xdr_gfs3_upcall_req);
-                                break;
-                        }
-                }
-                pthread_mutex_unlock (&conf->mutex);
                 break;
         }
         default:
                 default_notify (this, event, data);
                 break;
         }
+        ret = 0;
 out:
         return ret;
 }