Upcall: Send stat as part of cache_invalidation notifications

Have added support to send attributes of both entries and
its parent (include oldparent in case of RENAME fop) in the
same notification request to avoid multiple rpc requests.

Also, made changes in gfapi to send parent object and its
attributes changed in a single upcall event.

Change-Id: I92833da3bcec38d65216921c2ce4d10367c32ef1
BUG: 1200262
Signed-off-by: Soumya Koduri <skoduri@redhat.com>
Reviewed-on: http://review.gluster.org/10460
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>

14 files changed, 317 insertions, 198 deletions

diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index ae7bb087af5..4aad28005f2 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -61,8 +61,11 @@ glfs_get_upcall_cache_invalidation (struct gf_upcall *to_up_data,
 
         to_up_data->data = ca_data;
 
-        ca_data->flags = f_ca_data->flags;
+        ca_data->flags      = f_ca_data->flags;
         ca_data->expire_time_attr = f_ca_data->expire_time_attr;
+        ca_data->stat       = f_ca_data->stat;
+        ca_data->p_stat     = f_ca_data->p_stat;
+        ca_data->oldp_stat  = f_ca_data->oldp_stat;
 
         ret = 0;
 out:
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
index c3ac789569d..13b47a1b74d 100644
--- a/api/src/glfs-handleops.c
+++ b/api/src/glfs-handleops.c
@@ -1621,6 +1621,76 @@ out:
 
 GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2);
 
+int
+glfs_h_poll_cache_invalidation (struct glfs *fs,
+                                struct callback_arg *up_arg,
+                                struct gf_upcall *upcall_data)
+{
+        int                                 ret           = -1;
+        struct glfs_object                  *p_object     = NULL;
+        struct glfs_object                  *oldp_object  = NULL;
+        struct glfs_object                  *object       = NULL;
+        struct gf_upcall_cache_invalidation *ca_data      = NULL;
+        struct callback_inode_arg           *up_inode_arg = NULL;
+
+        ca_data = upcall_data->data;
+        GF_VALIDATE_OR_GOTO ("glfs_h_poll_cache_invalidation",
+                             ca_data, out);
+
+        object = glfs_h_create_from_handle (fs, upcall_data->gfid,
+                                            GFAPI_HANDLE_LENGTH,
+                                            NULL);
+        GF_VALIDATE_OR_GOTO ("glfs_h_poll_cache_invalidation",
+                             object, out);
+
+        up_inode_arg = calloc (1, sizeof (struct callback_inode_arg));
+        GF_VALIDATE_OR_GOTO ("glfs_h_poll_cache_invalidation",
+                             up_inode_arg, out);
+
+        up_arg->event_arg = up_inode_arg;
+
+        up_inode_arg->object = object;
+        up_inode_arg->flags = ca_data->flags;
+        up_inode_arg->expire_time_attr = ca_data->expire_time_attr;
+
+        /* XXX: Update stat as well incase of UP_*_TIMES.
+         * This will be addressed as part of INODE_UPDATE */
+        if (ca_data->flags & GFAPI_INODE_UPDATE_FLAGS) {
+                glfs_iatt_to_stat (fs, &ca_data->stat, &up_inode_arg->buf);
+        }
+
+        if (ca_data->flags & GFAPI_UP_PARENT_TIMES) {
+                p_object = glfs_h_create_from_handle (fs,
+                                                      ca_data->p_stat.ia_gfid,
+                                                      GFAPI_HANDLE_LENGTH,
+                                                      NULL);
+                GF_VALIDATE_OR_GOTO ("glfs_h_poll_cache_invalidation",
+                                       p_object, out);
+
+                glfs_iatt_to_stat (fs, &ca_data->p_stat, &up_inode_arg->p_buf);
+        }
+        up_inode_arg->p_object = p_object;
+
+        /* In case of RENAME, update old parent as well */
+        if (ca_data->flags & GFAPI_UP_RENAME) {
+                oldp_object = glfs_h_create_from_handle (fs,
+                                                     ca_data->oldp_stat.ia_gfid,
+                                                     GFAPI_HANDLE_LENGTH,
+                                                     NULL);
+                GF_VALIDATE_OR_GOTO ("glfs_h_poll_cache_invalidation",
+                                       oldp_object, out);
+
+                glfs_iatt_to_stat (fs, &ca_data->oldp_stat,
+                                   &up_inode_arg->oldp_buf);
+        }
+        up_inode_arg->oldp_object = oldp_object;
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
 /*
  * This API is used to poll for upcall events stored in the
  * upcall list. Current users of this API is NFS-Ganesha.
@@ -1628,11 +1698,17 @@ GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2);
  * into 'callback_arg' along with the handle object  to be passed
  * to NFS-Ganesha.
  *
- * On success, applications need to check for 'object' to decide
+ * On success, applications need to check for 'reason' to decide
  * if any upcall event is received.
  *
- * After processing the event, they need to free "object"
- * using glfs_h_close(..).
+ * Current supported upcall_events -
+ *      GFAPI_INODE_INVALIDATE -
+ *              'arg - callback_inode_arg
+ *
+ * After processing the event, applications need to free 'event_arg'.
+ *
+ * Incase of INODE_INVALIDATE, applications need to free "object",
+ * "p_object" and "oldp_object" using glfs_h_close(..).
  *
  * Also similar to I/Os, the application should ideally stop polling
  * before calling glfs_fini(..). Hence making an assumption that
@@ -1641,8 +1717,6 @@ GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2);
 int
 pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
 {
-        struct glfs_object                  *object         = NULL;
-        uuid_t                              gfid;
         upcall_entry                        *u_list         = NULL;
         upcall_entry                        *tmp            = NULL;
         xlator_t                            *subvol         = NULL;
@@ -1651,7 +1725,6 @@ pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
         glusterfs_ctx_t                     *ctx            = NULL;
         int                                 ret             = -1;
         struct gf_upcall                    *upcall_data    = NULL;
-        struct gf_upcall_cache_invalidation *ca_data        = NULL;
 
         if (!fs || !up_arg) {
                 errno = EINVAL;
@@ -1668,8 +1741,6 @@ pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
                 goto err;
         }
 
-        up_arg->object = NULL;
-
         /* Ideally applications should stop polling before calling
          * 'glfs_fini'. Yet cross check if cleanup has started
          */
@@ -1691,7 +1762,6 @@ pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
                 list_for_each_entry_safe (u_list, tmp,
                                           &fs->upcall_list,
                                           upcall_list) {
-                        gf_uuid_copy (gfid, u_list->upcall_data.gfid);
                         found = 1;
                         break;
                 }
@@ -1700,11 +1770,20 @@ pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
         pthread_mutex_unlock (&fs->upcall_list_mutex);
 
         if (found) {
-                object = glfs_h_create_from_handle (fs, gfid,
-                                                    GFAPI_HANDLE_LENGTH,
-                                                    NULL);
+                upcall_data = &u_list->upcall_data;
 
-                if (!object) {
+                switch (upcall_data->event_type) {
+                case GF_UPCALL_CACHE_INVALIDATION:
+                        /* XXX: Need to revisit this to support
+                         * GFAPI_INODE_UPDATE if required.
+                         */
+                        reason = GFAPI_INODE_INVALIDATE;
+                        ret = glfs_h_poll_cache_invalidation (fs,
+                                                              up_arg,
+                                                              upcall_data);
+                        if (!ret) {
+                                break;
+                        }
                         /* It could so happen that the file which got
                          * upcall notification may have got deleted
                          * by other thread. Irrespective of the error,
@@ -1714,32 +1793,15 @@ pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
                          * as up_arg->object will be NULL */
                         gf_log (subvol->name, GF_LOG_WARNING,
                                 "handle creation of %s failed: %s",
-                                uuid_utoa (gfid), strerror (errno));
+                                uuid_utoa (upcall_data->gfid),
+                                strerror (errno));
 
                         reason = GFAPI_CBK_EVENT_NULL;
-                } else {
-
-                        upcall_data = &u_list->upcall_data;
-
-                        switch (upcall_data->event_type) {
-                        case GF_UPCALL_CACHE_INVALIDATION:
-                                /* XXX: Need to revisit this to support
-                                 * GFAPI_INODE_UPDATE if required.
-                                 */
-                                ca_data = upcall_data->data;
-                                GF_VALIDATE_OR_GOTO ("glfs_h_poll_upcall",
-                                                     ca_data, out);
-                                reason = GFAPI_INODE_INVALIDATE;
-                                up_arg->flags = ca_data->flags;
-                                up_arg->expire_time_attr = ca_data->expire_time_attr;
-
-                                break;
-                        default:
-                                break;
-                        }
+                        break;
+                default:
+                        break;
                 }
 
-                up_arg->object = object;
                 up_arg->reason = reason;
 
                 list_del_init (&u_list->upcall_list);
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
index 02204a76385..b5813b23448 100644
--- a/api/src/glfs-handles.h
+++ b/api/src/glfs-handles.h
@@ -71,6 +71,9 @@
                                          permission checking */
 #define GFAPI_UP_RENAME  0x00000080   /* this is a rename op -
                                          delete the cache entry */
+#define GFAPI_UP_FORGET  0x00000100   /* inode_forget on server side -
+                                         invalidate the cache entry */
+#define GFAPI_UP_PARENT_TIMES   0x00000200   /* update parent dir times */
 
 #define GFAPI_INODE_UPDATE_FLAGS (GFAPI_UP_NLINK | GFAPI_UP_MODE | \
                                   GFAPI_UP_OWN | GFAPI_UP_SIZE | \
@@ -102,15 +105,30 @@ typedef struct glfs_object glfs_object_t;
  * Applications (currently NFS-Ganesha) can make use of this
  * structure to read upcall notifications sent by server.
  *
- * On success, applications need to check for 'object' to decide
+ * On success, applications need to check for 'reason' to decide
  * if any upcall event is received.
  *
- * After processing the event, they need to free "object"
- * using glfs_h_close(..).
+ * Currently supported upcall_events -
+ *      GFAPI_INODE_INVALIDATE -
+ *              'event_arg' - callback_inode_arg
+ *
+ * After processing the event, applications need to free 'event_arg'.
+ *
+ * Also similar to I/Os, the application should ideally stop polling
+ * before calling glfs_fini(..). Hence making an assumption that
+ * 'fs' & ctx structures cannot be freed while in this routine.
  */
 struct callback_arg {
         struct glfs             *fs; /* glfs object */
         int                     reason;  /* Upcall event type */
+        void                    *event_arg; /* changes based in the event type */
+};
+
+/*
+ * After processing upcall event, they need to free "object" , "p_object",
+ * "oldp_object" using glfs_h_close(..).
+ */
+struct callback_inode_arg {
         struct glfs_object      *object; /* Object which need to be acted upon */
         int                     flags; /* Cache UPDATE/INVALIDATE flags */
         struct stat             buf; /* Latest stat of this entry */
@@ -118,6 +136,12 @@ struct callback_arg {
                                                    * the application need to cache
                                                    * this entry
                                                    */
+        struct glfs_object      *p_object; /* parent Object to be updated */
+        struct stat             p_buf; /* Latest stat of parent dir handle */
+        struct glfs_object      *oldp_object; /* Old parent Object
+                                               * to be updated */
+        struct stat             oldp_buf; /* Latest stat of old parent
+                                           * dir handle */
 };
 
 /* reason list in callback_arg */
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index 20fbb5c3201..972f2e4cf49 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -14,6 +14,8 @@
 
 #include "xlator.h"
 #include "glusterfs.h"
+#include "upcall-utils.h"
+#include "glfs-handles.h"
 
 #define GLFS_SYMLINK_MAX_FOLLOW 2048
 
@@ -335,5 +337,9 @@ void glfs_free_from_ctx (struct glfs *fs)
 
 int glfs_get_upcall_cache_invalidation (struct gf_upcall *to_up_data,
                                         struct gf_upcall *from_up_data);
+int
+glfs_h_poll_cache_invalidation (struct glfs *fs,
+                                struct callback_arg *up_arg,
+                                struct gf_upcall *upcall_data);
 
 #endif /* !_GLFS_INTERNAL_H */
diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am
index c7b85bdf839..e5f9f52f323 100644
--- a/libglusterfs/src/Makefile.am
+++ b/libglusterfs/src/Makefile.am
@@ -48,7 +48,7 @@ noinst_HEADERS = common-utils.h defaults.h dict.h glusterfs.h hashfn.h timespec.
 	$(CONTRIBDIR)/libexecinfo/execinfo_compat.h \
 	unittest/unittest.h quota-common-utils.h rot-buffs.h \
 	$(CONTRIBDIR)/timer-wheel/timer-wheel.h compat-uuid.h \
-	tw.h
+	tw.h upcall-utils.h
 
 if !HAVE_LIBUUID
 # FIXME: unbundle libuuid, see compat-uuid.h.
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 50c86d91467..ba59e4d54db 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -370,11 +370,6 @@ typedef enum {
         GF_XATTROP_AND_ARRAY
 } gf_xattrop_flags_t;
 
-typedef enum {
-        GF_UPCALL_EVENT_NULL,
-        GF_UPCALL_CACHE_INVALIDATION,
-} gf_upcall_event_t;
-
 #define GF_SET_IF_NOT_PRESENT 0x1 /* default behaviour */
 #define GF_SET_OVERWRITE      0x2 /* Overwrite with the buf given */
 #define GF_SET_DIR_ONLY       0x4
@@ -621,18 +616,6 @@ struct gf_flock {
         gf_lkowner_t l_owner;
 };
 
-struct gf_upcall {
-        char   *client_uid;
-        uuid_t gfid;
-        u_int  event_type;
-        void   *data;
-};
-
-struct gf_upcall_cache_invalidation {
-        u_int flags;
-        u_int expire_time_attr;
-};
-
 #define GF_MUST_CHECK __attribute__((warn_unused_result))
 /*
  * Some macros (e.g. ALLOC_OR_GOTO) set variables in function scope, but the
diff --git a/libglusterfs/src/upcall-utils.h b/libglusterfs/src/upcall-utils.h
new file mode 100644
index 00000000000..001566fd85e
--- /dev/null
+++ b/libglusterfs/src/upcall-utils.h
@@ -0,0 +1,38 @@
+/*
+  Copyright (c) 2015, Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _UPCALL_UTILS_H
+#define _UPCALL_UTILS_H
+
+#include "iatt.h"
+#include "compat-uuid.h"
+#include "compat.h"
+
+typedef enum {
+        GF_UPCALL_EVENT_NULL,
+        GF_UPCALL_CACHE_INVALIDATION,
+} gf_upcall_event_t;
+
+struct gf_upcall {
+        char      *client_uid;
+        uuid_t    gfid;
+        uint32_t  event_type;
+        void      *data;
+};
+
+struct gf_upcall_cache_invalidation {
+        uint32_t flags;
+        uint32_t expire_time_attr;
+        struct iatt stat;
+        struct iatt p_stat; /* parent dir stat */
+        struct iatt oldp_stat; /* oldparent dir stat */
+};
+
+#endif /* _UPCALL_UTILS_H */
diff --git a/rpc/xdr/src/glusterfs3.h b/rpc/xdr/src/glusterfs3.h
index 6808064cb95..e50bd9e6c59 100644
--- a/rpc/xdr/src/glusterfs3.h
+++ b/rpc/xdr/src/glusterfs3.h
@@ -16,6 +16,8 @@
 #include "xdr-generic.h"
 #include "glusterfs3-xdr.h"
 #include "iatt.h"
+#include "protocol-common.h"
+#include "upcall-utils.h"
 
 #define xdr_decoded_remaining_addr(xdr)        ((&xdr)->x_private)
 #define xdr_decoded_remaining_len(xdr)         ((&xdr)->x_handy)
@@ -289,6 +291,9 @@ gf_proto_cache_invalidation_from_upcall (gfs3_cbk_cache_invalidation_req *gf_c_r
         gf_c_req->event_type       = gf_up_data->event_type;
         gf_c_req->flags            = gf_c_data->flags;
         gf_c_req->expire_time_attr = gf_c_data->expire_time_attr;
+        gf_stat_from_iatt (&gf_c_req->stat, &gf_c_data->stat);
+        gf_stat_from_iatt (&gf_c_req->parent_stat, &gf_c_data->p_stat);
+        gf_stat_from_iatt (&gf_c_req->oldparent_stat, &gf_c_data->oldp_stat);
 
 out:
         return;
@@ -319,6 +324,9 @@ gf_proto_cache_invalidation_to_upcall (gfs3_cbk_cache_invalidation_req *gf_c_req
 
         gf_c_data->flags            = gf_c_req->flags;
         gf_c_data->expire_time_attr = gf_c_req->expire_time_attr;
+        gf_stat_to_iatt (&gf_c_req->stat, &gf_c_data->stat);
+        gf_stat_to_iatt (&gf_c_req->parent_stat, &gf_c_data->p_stat);
+        gf_stat_to_iatt (&gf_c_req->oldparent_stat, &gf_c_data->oldp_stat);
 
 out:
         return;
diff --git a/tests/basic/gfapi/upcall-cache-invalidate.c b/tests/basic/gfapi/upcall-cache-invalidate.c
index 64811f3274a..cc2e6a0fe33 100644
--- a/tests/basic/gfapi/upcall-cache-invalidate.c
+++ b/tests/basic/gfapi/upcall-cache-invalidate.c
@@ -41,8 +41,9 @@ main (int argc, char *argv[])
         struct    callback_arg cbk;
         char      *logfile = NULL;
         char      *volname = NULL;
+        struct callback_inode_arg *in_arg = NULL;
 
-        cbk.object = NULL;
+        cbk.reason = 0;
 
         if (argc != 3) {
                 fprintf (stderr, "Invalid argument\n");
@@ -147,10 +148,18 @@ main (int argc, char *argv[])
                 if (cnt > 2) {
                         ret = glfs_h_poll_upcall(fs_tmp, &cbk);
                         LOG_ERR ("glfs_h_poll_upcall", ret);
-                        if (cbk.object) {
+                        /* Expect 'GFAPI_INODE_INVALIDATE' upcall event. */
+                        if (cbk.reason == GFAPI_INODE_INVALIDATE) {
+                                in_arg = cbk.event_arg;
                                 fprintf (stderr, " upcall event type - %d,"
-                                                 " flags - %d, expire_time_attr - %d\n" ,
-                                         cbk.reason, cbk.flags, cbk.expire_time_attr);
+                                         " object(%p), flags(%d), "
+                                         " expire_time_attr(%d)\n" ,
+                                         cbk.reason, in_arg->object,
+                                         in_arg->flags,
+                                         in_arg->expire_time_attr);
+                                ret = glfs_h_close (in_arg->object);
+                                LOG_ERR ("glfs_h_close", ret);
+                                free (in_arg);
                         } else {
                                 fprintf (stderr,
                                          "Dint receive upcall notify event");
diff --git a/xlators/features/upcall/src/upcall-cache-invalidation.h b/xlators/features/upcall/src/upcall-cache-invalidation.h
index c39962b68c4..edb8285827f 100644
--- a/xlators/features/upcall/src/upcall-cache-invalidation.h
+++ b/xlators/features/upcall/src/upcall-cache-invalidation.h
@@ -33,6 +33,7 @@
                                    delete the cache entry */
 #define UP_FORGET  0x00000100   /* inode_forget on server side -
                                    invalidate the cache entry */
+#define UP_PARENT_TIMES   0x00000200   /* update parent dir times */
 
 /* for fops - open, read, lk, */
 #define UP_UPDATE_CLIENT        (UP_ATIME)
@@ -48,7 +49,7 @@
 
 /* to invalidate parent directory entries for fops -rename, unlink,
  * rmdir, mkdir, create */
-#define UP_PARENT_DENTRY_FLAGS  (UP_TIMES)
+#define UP_PARENT_DENTRY_FLAGS  (UP_PARENT_TIMES)
 
 /* for fop - unlink, link, rmdir, mkdir */
 #define UP_NLINK_FLAGS          (UP_NLINK | UP_TIMES)
diff --git a/xlators/features/upcall/src/upcall-internal.c b/xlators/features/upcall/src/upcall-internal.c
index 19d19289624..b128ab3bdc9 100644
--- a/xlators/features/upcall/src/upcall-internal.c
+++ b/xlators/features/upcall/src/upcall-internal.c
@@ -294,7 +294,7 @@ upcall_cleanup_expired_clients (xlator_t *this,
                                         goto out;
                                 }
                                 gf_log (THIS->name, GF_LOG_TRACE,
-                                        "Cleaned up client_entry(%s) of",
+                                        "Cleaned up client_entry(%s)",
                                         up_client->client_uid);
                         }
                 }
@@ -450,24 +450,6 @@ upcall_reaper_thread_init (xlator_t *this)
         return ret;
 }
 
-void
-upcall_cache_invalidate_dir (call_frame_t *frame, xlator_t *this,
-                             client_t *client, inode_t *inode, uint32_t flags)
-{
-        dentry_t        *dentry;
-        dentry_t        *dentry_tmp;
-
-        if (!is_cache_invalidation_enabled(this))
-                return;
-
-        list_for_each_entry_safe (dentry, dentry_tmp,
-                                  &inode->dentry_list,
-                                  inode_list) {
-                upcall_cache_invalidate (frame, this, client,
-                                         dentry->inode, flags);
-        }
-}
-
 /*
  * Given a gfid, client, first fetch upcall_entry_t based on gfid.
  * Later traverse through the client list of that upcall entry. If this client
@@ -480,7 +462,8 @@ upcall_cache_invalidate_dir (call_frame_t *frame, xlator_t *this,
  */
 void
 upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client,
-                         inode_t *inode, uint32_t flags)
+                         inode_t *inode, uint32_t flags, struct iatt *stbuf,
+                         struct iatt *p_stbuf, struct iatt *oldp_stbuf)
 {
         upcall_client_t *up_client       = NULL;
         upcall_client_t *up_client_entry = NULL;
@@ -536,7 +519,8 @@ upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client,
                         upcall_client_cache_invalidate(this,
                                                        inode->gfid,
                                                        up_client_entry,
-                                                       flags);
+                                                       flags, stbuf,
+                                                       p_stbuf, oldp_stbuf);
                 }
 
                 if (!found) {
@@ -556,28 +540,50 @@ upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client,
 void
 upcall_client_cache_invalidate (xlator_t *this, uuid_t gfid,
                                 upcall_client_t *up_client_entry,
-                                uint32_t flags)
+                                uint32_t flags, struct iatt *stbuf,
+                                struct iatt *p_stbuf,
+                                struct iatt *oldp_stbuf)
 {
-        notify_event_data_t n_event_data;
-        time_t timeout   = 0;
+        struct gf_upcall                    up_req  = {0,};
+        struct gf_upcall_cache_invalidation ca_req  = {0,};
+        time_t                              timeout = 0;
+        int                                 ret     = -1;
         time_t t_expired = time(NULL) - up_client_entry->access_time;
 
         timeout = get_cache_invalidation_timeout(this);
 
         if (t_expired < timeout) {
                 /* Send notify call */
-                gf_uuid_copy(n_event_data.gfid, gfid);
-                n_event_data.client_entry = up_client_entry;
-                n_event_data.event_type = GF_UPCALL_CACHE_INVALIDATION;
-                n_event_data.invalidate_flags = flags;
-
-                /* Need to send inode flags */
-                this->notify (this, GF_EVENT_UPCALL, &n_event_data);
+                up_req.client_uid = up_client_entry->client_uid;
+                gf_uuid_copy (up_req.gfid, gfid);
+
+                ca_req.flags = flags;
+                ca_req.expire_time_attr =
+                                 up_client_entry->expire_time_attr;
+                if (stbuf)
+                        ca_req.stat = *stbuf;
+                if (p_stbuf)
+                        ca_req.p_stat = *p_stbuf;
+                if (oldp_stbuf)
+                        ca_req.oldp_stat = *oldp_stbuf;
+
+                up_req.data = &ca_req;
+                up_req.event_type = GF_UPCALL_CACHE_INVALIDATION;
 
                 gf_log (THIS->name, GF_LOG_TRACE,
                         "Cache invalidation notification sent to %s",
                         up_client_entry->client_uid);
 
+                /* Need to send inode flags */
+                ret = this->notify (this, GF_EVENT_UPCALL, &up_req);
+
+                /*
+                 * notify may fail as the client could have been
+                 * dis(re)connected. Cleanup the client entry.
+                 */
+                if (ret < 0)
+                        __upcall_cleanup_client_entry (up_client_entry);
+
         } else {
                 gf_log (THIS->name, GF_LOG_TRACE,
                         "Cache invalidation notification NOT sent to %s",
@@ -621,7 +627,8 @@ upcall_cache_forget (xlator_t *this, inode_t *inode, upcall_inode_ctx_t *up_inod
                         upcall_client_cache_invalidate(this,
                                                        inode->gfid,
                                                        up_client_entry,
-                                                       flags);
+                                                       flags, NULL,
+                                                       NULL, NULL);
                 }
 
         }
diff --git a/xlators/features/upcall/src/upcall-messages.h b/xlators/features/upcall/src/upcall-messages.h
index 7b004d678c3..5472b4a5cbd 100644
--- a/xlators/features/upcall/src/upcall-messages.h
+++ b/xlators/features/upcall/src/upcall-messages.h
@@ -57,6 +57,7 @@
  */
 #define UPCALL_MSG_NO_MEMORY             (GLFS_COMP_BASE_UPCALL + 1)
 #define UPCALL_MSG_INTERNAL_ERROR        (GLFS_COMP_BASE_UPCALL + 2)
+#define UPCALL_MSG_NOTIFY_FAILED         (GLFS_COMP_BASE_UPCALL + 3)
 
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 
diff --git a/xlators/features/upcall/src/upcall.c b/xlators/features/upcall/src/upcall.c
index c68c0258fb8..5f6c29e1b79 100644
--- a/xlators/features/upcall/src/upcall.c
+++ b/xlators/features/upcall/src/upcall.c
@@ -51,7 +51,8 @@ up_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 NULL, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
@@ -105,7 +106,8 @@ up_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_WRITE_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 postbuf, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (writev, frame, op_ret, op_errno,
@@ -166,7 +168,8 @@ up_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 NULL, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (readv, frame, op_ret, op_errno, vector,
@@ -224,7 +227,8 @@ up_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 NULL, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata);
@@ -278,7 +282,8 @@ up_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_WRITE_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 postbuf, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (truncate, frame, op_ret, op_errno,
@@ -339,7 +344,8 @@ up_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
          * Bug1200271.
          */
         flags = UP_ATTR_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 statpost, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (setattr, frame, op_ret, op_errno,
@@ -397,17 +403,9 @@ up_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret < 0) || !local) {
                 goto out;
         }
-        flags = UP_RENAME_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
-
-        /* Need to invalidate old and new parent entries as well */
-        flags = UP_PARENT_DENTRY_FLAGS;
-        upcall_cache_invalidate_dir (frame, this, client, local->inode, flags);
-
-        /* notify oldparent as well */
-        flags = UP_PARENT_DENTRY_FLAGS;
-        upcall_cache_invalidate_dir (frame, this, client,
-                                     local->rename_oldloc.inode, flags);
+        flags = (UP_RENAME_FLAGS | UP_PARENT_DENTRY_FLAGS);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 stbuf, postnewparent, postoldparent);
 
 out:
         UPCALL_STACK_UNWIND (rename, frame, op_ret, op_errno,
@@ -466,12 +464,9 @@ up_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret < 0) || !local) {
                 goto out;
         }
-        flags = UP_NLINK_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
-
-        flags = UP_PARENT_DENTRY_FLAGS;
-        /* invalidate parent's entry too */
-        upcall_cache_invalidate_dir (frame, this, client, local->inode, flags);
+        flags = (UP_NLINK_FLAGS | UP_PARENT_DENTRY_FLAGS);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 NULL, postparent, NULL);
 
 out:
         UPCALL_STACK_UNWIND (unlink, frame, op_ret, op_errno,
@@ -526,10 +521,10 @@ up_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret < 0) || !local) {
                 goto out;
         }
-        flags = UP_NLINK_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        flags = (UP_NLINK_FLAGS | UP_PARENT_DENTRY_FLAGS);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 stbuf, postparent, NULL);
 
-        /* do we need to update parent as well?? */
 out:
         UPCALL_STACK_UNWIND (link, frame, op_ret, op_errno,
                              inode, stbuf, preparent, postparent, xdata);
@@ -584,12 +579,10 @@ up_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret < 0) || !local) {
                 goto out;
         }
-        flags = UP_NLINK_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
 
-        /* invalidate parent's entry too */
-        flags = UP_PARENT_DENTRY_FLAGS;
-        upcall_cache_invalidate_dir (frame, this, client, local->inode, flags);
+        flags = (UP_NLINK_FLAGS | UP_PARENT_DENTRY_FLAGS);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 NULL, postparent, NULL);
 
 out:
         UPCALL_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
@@ -648,7 +641,8 @@ up_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         /* invalidate parent's entry too */
         flags = UP_PARENT_DENTRY_FLAGS;
-        upcall_cache_invalidate_dir (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 stbuf, postparent, NULL);
 
 out:
         UPCALL_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
@@ -709,7 +703,8 @@ up_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         /* As its a new file create, no need of sending notification */
         /* However invalidate parent's entry */
         flags = UP_PARENT_DENTRY_FLAGS;
-        upcall_cache_invalidate_dir (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 stbuf, postparent, NULL);
 
 out:
         UPCALL_STACK_UNWIND (create, frame, op_ret, op_errno, fd,
@@ -769,7 +764,8 @@ up_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 NULL, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf,
@@ -826,7 +822,8 @@ up_stat_cbk (call_frame_t *frame, void *cookie,
                 goto out;
         }
         flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 NULL, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (stat, frame, op_ret, op_errno, buf,
@@ -939,7 +936,8 @@ up_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 NULL, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
@@ -994,7 +992,8 @@ up_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 NULL, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf,
@@ -1054,7 +1053,8 @@ up_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         /* invalidate parent's entry too */
         flags = UP_PARENT_DENTRY_FLAGS;
-        upcall_cache_invalidate_dir (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 buf, postparent, NULL);
 
 out:
         UPCALL_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
@@ -1114,7 +1114,8 @@ up_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         /* invalidate parent's entry too */
         flags = UP_PARENT_DENTRY_FLAGS;
-        upcall_cache_invalidate_dir (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 buf, postparent, NULL);
 
 out:
         UPCALL_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
@@ -1172,7 +1173,8 @@ up_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 NULL, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd, xdata);
@@ -1227,7 +1229,8 @@ up_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 NULL, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
@@ -1282,7 +1285,8 @@ up_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 NULL, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries, xdata);
@@ -1396,7 +1400,8 @@ up_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_WRITE_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 post, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (fallocate, frame, op_ret, op_errno, pre,
@@ -1453,7 +1458,8 @@ up_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_WRITE_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 post, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (discard, frame, op_ret, op_errno, pre,
@@ -1510,7 +1516,8 @@ up_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
         flags = UP_WRITE_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate (frame, this, client, local->inode, flags,
+                                 post, NULL, NULL);
 
 out:
         UPCALL_STACK_UNWIND (zerofill, frame, op_ret, op_errno, pre,
@@ -1725,10 +1732,7 @@ notify (xlator_t *this, int32_t event, void *data, ...)
 {
         int                 ret              = -1;
         int32_t             val              = 0;
-        notify_event_data_t *notify_event    = NULL;
-        struct gf_upcall    up_req           = {0,};
-        upcall_client_t     *up_client_entry = NULL;
-        struct gf_upcall_cache_invalidation ca_req = {0,};
+        struct gf_upcall    *up_req          = NULL;
 
         switch (event) {
         case GF_EVENT_UPCALL:
@@ -1736,41 +1740,20 @@ notify (xlator_t *this, int32_t event, void *data, ...)
                 gf_log (this->name, GF_LOG_DEBUG, "Upcall Notify event = %d",
                         event);
 
-                notify_event = (notify_event_data_t *) data;
-                up_client_entry = notify_event->client_entry;
+                up_req = (struct gf_upcall *) data;
 
-                if (!up_client_entry) {
-                        goto out;
-                }
+                GF_VALIDATE_OR_GOTO(this->name, up_req, out);
+
+                ret = default_notify (this, event, up_req);
 
-                up_req.client_uid = up_client_entry->client_uid;
-
-                gf_uuid_copy (up_req.gfid, notify_event->gfid);
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Sending notify to the client- %s, gfid - %s",
-                        up_client_entry->client_uid, up_req.gfid);
-
-                switch (notify_event->event_type) {
-                case GF_UPCALL_CACHE_INVALIDATION:
-                        ca_req.flags = notify_event->invalidate_flags;
-                        ca_req.expire_time_attr =
-                                        up_client_entry->expire_time_attr;
-                        up_req.data = &ca_req;
-                        break;
-                default:
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_WARNING, 0,
+                                UPCALL_MSG_NOTIFY_FAILED,
+                                "Failed to notify cache invalidation"
+                                " to client(%s)",
+                                up_req->client_uid);
                         goto out;
                 }
-
-                up_req.event_type = notify_event->event_type;
-
-                ret = default_notify (this, event, &up_req);
-
-                /*
-                 * notify may fail as the client could have been
-                 * dis(re)connected. Cleanup the client entry.
-                 */
-                if (ret < 0)
-                        __upcall_cleanup_client_entry (up_client_entry);
         }
         break;
         default:
@@ -1784,7 +1767,15 @@ out:
 }
 
 struct xlator_fops fops = {
-        /* fops which do not trigger upcall
+        /* fops which change only "ATIME" do not result
+         * in any cache invalidation. Hence upcall
+         * notifications are not sent in this case.
+         * But however, we need to store/update the
+         * client info in the upcall state to be able
+         * to notify them incase of any changes done
+         * to the data.
+         *
+         * Below such fops do not trigger upcall
          * notifications but will add/update
          * clients info in the upcall inode ctx.*/
         .lookup      = up_lookup,
diff --git a/xlators/features/upcall/src/upcall.h b/xlators/features/upcall/src/upcall.h
index 41895786382..295ddc26167 100644
--- a/xlators/features/upcall/src/upcall.h
+++ b/xlators/features/upcall/src/upcall.h
@@ -16,15 +16,11 @@
 #endif
 
 #include "compat-errno.h"
-#include "stack.h"
-#include "call-stub.h"
 #include "upcall-mem-types.h"
 #include "client_t.h"
-#include "rpcsvc.h"
-#include "lkowner.h"
-#include "locking.h"
 #include "upcall-messages.h"
 #include "upcall-cache-invalidation.h"
+#include "upcall-utils.h"
 
 #define EXIT_IF_UPCALL_OFF(this, label) do {                       \
         if (!is_upcall_enabled(this))                              \
@@ -84,18 +80,6 @@ struct _upcall_inode_ctx_t {
 };
 typedef struct _upcall_inode_ctx_t upcall_inode_ctx_t;
 
-struct _notify_event_data {
-        uuid_t gfid;
-        upcall_client_t *client_entry;
-        gf_upcall_event_t event_type;
-        uint32_t invalidate_flags;
-        /* any extra data needed, like inode flags
-         * to be invalidated incase of cache invalidation,
-         * may be fd for lease recalls */
-        void *extra;
-};
-typedef struct _notify_event_data notify_event_data_t;
-
 struct upcall_local {
         /* XXX: need to check if we can store
          * pointers in 'local' which may get freed
@@ -137,13 +121,15 @@ int upcall_reaper_thread_init (xlator_t *this);
 gf_boolean_t is_upcall_enabled(xlator_t *this);
 
 /* Cache invalidation specific */
-void upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client,
-                              inode_t *inode, uint32_t flags);
+void upcall_cache_invalidate (call_frame_t *frame, xlator_t *this,
+                              client_t *client, inode_t *inode,
+                              uint32_t flags, struct iatt *stbuf,
+                              struct iatt *p_stbuf,
+                              struct iatt *oldp_stbuf);
 void upcall_client_cache_invalidate (xlator_t *xl, uuid_t gfid,
                                      upcall_client_t *up_client_entry,
-                                     uint32_t flags);
-void upcall_cache_invalidate_dir (call_frame_t *frame, xlator_t *this,
-                                  client_t *client, inode_t *inode,
-                                  uint32_t flags);
+                                     uint32_t flags, struct iatt *stbuf,
+                                     struct iatt *p_stbuf,
+                                     struct iatt *oldp_stbuf);
 
 #endif /* __UPCALL_H__ */