Upcall: New xlator to store various states and send cbk events

Framework on the server-side, to handle certain state of the files
accessed and send notifications to the clients connected.

A generic and extensible framework, used to maintain states in
the glusterfsd process for each of the files accessed
(including the clients info doing the fops) and send
notifications to the respective glusterfs clients incase of
any change in that state.

This patch handles "Inode Update/Invalidation" upcall event.

Feature page:
        URL: http://www.gluster.org/community/documentation/index.php/Features/Upcall-infrastructure

Below link has a writeup which explains the code changes done -
        URL: https://soumyakoduri.wordpress.com/2015/02/25/glusterfs-understanding-upcall-infrastructure-and-cache-invalidation-support/

Change-Id: Ie3d724be9a3419fcf18901a753e8ec2df2ac802f
BUG: 1200262
Signed-off-by: Soumya Koduri <skoduri@redhat.com>
Reviewed-on: http://review.gluster.org/9535
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>

12 files changed, 1791 insertions, 7 deletions

diff --git a/xlators/features/Makefile.am b/xlators/features/Makefile.am
index dcb3cc8e5a7..096f0e04bb2 100644
--- a/xlators/features/Makefile.am
+++ b/xlators/features/Makefile.am
@@ -1,4 +1,5 @@
 SUBDIRS = locks quota read-only mac-compat quiesce marker index barrier \
-          protect compress changelog gfid-access $(GLUPY_SUBDIR) qemu-block snapview-client snapview-server trash # path-converter # filter
+          protect compress changelog gfid-access $(GLUPY_SUBDIR) qemu-block \
+          upcall snapview-client snapview-server trash # path-converter # filter
 
 CLEANFILES =
diff --git a/xlators/features/upcall/Makefile.am b/xlators/features/upcall/Makefile.am
new file mode 100644
index 00000000000..a985f42a877
--- /dev/null
+++ b/xlators/features/upcall/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/upcall/src/Makefile.am b/xlators/features/upcall/src/Makefile.am
new file mode 100644
index 00000000000..2ac09551476
--- /dev/null
+++ b/xlators/features/upcall/src/Makefile.am
@@ -0,0 +1,21 @@
+xlator_LTLIBRARIES = upcall.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+upcall_la_LDFLAGS = -module -avoid-version
+
+upcall_la_SOURCES = upcall.c upcall-internal.c
+
+upcall_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+        $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+        $(top_builddir)/rpc/xdr/src/libgfxdr.la
+
+noinst_HEADERS = upcall.h upcall-mem-types.h upcall-messages.h \
+	upcall-cache-invalidation.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+        -I$(top_srcdir)/rpc/rpc-lib/src \
+        -I$(top_srcdir)/rpc/xdr/src
+
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/upcall/src/upcall-cache-invalidation.h b/xlators/features/upcall/src/upcall-cache-invalidation.h
new file mode 100644
index 00000000000..758ddf1dae8
--- /dev/null
+++ b/xlators/features/upcall/src/upcall-cache-invalidation.h
@@ -0,0 +1,79 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __UPCALL_CACHE_INVALIDATION_H__
+#define __UPCALL_CACHE_INVALIDATION_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+/* TODO: Below macros have to be replaced with
+ * xlator options - Bug1200271 */
+#define ON_CACHE_INVALIDATION 0 /* disable by default */
+
+/* The time period for which a client will be notified of cache_invalidation
+ * events post its last access */
+#define CACHE_INVALIDATION_PERIOD 60
+
+/* Flags sent for cache_invalidation */
+#define UP_NLINK   0x00000001   /* update nlink */
+#define UP_MODE    0x00000002   /* update mode and ctime */
+#define UP_OWN     0x00000004   /* update mode,uid,gid and ctime */
+#define UP_SIZE    0x00000008   /* update fsize */
+#define UP_TIMES   0x00000010   /* update all times */
+#define UP_ATIME   0x00000020   /* update atime only */
+#define UP_PERM    0x00000040   /* update fields needed for
+                                   permission checking */
+#define UP_RENAME  0x00000080   /* this is a rename op -
+                                   delete the cache entry */
+#define UP_FORGET  0x00000100   /* inode_forget on server side -
+                                   invalidate the cache entry */
+
+/* for fops - open, read, lk, */
+#define UP_IDEMPOTENT_FLAGS     (UP_ATIME)
+
+/* for fop - write, truncate */
+#define UP_WRITE_FLAGS          (UP_SIZE | UP_TIMES)
+
+/* for fop - setattr */
+#define UP_ATTR_FLAGS           (UP_SIZE | UP_TIMES | UP_OWN |        \
+                                 UP_MODE | UP_PERM)
+/* for fop - rename */
+#define UP_RENAME_FLAGS         (UP_RENAME)
+
+/* to invalidate parent directory entries for fops -rename, unlink,
+ * rmdir, mkdir, create */
+#define UP_PARENT_DENTRY_FLAGS  (UP_TIMES)
+
+/* for fop - unlink, link, rmdir, mkdir */
+#define UP_NLINK_FLAGS          (UP_NLINK | UP_TIMES)
+
+#define CACHE_INVALIDATE(frame, this, client, inode, p_flags) do {      \
+ if (ON_CACHE_INVALIDATION) {                                             \
+        (void)upcall_cache_invalidate (frame, this, client, inode, p_flags);  \
+ }                                                                      \
+} while (0)
+
+#define CACHE_INVALIDATE_DIR(frame, this, client, inode_p, p_flags) do {      \
+ if (ON_CACHE_INVALIDATION) {                                             \
+        dentry_t *dentry;                                               \
+        dentry_t *dentry_tmp;                                           \
+        list_for_each_entry_safe (dentry, dentry_tmp,                   \
+                                  &inode_p->dentry_list,                \
+                                  inode_list) {                         \
+                (void)upcall_cache_invalidate (frame, this, client,     \
+                                               dentry->inode, p_flags); \
+        }                                                               \
+ }                                                                      \
+} while (0)
+
+#endif /* __UPCALL_CACHE_INVALIDATION_H__ */
diff --git a/xlators/features/upcall/src/upcall-internal.c b/xlators/features/upcall/src/upcall-internal.c
new file mode 100644
index 00000000000..26473e2a7bd
--- /dev/null
+++ b/xlators/features/upcall/src/upcall-internal.c
@@ -0,0 +1,424 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <limits.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "compat.h"
+#include "xlator.h"
+#include "inode.h"
+#include "logging.h"
+#include "common-utils.h"
+
+#include "statedump.h"
+#include "syncop.h"
+
+#include "upcall.h"
+#include "upcall-mem-types.h"
+#include "glusterfs3-xdr.h"
+#include "protocol-common.h"
+#include "defaults.h"
+
+/*
+ * Allocate and add a new client entry to the given upcall entry
+ */
+upcall_client_t*
+add_upcall_client (call_frame_t *frame, uuid_t gfid,
+                   client_t *client,
+                   upcall_inode_ctx_t *up_inode_ctx)
+{
+        upcall_client_t *up_client_entry = NULL;
+
+        pthread_mutex_lock (&up_inode_ctx->client_list_lock);
+        {
+                up_client_entry = __add_upcall_client (frame,
+                                                       gfid,
+                                                       client,
+                                                       up_inode_ctx);
+        }
+        pthread_mutex_unlock (&up_inode_ctx->client_list_lock);
+
+        return up_client_entry;
+}
+
+upcall_client_t*
+__add_upcall_client (call_frame_t *frame, uuid_t gfid,
+                     client_t *client,
+                     upcall_inode_ctx_t *up_inode_ctx)
+{
+        upcall_client_t *up_client_entry = NULL;
+
+        up_client_entry = GF_CALLOC (1, sizeof(*up_client_entry),
+                                     gf_upcall_mt_upcall_client_entry_t);
+        if (!up_client_entry) {
+                gf_msg ("upcall", GF_LOG_WARNING, 0,
+                        UPCALL_MSG_NO_MEMORY,
+                        "Memory allocation failed");
+                return NULL;
+        }
+        INIT_LIST_HEAD (&up_client_entry->client_list);
+        up_client_entry->client_uid = gf_strdup(client->client_uid);
+        up_client_entry->access_time = time(NULL);
+        up_client_entry->expire_time_attr = CACHE_INVALIDATION_PERIOD;
+
+        list_add_tail (&up_client_entry->client_list,
+                       &up_inode_ctx->client_list);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "upcall_entry_t client added - %s",
+                up_client_entry->client_uid);
+
+        return up_client_entry;
+}
+
+/*
+ * Given gfid and client->uid, retrieve the corresponding upcall client entry.
+ * If none found, create a new entry.
+ */
+upcall_client_t*
+__get_upcall_client (call_frame_t *frame, uuid_t gfid, client_t *client,
+                     upcall_inode_ctx_t *up_inode_ctx)
+{
+        upcall_client_t *up_client_entry = NULL;
+        upcall_client_t *up_client       = NULL;
+        upcall_client_t *tmp             = NULL;
+        gf_boolean_t    found_client     = _gf_false;
+
+        list_for_each_entry_safe (up_client_entry, tmp,
+                                  &up_inode_ctx->client_list,
+                                  client_list) {
+                if (strcmp(client->client_uid,
+                           up_client_entry->client_uid) == 0) {
+                        /* found client entry. Update the access_time */
+                        up_client_entry->access_time = time(NULL);
+                        found_client = _gf_true;
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "upcall_entry_t client found - %s",
+                                up_client_entry->client_uid);
+                        break;
+                }
+        }
+
+        if (!found_client) { /* create one */
+                up_client_entry = __add_upcall_client (frame, gfid, client,
+                                                       up_inode_ctx);
+        }
+
+        return up_client_entry;
+}
+
+int
+__upcall_inode_ctx_set (inode_t *inode, xlator_t *this)
+{
+        upcall_inode_ctx_t *inode_ctx   = NULL;
+        int                ret          = -1;
+        uint64_t           ctx          = 0;
+
+        ret = __inode_ctx_get (inode, this, &ctx);
+
+        if (!ret)
+                goto out;
+
+        inode_ctx = GF_CALLOC (1, sizeof (upcall_inode_ctx_t),
+                               gf_upcall_mt_upcall_inode_ctx_t);
+
+        if (!inode_ctx) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        pthread_mutex_init (&inode_ctx->client_list_lock, NULL);
+        INIT_LIST_HEAD (&inode_ctx->client_list);
+
+        ret = __inode_ctx_set (inode, this, (uint64_t *) inode_ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to set inode ctx (%p)", inode);
+out:
+        return ret;
+}
+
+upcall_inode_ctx_t *
+__upcall_inode_ctx_get (inode_t *inode, xlator_t *this)
+{
+        upcall_inode_ctx_t *inode_ctx   = NULL;
+        uint64_t           ctx          = 0;
+        int                ret          = 0;
+
+        ret = __inode_ctx_get (inode, this, &ctx);
+
+        if (ret < 0) {
+                ret = __upcall_inode_ctx_set (inode, this);
+                if (ret < 0)
+                        goto out;
+
+                ret = __inode_ctx_get (inode, this, &ctx);
+                if (ret < 0)
+                        goto out;
+        }
+
+        inode_ctx = (upcall_inode_ctx_t *)(long) ctx;
+
+out:
+        return inode_ctx;
+}
+
+upcall_inode_ctx_t *
+upcall_inode_ctx_get (inode_t *inode, xlator_t *this)
+{
+        upcall_inode_ctx_t *inode_ctx = NULL;
+
+        LOCK (&inode->lock);
+        {
+                inode_ctx = __upcall_inode_ctx_get (inode, this);
+        }
+        UNLOCK (&inode->lock);
+
+        return inode_ctx;
+}
+
+int
+__upcall_cleanup_client_entry (upcall_client_t *up_client)
+{
+        list_del_init (&up_client->client_list);
+
+        GF_FREE (up_client->client_uid);
+        GF_FREE (up_client);
+
+        return 0;
+}
+
+/*
+ * Free Upcall inode_ctx client list
+ */
+int
+__upcall_cleanup_inode_ctx_client_list (upcall_inode_ctx_t *inode_ctx)
+{
+        upcall_client_t *up_client        = NULL;
+        upcall_client_t *tmp              = NULL;
+
+        list_for_each_entry_safe (up_client, tmp,
+                                  &inode_ctx->client_list,
+                                  client_list) {
+                __upcall_cleanup_client_entry (up_client);
+        }
+
+        return 0;
+}
+
+/*
+ * Free upcall_inode_ctx
+ */
+int
+upcall_cleanup_inode_ctx (xlator_t *this, inode_t *inode)
+{
+        uint64_t           ctx          = 0;
+        upcall_inode_ctx_t *inode_ctx   = NULL;
+        int                ret          = 0;
+
+        ret = inode_ctx_get (inode, this, &ctx);
+
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_TRACE,
+                        "Failed to get upcall_inode_ctx (%p)",
+                        inode);
+                goto out;
+        }
+
+        /* Invalidate all the upcall cache entries */
+        upcall_cache_forget (this, inode, inode_ctx);
+
+        /* Set inode context to NULL */
+        ret = __inode_ctx_set (inode, this, NULL);
+
+        if (!ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "_inode_ctx_set to NULL failed (%p)",
+                        inode);
+        }
+        inode_ctx = (upcall_inode_ctx_t *)(long) ctx;
+
+        if (inode_ctx) {
+                /* do we really need lock? */
+                pthread_mutex_lock (&inode_ctx->client_list_lock);
+                {
+                        if (!list_empty (&inode_ctx->client_list)) {
+                                __upcall_cleanup_inode_ctx_client_list (inode_ctx);
+                        }
+                }
+                pthread_mutex_unlock (&inode_ctx->client_list_lock);
+
+                pthread_mutex_destroy (&inode_ctx->client_list_lock);
+
+                GF_FREE (inode_ctx);
+        }
+
+out:
+        return ret;
+}
+
+/*
+ * Given a gfid, client, first fetch upcall_entry_t based on gfid.
+ * Later traverse through the client list of that upcall entry. If this client
+ * is not present in the list, create one client entry with this client info.
+ * Also check if there are other clients which need to be notified of this
+ * op. If yes send notify calls to them.
+ *
+ * Since sending notifications for cache_invalidation is a best effort,
+ * any errors during the process are logged and ignored.
+ */
+void
+upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client,
+                         inode_t *inode, uint32_t flags)
+{
+        upcall_client_t *up_client       = NULL;
+        upcall_client_t *up_client_entry = NULL;
+        upcall_client_t *tmp             = NULL;
+        upcall_inode_ctx_t *up_inode_ctx = NULL;
+        gf_boolean_t     found           = _gf_false;
+
+        up_inode_ctx = ((upcall_local_t *)frame->local)->upcall_inode_ctx;
+
+        if (!up_inode_ctx)
+                up_inode_ctx = upcall_inode_ctx_get (inode, this);
+
+        if (!up_inode_ctx) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "upcall_inode_ctx_get failed (%p)",
+                        inode);
+                return;
+        }
+
+        pthread_mutex_lock (&up_inode_ctx->client_list_lock);
+        {
+                list_for_each_entry_safe (up_client_entry, tmp,
+                                          &up_inode_ctx->client_list,
+                                          client_list) {
+
+                        if (!strcmp(client->client_uid,
+                                   up_client_entry->client_uid)) {
+                                up_client_entry->access_time = time(NULL);
+                                found = _gf_true;
+                        }
+
+                        /*
+                         * Ignore sending notifications in case of only UP_ATIME
+                         */
+                        if (!(flags & ~(UP_ATIME))) {
+                                if (found)
+                                        break;
+                                else /* we still need to find current client entry*/
+                                        continue;
+                        }
+
+                        /* any other client */
+
+                        /* XXX: Send notifications asynchrounously
+                         * instead of in the I/O path - BZ 1200264
+                         *  Also if the file is frequently accessed, set
+                         *  expire_time_attr to 0.
+                         */
+                        upcall_client_cache_invalidate(this,
+                                                       inode->gfid,
+                                                       up_client_entry,
+                                                       flags);
+                }
+
+                if (!found) {
+                        up_client_entry = __add_upcall_client (frame,
+                                                               inode->gfid,
+                                                               client,
+                                                               up_inode_ctx);
+                }
+        }
+        pthread_mutex_unlock (&up_inode_ctx->client_list_lock);
+}
+
+/*
+ * If the upcall_client_t has recently accessed the file (i.e, within
+ * CACHE_INVALIDATION_PERIOD), send a upcall notification.
+ */
+void
+upcall_client_cache_invalidate (xlator_t *this, uuid_t gfid,
+                                upcall_client_t *up_client_entry,
+                                uint32_t flags)
+{
+        notify_event_data_t n_event_data;
+        time_t t_expired = time(NULL) - up_client_entry->access_time;
+
+        if (t_expired < CACHE_INVALIDATION_PERIOD) {
+                /* Send notify call */
+                uuid_copy(n_event_data.gfid, gfid);
+                n_event_data.client_entry = up_client_entry;
+                n_event_data.event_type = CACHE_INVALIDATION;
+                n_event_data.invalidate_flags = flags;
+
+                /* Need to send inode flags */
+                this->notify (this, GF_EVENT_UPCALL, &n_event_data);
+
+                gf_log (THIS->name, GF_LOG_TRACE,
+                        "Cache invalidation notification sent to %s",
+                        up_client_entry->client_uid);
+
+        } else {
+                if (t_expired > (2*CACHE_INVALIDATION_PERIOD)) {
+                        /* Cleanup the entry */
+                        __upcall_cleanup_client_entry (up_client_entry);
+                }
+
+                gf_log (THIS->name, GF_LOG_TRACE,
+                        "Cache invalidation notification NOT sent to %s",
+                        up_client_entry->client_uid);
+        }
+}
+
+/*
+ * This is called during upcall_inode_ctx cleanup incase of 'inode_forget'.
+ * Send "UP_FORGET" to all the clients so that they invalidate their cache
+ * entry and do a fresh lookup next time when any I/O comes in.
+ */
+void
+upcall_cache_forget (xlator_t *this, inode_t *inode, upcall_inode_ctx_t *up_inode_ctx)
+{
+        upcall_client_t *up_client       = NULL;
+        upcall_client_t *up_client_entry = NULL;
+        upcall_client_t *tmp             = NULL;
+        uint32_t        flags            = 0;
+
+        if (!up_inode_ctx) {
+                return;
+        }
+
+        pthread_mutex_lock (&up_inode_ctx->client_list_lock);
+        {
+                list_for_each_entry_safe (up_client_entry, tmp,
+                                          &up_inode_ctx->client_list,
+                                          client_list) {
+                        flags = UP_FORGET;
+
+                        /* Set the access time to time(NULL)
+                         * to send notify */
+                        up_client_entry->access_time = time(NULL);
+
+                        upcall_client_cache_invalidate(this,
+                                                       inode->gfid,
+                                                       up_client_entry,
+                                                       flags);
+                }
+
+        }
+        pthread_mutex_unlock (&up_inode_ctx->client_list_lock);
+}
diff --git a/xlators/features/upcall/src/upcall-mem-types.h b/xlators/features/upcall/src/upcall-mem-types.h
new file mode 100644
index 00000000000..55793ec65ca
--- /dev/null
+++ b/xlators/features/upcall/src/upcall-mem-types.h
@@ -0,0 +1,24 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __UPCALL_MEM_TYPES_H__
+#define __UPCALL_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_upcall_mem_types_ {
+        gf_upcall_mt_conf_t = gf_common_mt_end + 1,
+        gf_upcall_mt_private_t,
+        gf_upcall_mt_upcall_inode_ctx_t,
+        gf_upcall_mt_upcall_client_entry_t,
+        gf_upcall_mt_end
+};
+#endif
+
diff --git a/xlators/features/upcall/src/upcall-messages.h b/xlators/features/upcall/src/upcall-messages.h
new file mode 100644
index 00000000000..c8483f16528
--- /dev/null
+++ b/xlators/features/upcall/src/upcall-messages.h
@@ -0,0 +1,62 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _UPCALL_MESSAGES_H_
+#define _UPCALL_MESSAGES_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glfs-message-id.h"
+
+/*! \file upcall-messages.h
+ *  \brief UPCALL log-message IDs and their descriptions.
+ */
+
+/* NOTE: Rules for message additions
+ * 1) Each instance of a message is _better_ left with a unique message ID, even
+ * if the message format is the same. Reasoning is that, if the message
+ * format needs to change in one instance, the other instances are not
+ * impacted or the new change does not change the ID of the instance being
+ * modified.
+ * 2) Addition of a message,
+ * - Should increment the GLFS_NUM_MESSAGES
+ * - Append to the list of messages defined, towards the end
+ * - Retain macro naming as glfs_msg_X (for redability across developers)
+ * NOTE: Rules for message format modifications
+ * 3) Check across the code if the message ID macro in question is reused
+ * anywhere. If reused then then the modifications should ensure correctness
+ * everywhere, or needs a new message ID as (1) above was not adhered to. If
+ * not used anywhere, proceed with the required modification.
+ * NOTE: Rules for message deletion
+ * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
+ * anywhere, then can be deleted, but will leave a hole by design, as
+ * addition rules specify modification to the end of the list and not filling
+ * holes.
+ */
+
+#define GLFS_COMP_BASE_UPCALL GLFS_MSGID_COMP_UPCALL
+#define GLFS_NUM_MESSAGES 1
+#define GLFS_MSGID_END (GLFS_COMP_BASE_UPCALL + GLFS_NUM_MESSAGES + 1)
+
+#define glfs_msg_start_x GLFS_COMP_BASE_UPCALL, "Invalid: Start of messages"
+
+/*!
+ * @messageid 110001
+ * @diagnosis Out of Memory
+ * @recommendedaction None
+ */
+#define UPCALL_MSG_NO_MEMORY             (GLFS_COMP_BASE_UPCALL + 1)
+
+#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+
+#endif /* !_UPCALL_MESSAGES_H_ */
diff --git a/xlators/features/upcall/src/upcall.c b/xlators/features/upcall/src/upcall.c
new file mode 100644
index 00000000000..b7f2e975bba
--- /dev/null
+++ b/xlators/features/upcall/src/upcall.c
@@ -0,0 +1,956 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <pthread.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "compat.h"
+#include "xlator.h"
+#include "inode.h"
+#include "logging.h"
+#include "common-utils.h"
+
+#include "statedump.h"
+#include "syncop.h"
+
+#include "upcall.h"
+#include "upcall-mem-types.h"
+#include "glusterfs3-xdr.h"
+#include "protocol-common.h"
+#include "defaults.h"
+
+int
+up_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        client_t         *client        = NULL;
+        uint32_t         flags          = 0;
+        upcall_local_t   *local         = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        client = frame->root->client;
+        local = frame->local;
+
+        if ((op_ret < 0) || !local) {
+                goto out;
+        }
+        flags = UP_IDEMPOTENT_FLAGS;
+        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+
+out:
+        UPCALL_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
+
+        return 0;
+}
+
+
+int
+up_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
+{
+        int32_t          op_errno        = -1;
+        upcall_local_t   *local          = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        local = upcall_local_init (frame, this, fd->inode);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+out:
+        STACK_WIND (frame, up_open_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
+                    loc, flags, fd, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        UPCALL_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+int
+up_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        client_t         *client        = NULL;
+        uint32_t         flags          = 0;
+        upcall_local_t   *local         = NULL;
+
+        client = frame->root->client;
+        local = frame->local;
+
+        if ((op_ret < 0) || !local) {
+                goto out;
+        }
+        flags = UP_WRITE_FLAGS;
+        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+
+out:
+        UPCALL_STACK_UNWIND (writev, frame, op_ret, op_errno,
+                             prebuf, postbuf, xdata);
+
+        return 0;
+}
+
+
+int
+up_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+            struct iovec *vector, int count, off_t off, uint32_t flags,
+            struct iobref *iobref, dict_t *xdata)
+{
+        int32_t          op_errno        = -1;
+        upcall_local_t   *local          = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        local = upcall_local_init (frame, this, fd->inode);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+out:
+        STACK_WIND (frame, up_writev_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
+                    fd, vector, count, off, flags, iobref, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        UPCALL_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+up_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int op_ret, int op_errno,
+               struct iovec *vector, int count, struct iatt *stbuf,
+               struct iobref *iobref, dict_t *xdata)
+{
+        client_t         *client        = NULL;
+        uint32_t         flags          = 0;
+        upcall_local_t   *local         = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        client = frame->root->client;
+        local = frame->local;
+
+        if ((op_ret < 0) || !local) {
+                goto out;
+        }
+        flags = UP_IDEMPOTENT_FLAGS;
+        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+
+out:
+        UPCALL_STACK_UNWIND (readv, frame, op_ret, op_errno, vector,
+                             count, stbuf, iobref, xdata);
+
+        return 0;
+}
+
+int
+up_readv (call_frame_t *frame, xlator_t *this,
+          fd_t *fd, size_t size, off_t offset,
+          uint32_t flags, dict_t *xdata)
+{
+        int32_t          op_errno        = -1;
+        upcall_local_t   *local          = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        local = upcall_local_init (frame, this, fd->inode);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+out:
+        STACK_WIND (frame, up_readv_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        UPCALL_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0,
+                             NULL, NULL, NULL);
+
+        return 0;
+}
+
+int32_t
+up_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                dict_t *xdata)
+{
+        client_t         *client        = NULL;
+        uint32_t         flags          = 0;
+        upcall_local_t   *local         = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        client = frame->root->client;
+        local = frame->local;
+
+        if ((op_ret < 0) || !local) {
+                goto out;
+        }
+        flags = UP_IDEMPOTENT_FLAGS;
+        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+
+out:
+        UPCALL_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata);
+
+        return 0;
+}
+
+int
+up_lk (call_frame_t *frame, xlator_t *this,
+       fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+        int32_t          op_errno        = -1;
+        upcall_local_t   *local          = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        local = upcall_local_init (frame, this, fd->inode);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+out:
+        STACK_WIND (frame, up_lk_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lk,
+                    fd, cmd, flock, xdata);
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        UPCALL_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+int
+up_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        client_t         *client        = NULL;
+        uint32_t         flags          = 0;
+        upcall_local_t   *local         = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        client = frame->root->client;
+        local = frame->local;
+
+        if ((op_ret < 0) || !local) {
+                goto out;
+        }
+        flags = UP_WRITE_FLAGS;
+        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+
+out:
+        UPCALL_STACK_UNWIND (truncate, frame, op_ret, op_errno,
+                             prebuf, postbuf, xdata);
+
+        return 0;
+}
+
+int
+up_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
+{
+        int32_t          op_errno        = -1;
+        upcall_local_t   *local          = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        local = upcall_local_init (frame, this, loc->inode);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+out:
+        STACK_WIND (frame, up_truncate_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
+                    loc, offset, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        UPCALL_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+up_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int op_ret, int op_errno, struct iatt *statpre,
+                 struct iatt *statpost, dict_t *xdata)
+{
+        client_t         *client        = NULL;
+        uint32_t         flags          = 0;
+        upcall_local_t   *local         = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        client = frame->root->client;
+        local = frame->local;
+
+        if ((op_ret < 0) || !local) {
+                goto out;
+        }
+        /* XXX: setattr -> UP_SIZE or UP_OWN or UP_MODE or UP_TIMES
+         * or INODE_UPDATE (or UP_PERM esp incase of ACLs -> INODE_INVALIDATE)
+         * Need to check what attr is changed and accordingly pass UP_FLAGS.
+         * Bug1200271.
+         */
+        flags = UP_ATTR_FLAGS;
+        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+
+out:
+        UPCALL_STACK_UNWIND (setattr, frame, op_ret, op_errno,
+                             statpre, statpost, xdata);
+
+        return 0;
+}
+
+int
+up_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        int32_t          op_errno        = -1;
+        upcall_local_t   *local          = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        local = upcall_local_init (frame, this, loc->inode);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+out:
+        STACK_WIND (frame, up_setattr_cbk,
+                           FIRST_CHILD(this),
+                           FIRST_CHILD(this)->fops->setattr,
+                            loc, stbuf, valid, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        UPCALL_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+up_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
+{
+        client_t         *client        = NULL;
+        uint32_t         flags          = 0;
+        upcall_local_t   *local         = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        client = frame->root->client;
+        local = frame->local;
+
+        if ((op_ret < 0) || !local) {
+                goto out;
+        }
+        flags = UP_RENAME_FLAGS;
+        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+
+        /* Need to invalidate old and new parent entries as well */
+        flags = UP_PARENT_DENTRY_FLAGS;
+        CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags);
+
+        /* XXX: notify oldparent as well */
+/*        if (uuid_compare (preoldparent->ia_gfid, prenewparent->ia_gfid))
+                CACHE_INVALIDATE (frame, this, client, prenewparent->ia_gfid, flags);*/
+
+out:
+        UPCALL_STACK_UNWIND (rename, frame, op_ret, op_errno,
+                             stbuf, preoldparent, postoldparent,
+                             prenewparent, postnewparent, xdata);
+
+        return 0;
+}
+
+int
+up_rename (call_frame_t *frame, xlator_t *this,
+          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        int32_t          op_errno        = -1;
+        upcall_local_t   *local          = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        local = upcall_local_init (frame, this, oldloc->inode);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+out:
+        STACK_WIND (frame, up_rename_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        UPCALL_STACK_UNWIND (rename, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+up_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        client_t         *client        = NULL;
+        uint32_t         flags          = 0;
+        upcall_local_t   *local         = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        client = frame->root->client;
+        local = frame->local;
+
+        if ((op_ret < 0) || !local) {
+                goto out;
+        }
+        flags = UP_NLINK_FLAGS;
+        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+
+        flags = UP_PARENT_DENTRY_FLAGS;
+        /* invalidate parent's entry too */
+        CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags);
+
+out:
+        UPCALL_STACK_UNWIND (unlink, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+
+        return 0;
+}
+
+int
+up_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
+{
+        int32_t          op_errno        = -1;
+        upcall_local_t   *local          = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        local = upcall_local_init (frame, this, loc->inode);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+out:
+        STACK_WIND (frame, up_unlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        UPCALL_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+up_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        client_t         *client        = NULL;
+        uint32_t         flags          = 0;
+        upcall_local_t   *local         = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        client = frame->root->client;
+        local = frame->local;
+
+        if ((op_ret < 0) || !local) {
+                goto out;
+        }
+        flags = UP_NLINK_FLAGS;
+        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+
+        /* do we need to update parent as well?? */
+out:
+        UPCALL_STACK_UNWIND (link, frame, op_ret, op_errno,
+                             inode, stbuf, preparent, postparent, xdata);
+
+        return 0;
+}
+
+int
+up_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+         loc_t *newloc, dict_t *xdata)
+{
+        int32_t          op_errno        = -1;
+        upcall_local_t   *local          = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        local = upcall_local_init (frame, this, oldloc->inode);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+out:
+        STACK_WIND (frame, up_link_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        UPCALL_STACK_UNWIND (link, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+up_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        client_t         *client        = NULL;
+        uint32_t         flags          = 0;
+        upcall_local_t   *local         = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        client = frame->root->client;
+        local = frame->local;
+
+        if ((op_ret < 0) || !local) {
+                goto out;
+        }
+        flags = UP_NLINK_FLAGS;
+        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+
+        /* invalidate parent's entry too */
+        flags = UP_PARENT_DENTRY_FLAGS;
+        CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags);
+
+out:
+        UPCALL_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+
+        return 0;
+}
+
+int
+up_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
+{
+        int32_t          op_errno        = -1;
+        upcall_local_t   *local          = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        local = upcall_local_init (frame, this, loc->inode);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+out:
+        STACK_WIND (frame, up_rmdir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
+                    loc, flags, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        UPCALL_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+up_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, inode_t *inode,
+                struct iatt *stbuf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        client_t         *client        = NULL;
+        uint32_t         flags          = 0;
+        upcall_local_t   *local         = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        client = frame->root->client;
+        local = frame->local;
+
+        if ((op_ret < 0) || !local) {
+                goto out;
+        }
+        flags = UP_NLINK_FLAGS;
+        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+
+        /* invalidate parent's entry too */
+        flags = UP_PARENT_DENTRY_FLAGS;
+        CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags);
+
+out:
+        UPCALL_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
+                             inode, stbuf, preparent, postparent, xdata);
+
+        return 0;
+}
+
+int
+up_mkdir (call_frame_t *frame, xlator_t *this,
+          loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
+{
+        int32_t          op_errno        = -1;
+        upcall_local_t   *local          = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        local = upcall_local_init (frame, this, loc->inode);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+out:
+        STACK_WIND (frame, up_mkdir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                    loc, mode, umask, params);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        UPCALL_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+up_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, fd_t *fd, inode_t *inode,
+                struct iatt *stbuf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        client_t         *client        = NULL;
+        uint32_t         flags          = 0;
+        upcall_local_t   *local         = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        client = frame->root->client;
+        local = frame->local;
+
+        if ((op_ret < 0) || !local) {
+                goto out;
+        }
+
+        /* As its a new file create, no need of sending notification */
+        /* However invalidate parent's entry */
+        flags = UP_PARENT_DENTRY_FLAGS;
+        CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags);
+
+out:
+        UPCALL_STACK_UNWIND (create, frame, op_ret, op_errno, fd,
+                             inode, stbuf, preparent, postparent, xdata);
+
+        return 0;
+}
+
+int
+up_create (call_frame_t *frame, xlator_t *this,
+          loc_t *loc, int32_t flags, mode_t mode,
+          mode_t umask, fd_t *fd, dict_t *params)
+{
+        int32_t          op_errno        = -1;
+        upcall_local_t   *local          = NULL;
+
+        EXIT_IF_UPCALL_OFF (out);
+
+        local = upcall_local_init (frame, this, loc->inode);
+
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+out:
+        STACK_WIND (frame, up_create_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, params);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        UPCALL_STACK_UNWIND (create, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_upcall_mt_end + 1);
+
+        if (ret != 0) {
+                gf_msg ("upcall", GF_LOG_WARNING, 0,
+                        UPCALL_MSG_NO_MEMORY,
+                        "Memory allocation failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+void
+upcall_local_wipe (xlator_t *this, upcall_local_t *local)
+{
+        if (local) {
+                inode_unref (local->inode);
+                mem_put (local);
+        }
+}
+
+upcall_local_t *
+upcall_local_init (call_frame_t *frame, xlator_t *this, inode_t *inode)
+{
+        upcall_local_t *local = NULL;
+
+        local = mem_get0 (THIS->local_pool);
+
+        if (!local)
+                goto out;
+
+        local->inode = inode_ref (inode);
+
+        /* Shall we get inode_ctx and store it here itself? */
+        local->upcall_inode_ctx = upcall_inode_ctx_get (inode, this);
+
+        frame->local = local;
+
+out:
+        return local;
+}
+
+int
+init (xlator_t *this)
+{
+        int                       ret        = -1;
+        upcalls_private_t        *priv       = NULL;
+
+        priv = GF_CALLOC (1, sizeof (*priv),
+                          gf_upcall_mt_private_t);
+        if (!priv) {
+                gf_msg ("upcall", GF_LOG_WARNING, 0,
+                        UPCALL_MSG_NO_MEMORY,
+                        "Memory allocation failed");
+                goto out;
+        }
+
+        this->private = priv;
+        this->local_pool = mem_pool_new (upcall_local_t, 512);
+        ret = 0;
+
+out:
+        if (ret) {
+                GF_FREE (priv);
+        }
+
+        return ret;
+}
+
+int
+fini (xlator_t *this)
+{
+        upcalls_private_t *priv = NULL;
+
+        priv = this->private;
+        if (!priv) {
+                return 0;
+        }
+        this->private = NULL;
+        GF_FREE (priv);
+
+        return 0;
+}
+
+int
+upcall_forget (xlator_t *this, inode_t *inode)
+{
+        upcall_cleanup_inode_ctx (this, inode);
+        return 0;
+}
+
+int
+upcall_release (xlator_t *this, fd_t *fd)
+{
+        return 0;
+}
+
+int
+notify (xlator_t *this, int32_t event, void *data, ...)
+{
+        int                 ret              = -1;
+        int32_t             val              = 0;
+        notify_event_data_t *notify_event    = NULL;
+        struct gf_upcall    up_req           = {0,};
+        upcall_client_t     *up_client_entry = NULL;
+
+        switch (event) {
+        case GF_EVENT_UPCALL:
+        {
+                gf_log (this->name, GF_LOG_DEBUG, "Upcall Notify event = %d",
+                        event);
+
+                notify_event = (notify_event_data_t *) data;
+                up_client_entry = notify_event->client_entry;
+
+                if (!up_client_entry) {
+                        goto out;
+                }
+
+                up_req.client_uid = up_client_entry->client_uid;
+
+                memcpy (up_req.gfid, notify_event->gfid, 16);
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Sending notify to the client- %s, gfid - %s",
+                        up_client_entry->client_uid, up_req.gfid);
+
+                switch (notify_event->event_type) {
+                case CACHE_INVALIDATION:
+                        GF_ASSERT (notify_event->extra);
+                        up_req.flags = notify_event->invalidate_flags;
+                        up_req.expire_time_attr = up_client_entry->expire_time_attr;
+                        break;
+                default:
+                        goto out;
+                }
+
+                up_req.event_type = notify_event->event_type;
+
+                ret = default_notify (this, event, &up_req);
+
+                /*
+                 * notify may fail as the client could have been
+                 * dis(re)connected. Cleanup the client entry.
+                 */
+                if (ret < 0)
+                        __upcall_cleanup_client_entry (up_client_entry);
+        }
+        break;
+        default:
+                default_notify (this, event, data);
+        break;
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+struct xlator_fops fops = {
+        .open        = up_open,
+        .readv       = up_readv,
+        .writev      = up_writev,
+        .truncate    = up_truncate,
+        .lk          = up_lk,
+        .setattr     = up_setattr,
+        .rename      = up_rename,
+        .unlink      = up_unlink, /* invalidate both file and parent dir */
+        .rmdir       = up_rmdir, /* same as above */
+        .link        = up_link, /* invalidate both file and parent dir */
+        .create      = up_create, /* update only direntry */
+        .mkdir       = up_mkdir, /* update only dirent */
+#ifdef WIP
+        .ftruncate   = up_ftruncate, /* reqd? */
+        .getattr     = up_getattr, /* ?? */
+        .getxattr    = up_getxattr, /* ?? */
+        .access      = up_access,
+        .lookup      = up_lookup,
+        .symlink     = up_symlink, /* invalidate both file and parent dir maybe */
+        .readlink    = up_readlink, /* Needed? readlink same as read? */
+        .readdirp    = up_readdirp,
+        .readdir     = up_readdir,
+/*  other fops to be considered - Bug1200271
+ *   lookup, stat, opendir, readdir, readdirp, readlink, mknod, statfs, flush,
+ *   fsync, mknod, fsyncdir, setxattr, removexattr, rchecksum, fallocate, discard,
+ *   zerofill, (also variants of above similar to fsetattr)
+ */
+#endif
+};
+
+struct xlator_cbks cbks = {
+        .forget = upcall_forget,
+        .release = upcall_release,
+};
+
+struct volume_options options[] = {
+        { .key = {NULL} },
+};
diff --git a/xlators/features/upcall/src/upcall.h b/xlators/features/upcall/src/upcall.h
new file mode 100644
index 00000000000..a5aff9d091e
--- /dev/null
+++ b/xlators/features/upcall/src/upcall.h
@@ -0,0 +1,134 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __UPCALL_H__
+#define __UPCALL_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "compat-errno.h"
+#include "stack.h"
+#include "call-stub.h"
+#include "upcall-mem-types.h"
+#include "client_t.h"
+#include "rpcsvc.h"
+#include "lkowner.h"
+#include "locking.h"
+#include "upcall-messages.h"
+#include "upcall-cache-invalidation.h"
+
+#define EXIT_IF_UPCALL_OFF(label) do {                     \
+        if (!(ON_CACHE_INVALIDATION))                      \
+                goto label;                                \
+} while (0)
+
+#define UPCALL_STACK_UNWIND(fop, frame, params ...) do {        \
+        upcall_local_t *__local = NULL;                         \
+        xlator_t *__xl          = NULL;                         \
+        if (frame) {                                            \
+                        __xl         = frame->this;             \
+                        __local      = frame->local;            \
+                        frame->local = NULL;                    \
+                }                                               \
+                STACK_UNWIND_STRICT (fop, frame, params);       \
+                upcall_local_wipe (__xl, __local);              \
+        } while (0)
+
+#define UPCALL_STACK_DESTROY(frame) do {                   \
+                upcall_local_t *__local = NULL;            \
+                xlator_t    *__xl    = NULL;               \
+                __xl                 = frame->this;        \
+                __local              = frame->local;       \
+                frame->local         = NULL;               \
+                STACK_DESTROY (frame->root);               \
+                upcall_local_wipe (__xl, __local);         \
+        } while (0)
+
+struct _upcalls_private_t {
+        int client_id; /* Not sure if reqd */
+};
+typedef struct _upcalls_private_t upcalls_private_t;
+
+enum _upcall_event_type_t {
+        EVENT_NULL,
+        CACHE_INVALIDATION,
+};
+typedef enum _upcall_event_type_t upcall_event_type_t;
+
+struct _upcall_client_t {
+        struct list_head client_list;
+        /* strdup to store client_uid, strdup. Free it explicitly */
+        char *client_uid;
+        time_t access_time; /* time last accessed */
+        /* the amount of time which client can cache this entry */
+        uint32_t expire_time_attr;
+};
+typedef struct _upcall_client_t upcall_client_t;
+
+/* Upcall entries are maintained in inode_ctx */
+struct _upcall_inode_ctx_t {
+        struct list_head client_list;
+        pthread_mutex_t client_list_lock; /* mutex for clients list
+                                                of this upcall entry */
+};
+typedef struct _upcall_inode_ctx_t upcall_inode_ctx_t;
+
+struct _notify_event_data {
+        uuid_t gfid;
+        upcall_client_t *client_entry;
+        upcall_event_type_t event_type;
+        uint32_t invalidate_flags;
+        /* any extra data needed, like inode flags
+         * to be invalidated incase of cache invalidation,
+         * may be fd for lease recalls */
+        void *extra;
+};
+typedef struct _notify_event_data notify_event_data_t;
+
+struct upcall_local {
+        /* XXX: need to check if we can store
+         * pointers in 'local' which may get freed
+         * in future by other thread
+         */
+        upcall_inode_ctx_t *upcall_inode_ctx;
+        inode_t   *inode;
+};
+typedef struct upcall_local upcall_local_t;
+
+void upcall_local_wipe (xlator_t *this, upcall_local_t *local);
+upcall_local_t *upcall_local_init (call_frame_t *frame, xlator_t *this, inode_t *inode);
+
+upcall_client_t *add_upcall_client (call_frame_t *frame, uuid_t gfid,
+                                    client_t *client,
+                                    upcall_inode_ctx_t *up_inode_ctx);
+upcall_client_t *__add_upcall_client (call_frame_t *frame, uuid_t gfid,
+                                      client_t *client,
+                                      upcall_inode_ctx_t *up_inode_ctx);
+upcall_client_t *__get_upcall_client (call_frame_t *frame, uuid_t gfid,
+                                      client_t *client,
+                                      upcall_inode_ctx_t *up_inode_ctx);
+int __upcall_cleanup_client_entry (upcall_client_t *up_client);
+
+int __upcall_inode_ctx_set (inode_t *inode, xlator_t *this);
+upcall_inode_ctx_t *__upcall_inode_ctx_get (inode_t *inode, xlator_t *this);
+upcall_inode_ctx_t *upcall_inode_ctx_get (inode_t *inode, xlator_t *this);
+int upcall_cleanup_inode_ctx (xlator_t *this, inode_t *inode);
+
+void upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client,
+                              inode_t *inode, uint32_t flags);
+void upcall_client_cache_invalidate (xlator_t *xl, uuid_t gfid,
+                                     upcall_client_t *up_client_entry,
+                                     uint32_t flags);
+void upcall_cache_forget (xlator_t *this, inode_t *inode,
+                          upcall_inode_ctx_t *up_inode_ctx);
+
+#endif /* __UPCALL_H__ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 779d6be34a4..f20490423f6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -569,7 +569,6 @@ optget_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
         return 0;
 }
 
-
 /* This getter considers defaults also. */
 static int
 volgen_dict_get (dict_t *dict, char *key, char **value)
@@ -1848,6 +1847,29 @@ out:
 }
 
 static int
+brick_graph_add_upcall (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                         dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+{
+
+        xlator_t        *xl = NULL;
+        int             ret = -1;
+
+        if (!graph || !volinfo || !set_dict)
+                goto out;
+
+        xl = volgen_graph_add (graph, "features/upcall", volinfo->volname);
+        if (!xl) {
+                gf_log ("glusterd", GF_LOG_WARNING,
+                        "failed to add features/upcall to graph");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
 brick_graph_add_server (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                          dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
@@ -2044,6 +2066,7 @@ static volgen_brick_xlator_t server_graph_table[] = {
         {brick_graph_add_index, "index"},
         {brick_graph_add_barrier, NULL},
         {brick_graph_add_iot, "io-threads"},
+        {brick_graph_add_upcall, "upcall"},
         {brick_graph_add_pump, NULL},
         {brick_graph_add_locks, "locks"},
         {brick_graph_add_acl, "acl"},
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index 947de76c926..0d742aae056 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -87,6 +87,7 @@ typedef enum {
         GF_XLATOR_POSIX = 0,
         GF_XLATOR_ACL,
         GF_XLATOR_LOCKS,
+        GF_XLATOR_UPCALL,
         GF_XLATOR_IOT,
         GF_XLATOR_INDEX,
         GF_XLATOR_MARKER,
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index d7eab16bf13..023f2a6234f 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -27,6 +27,12 @@
 #include "authenticate.h"
 #include "event.h"
 
+rpcsvc_cbk_program_t server_cbk_prog = {
+        .progname  = "Gluster Callback",
+        .prognum   = GLUSTER_CBK_PROGRAM,
+        .progver   = GLUSTER_CBK_VERSION,
+};
+
 void
 grace_time_handler (void *data)
 {
@@ -1093,22 +1099,72 @@ fini (xlator_t *this)
 int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
-        int          ret = 0;
-        int32_t      val = 0;
-        dict_t      *dict = NULL;
-        dict_t      *output = NULL;
-        va_list      ap;
+        int              ret          = 0;
+        int32_t          val          = 0;
+        dict_t           *dict        = NULL;
+        dict_t           *output      = NULL;
+        va_list          ap;
+        client_t         *client      = NULL;
+        char             *client_uid  = NULL;
+        struct gf_upcall *upcall_data = NULL;
+        gfs3_upcall_req  up_req;
+        server_conf_t    *conf        = NULL;
+        rpc_transport_t  *xprt        = NULL;
 
         dict = data;
         va_start (ap, data);
         output = va_arg (ap, dict_t*);
         va_end (ap);
 
+        conf = this->private;
+        if (!conf)
+                return 0;
+
         switch (event) {
+        case GF_EVENT_UPCALL:
+        {
+                if (!data) {
+                        ret = -1;
+                        goto out;
+                }
+
+                upcall_data = (struct gf_upcall *)data;
+
+                client_uid = upcall_data->client_uid;
+
+                if (!client_uid) {
+                        ret = -1;
+                        goto out;
+                }
+
+                gf_proto_upcall_from_upcall (&up_req, upcall_data);
+
+                pthread_mutex_lock (&conf->mutex);
+                {
+                        list_for_each_entry (xprt, &conf->xprt_list, list) {
+                                client = xprt->xl_private;
+
+                                if (strcmp(client->client_uid, client_uid))
+                                        continue;
+
+                                rpcsvc_request_submit(
+                                        conf->rpc, xprt,
+                                        &server_cbk_prog,
+                                        GF_CBK_UPCALL,
+                                        &up_req,
+                                        this->ctx,
+                                        (xdrproc_t)xdr_gfs3_upcall_req);
+                                break;
+                        }
+                }
+                pthread_mutex_unlock (&conf->mutex);
+                break;
+        }
         default:
                 default_notify (this, event, data);
                 break;
         }
+out:
         return ret;
 }