fuse: auxiliary gfid mount support

* files can be accessed directly through their gfid and not just
  through their paths. For eg., if the gfid of a file is
  f3142503-c75e-45b1-b92a-463cf4c01f99, that file can be accessed
  using <gluster-mount>/.gfid/f3142503-c75e-45b1-b92a-463cf4c01f99

  .gfid is a virtual directory used to seperate out the namespace
  for accessing files through gfid. This way, we do not conflict with
  filenames which can be qualified as uuids.

* A new file/directory/symlink can be created with a pre-specified
  gfid. A setxattr done on parent directory with fuse_auxgfid_newfile_args_t
  initialized with appropriate fields as value to key "glusterfs.gfid.newfile"
  results in the entry <parent>/bname whose gfid is set to args.gfid. The
  contents of the structure should be in network byte order.

  struct auxfuse_symlink_in {
        char     linkpath[]; /* linkpath is a null terminated string */
  } __attribute__ ((__packed__));

  struct auxfuse_mknod_in {
        unsigned int   mode;
        unsigned int   rdev;
        unsigned int   umask;
  } __attribute__ ((__packed__));

  struct auxfuse_mkdir_in {
        unsigned int   mode;
        unsigned int   umask;
  } __attribute__ ((__packed__));

  typedef struct {
        unsigned int  uid;
        unsigned int  gid;
        char          gfid[UUID_CANONICAL_FORM_LEN + 1]; /* a null terminated gfid string
                                                      * in canonical form.
                                                      */
        unsigned int  st_mode;
        char          bname[];     /* bname is a null terminated string */

        union {
                struct auxfuse_mkdir_in   mkdir;
                struct auxfuse_mknod_in   mknod;
                struct auxfuse_symlink_in symlink;
        } __attribute__ ((__packed__)) args;
  } __attribute__ ((__packed__)) fuse_auxgfid_newfile_args_t;

  An initial consumer of this feature would be geo-replication to
  create files on slave mount with same gfids as that on master.
  It will also help gsyncd to access files directly through their
  gfids. gsyncd in its newer version will be consuming a changelog
  (of master) containing operations on gfids and sync corresponding
  files to slave.

* Also, bring in support to heal gfids with a specific value.
  fuse-bridge sends across a gfid during a lookup, which storage
  translators assign to an inode (file/directory etc) if there is
  no gfid associated it. This patch brings in support
  to specify that gfid value from an application, instead of relying
  on random gfid generated by fuse-bridge.

  gfids can be healed through setxattr interface. setxattr should be
  done on parent directory. The key used is "glusterfs.gfid.heal"
  and the value should be the following structure whose contents
  should be in network byte order.

  typedef struct {
        char      gfid[UUID_CANONICAL_FORM_LEN + 1]; /* a null terminated gfid
                                                      * string in canonical form
                                                      */
        char      bname[]; /* a null terminated basename */
  } __attribute__((__packed__)) fuse_auxgfid_heal_args_t;

  This feature can be used for upgrading older geo-rep setups where gfids
  of files are different on master and slave to newer setups where they
  should be same. One can delete gfids on slave using setxattr -x and
  .glusterfs and issue stat on all the files with gfids from master.

Thanks to "Amar Tumballi" <amarts@redhat.com> and "Csaba Henk"
<csaba@redhat.com> for their inputs.

Signed-off-by: Raghavendra G <rgowdapp@redhat.com>
Change-Id: Ie8ddc0fb3732732315c7ec49eab850c16d905e4e
BUG: 952029
Reviewed-on: http://review.gluster.com/#/c/4702
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Tested-by: Amar Tumballi <amarts@redhat.com>
Reviewed-on: http://review.gluster.org/4702
Reviewed-by: Xavier Hernandez <xhernandez@datalab.es>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

13 files changed, 1317 insertions, 136 deletions

diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 213b589a5..3bf78d5aa 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -145,6 +145,10 @@ static struct argp_option gf_options[] = {
          "Mount the filesystem with POSIX ACL support"},
         {"selinux", ARGP_SELINUX_KEY, 0, 0,
          "Enable SELinux label (extened attributes) support on inodes"},
+#ifdef GF_LINUX_HOST_OS
+        {"aux-gfid-mount", ARGP_AUX_GFID_MOUNT_KEY, 0, 0,
+         "Enable access to filesystem through gfid directly"},
+#endif
         {"enable-ino32", ARGP_INODE32_KEY, "BOOL", OPTION_ARG_OPTIONAL,
          "Use 32-bit inodes when mounting to workaround broken applications"
          "that don't support 64-bit inodes"},
@@ -349,6 +353,17 @@ set_fuse_mount_options (glusterfs_ctx_t *ctx, dict_t *options)
                 }
         }
 
+        if (cmd_args->aux_gfid_mount) {
+                ret = dict_set_static_ptr (options, "auxiliary-gfid-mount",
+                                           "on");
+                if (ret < 0) {
+                        gf_log ("glusterfsd", GF_LOG_ERROR,
+                                "failed to set dict value for key "
+                                "aux-gfid-mount");
+                        goto err;
+                }
+        }
+
         if (cmd_args->enable_ino32) {
                 ret = dict_set_static_ptr (options, "enable-ino32", "on");
                 if (ret < 0) {
@@ -688,6 +703,10 @@ parse_opts (int key, char *arg, struct argp_state *state)
 					   "*-md-cache.cache-selinux=true");
                 break;
 
+        case ARGP_AUX_GFID_MOUNT_KEY:
+                cmd_args->aux_gfid_mount = 1;
+                break;
+
         case ARGP_INODE32_KEY:
                 cmd_args->enable_ino32 = 1;
                 break;
diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h
index be91ce548..06b4aacc8 100644
--- a/glusterfsd/src/glusterfsd.h
+++ b/glusterfsd/src/glusterfsd.h
@@ -85,6 +85,7 @@ enum argp_option_keys {
         ARGP_INODE32_KEY                  = 163,
 	ARGP_FUSE_MOUNTOPTS_KEY		  = 164,
         ARGP_FUSE_USE_READDIRP_KEY        = 165,
+	ARGP_AUX_GFID_MOUNT_KEY		  = 166,
 };
 
 struct _gfd_vol_top_priv_t {
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 8643d7565..6c09e89b2 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -102,6 +102,7 @@
 #define UUID_CANONICAL_FORM_LEN 36
 
 #define GLUSTERFS_INTERNAL_FOP_KEY  "glusterfs-internal-fop"
+#define GLUSTERFS_CREATE_MODE_KEY "glusterfs-create-mode"
 
 #define ZR_FILE_CONTENT_STR     "glusterfs.file."
 #define ZR_FILE_CONTENT_STRLEN 15
@@ -320,6 +321,7 @@ struct _cmd_args {
         int              mac_compat;
 	int		 fopen_keep_cache;
 	int		 gid_timeout;
+        int              aux_gfid_mount;
 	struct list_head xlator_options;  /* list of xlator_option_t */
 
 	/* fuse options */
@@ -348,6 +350,7 @@ struct _cmd_args {
         int             brick_port;
         char           *brick_name;
         int             brick_port2;
+
 };
 typedef struct _cmd_args cmd_args_t;
 
diff --git a/libglusterfs/src/inode.c b/libglusterfs/src/inode.c
index 6f1c8ec3f..add686fd2 100644
--- a/libglusterfs/src/inode.c
+++ b/libglusterfs/src/inode.c
@@ -310,7 +310,7 @@ __inode_destroy (inode_t *inode)
                 goto noctx;
         }
 
-        for (index = 0; index < inode->table->xl->graph->xl_count; index++) {
+        for (index = 0; index < inode->table->ctxcount; index++) {
                 if (inode->_ctx[index].xl_key) {
                         xl = (xlator_t *)(long)inode->_ctx[index].xl_key;
                         old_THIS = THIS;
@@ -528,10 +528,9 @@ __inode_create (inode_table_t *table)
         INIT_LIST_HEAD (&newi->hash);
         INIT_LIST_HEAD (&newi->dentry_list);
 
-        newi->_ctx = GF_CALLOC (1, (sizeof (struct _inode_ctx) *
-                                    table->xl->graph->xl_count),
+        newi->_ctx = GF_CALLOC (1,
+                                (sizeof (struct _inode_ctx) * table->ctxcount),
                                 gf_common_mt_inode_ctx);
-
         if (newi->_ctx == NULL) {
                 LOCK_DESTROY (&newi->lock);
                 mem_put (newi);
@@ -1316,6 +1315,7 @@ inode_table_new (size_t lru_limit, xlator_t *xl)
                 return NULL;
 
         new->xl = xl;
+        new->ctxcount = xl->graph->xl_count + 1;
 
         new->lru_limit = lru_limit;
 
@@ -1466,7 +1466,7 @@ __inode_ctx_set2 (inode_t *inode, xlator_t *xlator, uint64_t *value1_p,
         if (!inode || !xlator)
                 return -1;
 
-        for (index = 0; index < xlator->graph->xl_count; index++) {
+        for (index = 0; index < inode->table->ctxcount; index++) {
                 if (!inode->_ctx[index].xl_key) {
                         if (set_idx == -1)
                                 set_idx = index;
@@ -1523,12 +1523,12 @@ __inode_ctx_get2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
         if (!inode || !xlator)
                 return -1;
 
-        for (index = 0; index < xlator->graph->xl_count; index++) {
+        for (index = 0; index < inode->table->ctxcount; index++) {
                 if (inode->_ctx[index].xl_key == xlator)
                         break;
         }
 
-        if (index == xlator->graph->xl_count) {
+        if (index == inode->table->ctxcount) {
                 ret = -1;
                 goto out;
         }
@@ -1575,12 +1575,13 @@ inode_ctx_del2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
 
         LOCK (&inode->lock);
         {
-                for (index = 0; index < xlator->graph->xl_count; index++) {
+                for (index = 0; index < inode->table->ctxcount;
+                     index++) {
                         if (inode->_ctx[index].xl_key == xlator)
                                 break;
                 }
 
-                if (index == xlator->graph->xl_count) {
+                if (index == inode->table->ctxcount) {
                         ret = -1;
                         goto unlock;
                 }
@@ -1628,14 +1629,15 @@ inode_dump (inode_t *inode, char *prefix)
                 gf_proc_dump_write("ref", "%u", inode->ref);
                 gf_proc_dump_write("ia_type", "%d", inode->ia_type);
                 if (inode->_ctx) {
-                        inode_ctx = GF_CALLOC (inode->table->xl->graph->xl_count,
+                        inode_ctx = GF_CALLOC (inode->table->ctxcount,
                                                sizeof (*inode_ctx),
                                                gf_common_mt_inode_ctx);
                         if (inode_ctx == NULL) {
                                 goto unlock;
                         }
 
-                        for (i = 0; i < inode->table->xl->graph->xl_count; i++) {
+                        for (i = 0; i < inode->table->ctxcount;
+                             i++) {
                                 inode_ctx[i] = inode->_ctx[i];
                         }
                 }
@@ -1652,7 +1654,7 @@ unlock:
         UNLOCK(&inode->lock);
 
         if (inode_ctx && (dump_options.xl_options.dump_inodectx == _gf_true)) {
-                for (i = 0; i < inode->table->xl->graph->xl_count; i++) {
+                for (i = 0; i < inode->table->ctxcount; i++) {
                         if (inode_ctx[i].xl_key) {
                                 xl = (xlator_t *)(long)inode_ctx[i].xl_key;
                                 if (xl->dumpops && xl->dumpops->inodectx)
diff --git a/libglusterfs/src/inode.h b/libglusterfs/src/inode.h
index 199ce4484..cf766a31b 100644
--- a/libglusterfs/src/inode.h
+++ b/libglusterfs/src/inode.h
@@ -56,6 +56,7 @@ struct _inode_table {
         struct mem_pool   *inode_pool;  /* memory pool for inodes */
         struct mem_pool   *dentry_pool; /* memory pool for dentrys */
         struct mem_pool   *fd_mem_pool; /* memory pool for fd_t */
+        int                ctxcount;    /* number of slots in inode->ctx */
 };
 
 
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 746dd1a8a..8b34d1a73 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -1436,7 +1436,6 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         conf = this->private;
         if (!conf)
@@ -2144,7 +2143,6 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
         conf   = this->private;
@@ -2503,7 +2501,6 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         conf   = this->private;
 
@@ -2717,7 +2714,6 @@ dht_removexattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         local = dht_local_init (frame, loc, NULL, GF_FOP_REMOVEXATTR);
         if (!local) {
@@ -2949,7 +2945,6 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
         conf = this->private;
@@ -4885,7 +4880,6 @@ dht_entrylk (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         local = dht_local_init (frame, loc, NULL, GF_FOP_ENTRYLK);
         if (!local) {
diff --git a/xlators/mount/fuse/src/Makefile.am b/xlators/mount/fuse/src/Makefile.am
index 653121d18..04c656b93 100644
--- a/xlators/mount/fuse/src/Makefile.am
+++ b/xlators/mount/fuse/src/Makefile.am
@@ -6,6 +6,9 @@ noinst_HEADERS_common = $(CONTRIBDIR)/fuse-include/fuse-mount.h\
 	$(CONTRIBDIR)/fuse-include/fuse-misc.h fuse-mem-types.h \
 	fuse-bridge.h
 
+fuse_HEADERS = glfs-fuse-bridge.h
+fusedir = $(includedir)/glusterfs/
+
 if GF_DARWIN_HOST_OS
     noinst_HEADERS = $(noinst_HEADERS_common) $(noinst_HEADERS_darwin)
 else
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index 0291859c0..85ecce958 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -9,9 +9,13 @@
 */
 
 #include <sys/wait.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <dirent.h>
 #include "fuse-bridge.h"
 #include "mount-gluster-compat.h"
 #include "glusterfs.h"
+#include "byte-order.h"
 
 #ifdef __NetBSD__
 #undef open /* in perfuse.h, pulled from mount-gluster-compat.h */
@@ -24,6 +28,14 @@ void fini (xlator_t *this_xl);
 
 static void fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino);
 
+static inline void
+fuse_lookup_gfid_dir (xlator_t *this, fuse_state_t *state, char *name,
+                      gf_lookup_namespace_t ns);
+
+static int
+fuse_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata);
 /*
  * Send an invalidate notification up to fuse to purge the file from local
  * page cache.
@@ -43,10 +55,16 @@ fuse_invalidate(xlator_t *this, inode_t *inode)
         if (!priv->fopen_keep_cache)
                 return 0;
 
-        nodeid = inode_to_fuse_nodeid(inode);
+        /* shouldn't matter what namespace we use here, since if it
+         * is auxiliary gfid mount, we are going to invalidate both inodes
+         * corresponding to gfid and path namespaces.
+         */
+        nodeid = inode_to_fuse_nodeid(this, inode, GF_FUSE_PATH_NAMESPACE);
+
         gf_log(this->name, GF_LOG_DEBUG, "Invalidate inode id %lu.", nodeid);
         fuse_log_eh (this, "Sending invalidate inode id: %lu gfid: %s", nodeid,
                      uuid_utoa (inode->gfid));
+
         fuse_invalidate_inode(this, nodeid);
 
         return 0;
@@ -214,10 +232,14 @@ fuse_invalidate_entry (xlator_t *this, uint64_t fuse_ino)
         list_for_each_entry (dentry, &inode->dentry_list, inode_list) {
                 nlen = strlen (dentry->name);
                 fouh->len = sizeof (*fouh) + sizeof (*fnieo) + nlen + 1;
-                fnieo->parent = inode_to_fuse_nodeid (dentry->parent);
+
+                /* entries in GF_FUSE_GFID_NAMESPACE are never linked */
+                fnieo->parent = inode_to_fuse_nodeid (this, dentry->parent,
+                                                      GF_FUSE_PATH_NAMESPACE);
 
                 fnieo->namelen = nlen;
-                strcpy (inval_buf + sizeof (*fouh) + sizeof (*fnieo), dentry->name);
+                strcpy (inval_buf + sizeof (*fouh) + sizeof (*fnieo),
+                        dentry->name);
 
                 rv = write (priv->revchan_out, inval_buf, fouh->len);
                 if (rv != fouh->len) {
@@ -245,6 +267,43 @@ fuse_invalidate_entry (xlator_t *this, uint64_t fuse_ino)
                 inode_unref (inode);
 }
 
+static int32_t
+fuse_write_invalidate_request (xlator_t *this, struct fuse_out_header *fouh,
+                               struct fuse_notify_inval_inode_out *fniio,
+                               uint64_t fuse_ino, inode_t *inode)
+{
+        int32_t         rv = -1;
+        fuse_private_t *priv = NULL;
+
+        if (!this || !fouh || !fniio || !fuse_ino || !inode)
+                goto out;
+
+        priv = this->private;
+
+        /* inval the entire mapping until we learn how to be more granular */
+        fniio->ino = fuse_ino;
+
+        rv = write(priv->revchan_out, fouh, fouh->len);
+        if (rv != fouh->len) {
+                gf_log("glusterfs-fuse", GF_LOG_ERROR, "kernel notification "
+                        "daemon defunct");
+                close(priv->fd);
+        }
+
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "INVALIDATE inode: %lu",
+               fuse_ino);
+
+        if (inode) {
+                fuse_log_eh (this, "Invalidated inode %lu (gfid: %s)",
+                             fuse_ino, uuid_utoa (inode->gfid));
+        } else {
+                fuse_log_eh (this, "Invalidated inode %lu ", fuse_ino);
+        }
+
+out:
+        return rv;
+}
+
 /*
  * Send an inval inode notification to fuse. This causes an invalidation of the
  * entire page cache mapping on the inode.
@@ -252,12 +311,14 @@ fuse_invalidate_entry (xlator_t *this, uint64_t fuse_ino)
 static void
 fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino)
 {
-        struct fuse_out_header *fouh = NULL;
-        struct fuse_notify_inval_inode_out *fniio = NULL;
-        fuse_private_t *priv = NULL;
-        int rv = 0;
-        char inval_buf[INVAL_BUF_SIZE] = {0};
-        inode_t    *inode = NULL;
+        struct fuse_out_header             *fouh   = NULL;
+        struct fuse_notify_inval_inode_out *fniio  = NULL;
+        fuse_private_t                     *priv   = NULL;
+        int                                 rv     = 0;
+        char inval_buf[INVAL_BUF_SIZE]             = {0};
+        inode_t                            *inode  = NULL;
+        gf_fuse_nodeid_t                   *nodeid = NULL;
+        uint64_t                            value  = 0;
 
         fouh = (struct fuse_out_header *) inval_buf;
         fniio = (struct fuse_notify_inval_inode_out *) (fouh + 1);
@@ -272,31 +333,46 @@ fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino)
         fouh->len = sizeof(struct fuse_out_header) +
                 sizeof(struct fuse_notify_inval_inode_out);
 
-        /* inval the entire mapping until we learn how to be more granular */
-        fniio->ino = fuse_ino;
         fniio->off = 0;
         fniio->len = -1;
 
         inode = fuse_ino_to_inode (fuse_ino, this);
 
-        rv = write(priv->revchan_out, inval_buf, fouh->len);
-        if (rv != fouh->len) {
-                gf_log("glusterfs-fuse", GF_LOG_ERROR, "kernel notification "
-                        "daemon defunct");
-                close(priv->fd);
-        }
+        if (priv->aux_gfid_mount) {
+                inode_ctx_get (inode, this, &value);
+                nodeid = (void *)value;
+                if (nodeid == NULL) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "nodeid is NULL on an auxiliary mount");
+                        goto out;
+                }
 
-        gf_log("glusterfs-fuse", GF_LOG_TRACE, "INVALIDATE inode: %lu", fuse_ino);
+                rv = fuse_write_invalidate_request (this, fouh, fniio,
+                                                    (uint64_t) &nodeid->inode_path_ns,
+                                                    inode);
+                if (rv < 0) {
+                        goto out;
+                }
 
-        if (inode) {
-                fuse_log_eh (this, "Invalidated inode %lu (gfid: %s)",
-                             fuse_ino, uuid_utoa (inode->gfid));
+                rv = fuse_write_invalidate_request (this, fouh, fniio,
+                                                    (uint64_t) &nodeid->inode_gfid_ns,
+                                                    inode);
+                if (rv < 0) {
+                        goto out;
+                }
         } else {
-                fuse_log_eh (this, "Invalidated inode %lu ", fuse_ino);
+                rv = fuse_write_invalidate_request (this, fouh, fniio,
+                                                    fuse_ino, inode);
+                if (rv < 0) {
+                        goto out;
+                }
         }
 
+out:
         if (inode)
                 inode_unref (inode);
+
+        return;
 }
 
 int
@@ -340,10 +416,12 @@ fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         struct fuse_entry_out  feo          = {0, };
         fuse_private_t        *priv         = NULL;
         inode_t               *linked_inode = NULL;
+        gf_lookup_namespace_t  ns           = 0;
 
         priv = this->private;
         state = frame->root->state;
         finh = state->finh;
+        ns = (gf_lookup_namespace_t) cookie;
 
         if (op_ret == 0) {
                 if (__is_root_gfid (state->loc.inode->gfid))
@@ -386,15 +464,20 @@ fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 gf_fop_list[frame->root->op], state->loc.path);
                 }
 
-                linked_inode = inode_link (inode, state->loc.parent,
-                                           state->loc.name, buf);
+                if (ns == GF_FUSE_GFID_NAMESPACE) {
+                        linked_inode = inode_link (inode, NULL, NULL, buf);
+                } else {
+                        linked_inode = inode_link (inode, state->loc.parent,
+                                                   state->loc.name, buf);
 
-                if (linked_inode != inode) {
+                        if (linked_inode != inode) {
+                        }
                 }
 
+
                 inode_lookup (linked_inode);
 
-                feo.nodeid = inode_to_fuse_nodeid (linked_inode);
+                feo.nodeid = inode_to_fuse_nodeid (this, linked_inode, ns);
 
                 inode_unref (linked_inode);
 
@@ -456,11 +539,9 @@ fuse_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  struct iatt *postparent)
 {
         fuse_state_t            *state = NULL;
-        call_frame_t            *prev = NULL;
         inode_table_t           *itable = NULL;
 
         state = frame->root->state;
-        prev  = cookie;
 
         if (op_ret == -1 && state->is_revalidate == 1) {
                 itable = state->itable;
@@ -479,7 +560,8 @@ fuse_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 fuse_gfid_set (state);
 
                 STACK_WIND (frame, fuse_lookup_cbk,
-                            prev->this, prev->this->fops->lookup,
+                            state->active_subvol,
+                            state->active_subvol->fops->lookup,
                             &state->loc, state->xdata);
                 return 0;
         }
@@ -527,27 +609,344 @@ fuse_lookup_resume (fuse_state_t *state)
                   lookup, &state->loc, state->xdata);
 }
 
+static inline int
+fuse_handle_gfiddir_rename_op (fuse_state_t *state, void *msg)
+{
+        struct fuse_rename_in *fri        = msg;
+        char                  *oldname    = (char *)(fri + 1);
+        char                  *newname    = oldname + strlen (oldname) + 1;
+        uuid_t                 aux_gfid   = {0, };
+        inode_t               *src_parent = NULL, *dst_parent = NULL;
+        int                    ret        = -1;
+
+        aux_gfid[15] = 0xd;
+
+        src_parent = fuse_ino_to_inode (state->finh->nodeid, state->this);
+        dst_parent = fuse_ino_to_inode (fri->newdir, state->this);
+
+        if ((uuid_compare (aux_gfid, src_parent->gfid) == 0)
+            || (uuid_compare (aux_gfid, dst_parent->gfid) == 0)) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "RENAME %"PRIu64" rename into or from virtual "
+                        "gfid-directory is not allowed.",
+                        state->finh->unique);
+
+                send_fuse_err (state->this, state->finh, ENOTSUP);
+                free_fuse_state (state);
+                ret = 0;
+        }
+
+        if ((__is_root_gfid (src_parent->gfid)
+             && (strncmp (oldname, GF_GFID_DIR, strlen (GF_GFID_DIR) == 0)))
+            || (__is_root_gfid (dst_parent->gfid)
+                && (strncmp (newname, GF_GFID_DIR,
+                             strlen (GF_GFID_DIR))) == 0)) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "RENAME %"PRIu64" renaming virtual "
+                        "gfid-directory is not allowed.",
+                        state->finh->unique);
+
+                send_fuse_err (state->this, state->finh, ENOTSUP);
+                free_fuse_state (state);
+                ret = 0;
+        }
+
+        inode_unref (src_parent);
+        inode_unref (dst_parent);
+
+        return ret;
+}
+
+static inline int
+fuse_handle_gfiddir_inode_op (fuse_state_t *state, glusterfs_fop_t fop)
+{
+        uuid_t   aux_gfid = {0, };
+        int      ret      = -1;
+        inode_t *inode    = NULL;
+
+        aux_gfid[15] = 0xd;
+
+        inode = fuse_ino_to_inode (state->finh->nodeid, state->this);
+
+        if (uuid_compare (aux_gfid, inode->gfid) == 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "%s %"PRIu64" %s on virtual "
+                        "gfid-directory is not allowed.",
+                        gf_fop_list[fop], state->finh->unique,
+                        gf_fop_list[fop]);
+
+                send_fuse_err (state->this, state->finh, ENOTSUP);
+                free_fuse_state (state);
+                ret = 0;
+        }
+
+        inode_unref (inode);
+
+        return ret;
+}
+
+static inline int
+fuse_handle_gfiddir_rmdir_op (fuse_state_t *state, char *name)
+{
+        int      ret      = -1;
+        inode_t *parent   = NULL;
+
+        parent = fuse_ino_to_inode (state->finh->nodeid, state->this);
+
+        if (__is_root_gfid (parent->gfid) && (strncmp (name, GF_GFID_DIR,
+                                                       strlen (GF_GFID_DIR)
+                                                       == 0))) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "RMDIR %"PRIu64" removing virtual "
+                        "gfid-directory is not allowed.",
+                        state->finh->unique);
+
+                send_fuse_err (state->this, state->finh, ENOTSUP);
+                free_fuse_state (state);
+                ret = -1;
+        }
+
+        return ret;
+}
+
+static inline int
+fuse_handle_gfiddir_entry_op (fuse_state_t *state, glusterfs_fop_t fop)
+{
+        uuid_t   aux_gfid   = {0, };
+        int      ret        = -1;
+        inode_t *parent = NULL;
+
+        aux_gfid[15] = 0xd;
+
+        parent = fuse_ino_to_inode (state->finh->nodeid, state->this);
+
+        if (uuid_compare (aux_gfid, parent->gfid) == 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "%s %"PRIu64" operation not allowed in virtual "
+                        "gfid-directory.", gf_fop_list [fop],
+                        state->finh->unique);
+
+                send_fuse_err (state->this, state->finh, ENOTSUP);
+                free_fuse_state (state);
+                ret = 0;
+        }
+
+        inode_unref (parent);
+        return ret;
+}
+
+static inline int
+fuse_handle_gfiddir_op (xlator_t *this, fuse_state_t *state, void *msg,
+                        glusterfs_fop_t fop)
+{
+        int ret = -1;
+
+        switch (fop) {
+        case GF_FOP_SETATTR:
+        case GF_FOP_ACCESS:
+        case GF_FOP_OPENDIR:
+        case GF_FOP_SETXATTR:
+        case GF_FOP_GETXATTR:
+        case GF_FOP_REMOVEXATTR:
+                ret = fuse_handle_gfiddir_inode_op (state, fop);
+                break;
+
+        case GF_FOP_RMDIR:
+                ret = fuse_handle_gfiddir_rmdir_op (state, msg);
+                break;
+
+        case GF_FOP_RENAME:
+                ret = fuse_handle_gfiddir_rename_op (state, msg);
+                break;
+
+        case GF_FOP_LINK:
+                ret = fuse_handle_gfiddir_entry_op (state, fop);
+
+        case GF_FOP_UNLINK:
+                ret = fuse_handle_gfiddir_entry_op (state, fop);
+
+        default:
+                break;
+        }
+
+        return ret;
+}
+
+static inline int
+fuse_lookup_aux_gfid (xlator_t *this, fuse_state_t *state, inode_t *parent,
+                      char *name)
+{
+        int           ret      = -1, op_errno = EINVAL;
+        call_frame_t *frame    = NULL;
+        uuid_t        aux_gfid = {0, };
+
+        aux_gfid[15] = 0xd;
+
+        ret = uuid_parse (name, state->gfid);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "parsing application provided gfid (%s) failed", name);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        fuse_gfid_set (state);
+
+        if (uuid_compare (aux_gfid, state->gfid) == 0) {
+                fuse_lookup_gfid_dir (this, state, NULL,
+                                      GF_FUSE_GFID_NAMESPACE);
+                goto out;
+        }
+
+        state->loc.inode = inode_find (state->itable, state->gfid);
+        if (state->loc.inode == NULL) {
+                state->loc.inode = inode_new (state->itable);
+                if (state->loc.inode == NULL) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+        } else {
+                state->is_revalidate = 1;
+        }
+
+        uuid_copy (state->loc.gfid, state->gfid);
+        FUSE_FOP_COOKIE (state, this, fuse_lookup_cbk,
+                         (void *)GF_FUSE_GFID_NAMESPACE,
+                         GF_FOP_LOOKUP, lookup, &state->loc, state->xdata);
+
+out:
+        return 0;
+
+err:
+        frame = get_call_frame_for_req (state);
+        frame->root->state = state;
+        fuse_entry_cbk (frame, (void *)GF_FUSE_GFID_NAMESPACE, this, -1,
+                        op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+static inline void
+fuse_lookup_gfid_dir (xlator_t *this, fuse_state_t *state, char *name,
+                      gf_lookup_namespace_t ns)
+{
+        call_frame_t   *frame    = NULL;
+        fuse_private_t *priv     = NULL;
+        int32_t         op_errno = EINVAL, op_ret = -1;
+
+        priv = this->private;
+
+        frame = get_call_frame_for_req (state);
+        if (frame == NULL) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        frame->root->state = state;
+
+        /* virtual stat entry */
+        priv->gfiddir_stbuf.ia_gfid[15] = GF_AUX_GFID;
+        priv->gfiddir_stbuf.ia_ino = 13;
+        priv->gfiddir_stbuf.ia_type = IA_IFDIR;
+        priv->gfiddir_stbuf.ia_nlink = 2;
+
+        state->loc.inode = inode_find (state->itable,
+                                       priv->gfiddir_stbuf.ia_gfid);
+
+        if (state->loc.inode == NULL) {
+                /* need to build 'state->loc' ourselves */
+                state->loc.inode = inode_new (state->itable);
+                if (state->loc.inode == NULL) {
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+        }
+
+        op_ret = op_errno = 0;
+out:
+        switch (state->finh->opcode) {
+        case FUSE_LOOKUP:
+                fuse_entry_cbk (frame, (void *)ns, this, op_ret, op_errno,
+                                state->loc.inode, &priv->gfiddir_stbuf, NULL);
+                break;
+
+        case FUSE_GETATTR:
+                fuse_attr_cbk (frame, NULL, this, op_ret, op_errno,
+                               &priv->gfiddir_stbuf, NULL);
+                break;
+        default:
+                break;
+        }
+
+        return;
+}
+
+
 static void
 fuse_lookup (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        char         *name      = msg;
-        fuse_state_t *state     = NULL;
+        char           *name     = msg;
+        fuse_state_t   *state    = NULL;
+        fuse_private_t *priv     = NULL;
+        inode_t        *parent   = NULL;
+        uuid_t          aux_gfid = {0,};
 
+        priv = this->private;
         GET_STATE (this, finh, state);
 
+        if (!priv->aux_gfid_mount) {
+                goto normal_mount;
+        }
+
+        if ((finh->nodeid == 1) && (strcmp (GF_GFID_DIR, name) == 0)) {
+                fuse_lookup_gfid_dir (this, state, name,
+                                      GF_FUSE_PATH_NAMESPACE);
+                return;
+        }
+
+        aux_gfid[15] = GF_AUX_GFID;
+
+        parent = fuse_ino_to_inode (finh->nodeid, state->this);
+
+        if (uuid_compare (aux_gfid, parent->gfid) == 0) {
+                fuse_lookup_aux_gfid (this, state, parent, name);
+                inode_unref (parent);
+                return;
+        }
+
+        inode_unref (parent);
+
+normal_mount:
         (void) fuse_resolve_entry_init (state, &state->resolve,
                                         finh->nodeid, name);
 
         fuse_resolve_and_resume (state, fuse_lookup_resume);
+
+        return;
+}
+
+int32_t
+fuse_xlator_forget (xlator_t *this, inode_t *inode)
+{
+        gf_fuse_nodeid_t *nodeid = NULL;
+        uint64_t          value  = 0;
+
+        inode_ctx_del (inode, this, &value);
+        nodeid = (void *)value;
+
+        if (nodeid != NULL) {
+                mem_put (nodeid);
+        }
+
+        return 0;
 }
 
 static void
 fuse_forget (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
 {
-        struct fuse_forget_in *ffi = msg;
-
-        inode_t      *fuse_inode;
+        struct fuse_forget_in *ffi        = msg;
+        inode_t               *fuse_inode = NULL;
 
         if (finh->nodeid == 1) {
                 GF_FREE (finh);
@@ -648,6 +1047,10 @@ fuse_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         state->loc.path ? state->loc.path : "ERR",
                         buf->ia_ino);
 
+                if (__is_root_gfid (buf->ia_gfid)) {
+                        priv->root_stbuf = *buf;
+                }
+
                 buf->ia_blksize = this->ctx->page_size;
                 gf_fuse_stat2attr (buf, &fao.attr, priv->enable_ino32);
 
@@ -732,9 +1135,13 @@ fuse_getattr_resume (fuse_state_t *state)
 static void
 fuse_getattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-        int32_t       ret = -1;
+        fuse_state_t   *state    = NULL;
+        int32_t         ret      = -1;
+        fuse_private_t *priv     = NULL;
+        uuid_t          aux_gfid = {0, };
+        inode_t        *inode    = NULL;
 
+        priv = this->private;
         GET_STATE (this, finh, state);
 
         if (finh->nodeid == 1) {
@@ -757,6 +1164,21 @@ fuse_getattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 return;
         }
 
+        if (priv->aux_gfid_mount) {
+                aux_gfid[15] = GF_AUX_GFID;
+
+                inode = fuse_ino_to_inode (finh->nodeid, state->this);
+
+                if (uuid_compare (aux_gfid, inode->gfid) == 0) {
+                        fuse_lookup_gfid_dir (this, state, NULL,
+                                              GF_FUSE_PATH_NAMESPACE);
+                        inode_unref (inode);
+                        return;
+                }
+
+                inode_unref (inode);
+        }
+
         fuse_resolve_inode_init (state, &state->resolve, state->finh->nodeid);
 
         fuse_resolve_and_resume (state, fuse_getattr_resume);
@@ -1055,12 +1477,13 @@ fuse_setattr_resume (fuse_state_t *state)
 static void
 fuse_setattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_setattr_in *fsi = msg;
-
-        fuse_private_t  *priv = NULL;
-        fuse_state_t *state = NULL;
+        struct fuse_setattr_in *fsi      = msg;
+        fuse_private_t         *priv     = NULL;
+        fuse_state_t           *state    = NULL;
+        int                     ret      = 0;
 
         GET_STATE (this, finh, state);
+        priv = this->private;
 
         if (fsi->valid & FATTR_FH &&
             !(fsi->valid & (FATTR_ATIME|FATTR_MTIME))) {
@@ -1069,6 +1492,13 @@ fuse_setattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 state->fd = FH_TO_FD (fsi->fh);
                 fuse_resolve_fd_init (state, &state->resolve, state->fd);
         } else {
+                if (priv->aux_gfid_mount) {
+                        ret = fuse_handle_gfiddir_op (this, state, msg,
+                                                      GF_FOP_SETATTR);
+                        if (ret == 0)
+                                return;
+                }
+
                 fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
         }
 
@@ -1234,11 +1664,21 @@ fuse_access_resume (fuse_state_t *state)
 static void
 fuse_access (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_access_in *fai = msg;
-        fuse_state_t *state = NULL;
+        struct fuse_access_in *fai   = msg;
+        fuse_state_t          *state = NULL;
+        fuse_private_t        *priv  = NULL;
+        int                    ret   = 0;
 
         GET_STATE (this, finh, state);
 
+        priv = this->private;
+
+        if (priv->aux_gfid_mount) {
+                ret = fuse_handle_gfiddir_op (this, state, msg, GF_FOP_ACCESS);
+                if (ret == 0)
+                        return;
+        }
+
         fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         state->mask = fai->mask;
@@ -1324,6 +1764,11 @@ fuse_readlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 void
 fuse_mknod_resume (fuse_state_t *state)
 {
+        fuse_private_t *priv = NULL;
+        int             ret  = 0;
+
+        priv = state->this->private;
+
         if (!state->loc.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "MKNOD %"PRId64"/%s (%s/%s) resolution failed",
@@ -1340,11 +1785,31 @@ fuse_mknod_resume (fuse_state_t *state)
         }
 
         if (state->loc.inode) {
-                gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
+                gf_log (state->this->name, GF_LOG_DEBUG,
+                        "inode already present");
                 inode_unref (state->loc.inode);
                 state->loc.inode = NULL;
         }
 
+        if (state->xdata == NULL) {
+                state->xdata = dict_new ();
+        }
+
+        if (priv->aux_gfid_mount) {
+                ret = dict_set_str (state->xdata, GLUSTERFS_INTERNAL_FOP_KEY,
+                                    "gfid-mount");
+
+                if (ret < 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                                "MKNOD %"PRId64"/%s (%s/%s) setting a key to "
+                                "overload mknod to act as link(2) when target "
+                                "already exists failed",
+                                state->finh->nodeid, state->resolve.bname,
+                                uuid_utoa (state->resolve.gfid),
+                                state->resolve.bname);
+                }
+        }
+
         state->loc.inode = inode_new (state->loc.parent->table);
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
@@ -1359,12 +1824,11 @@ fuse_mknod_resume (fuse_state_t *state)
 static void
 fuse_mknod (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_mknod_in *fmi = msg;
-        char         *name = (char *)(fmi + 1);
-
-        fuse_state_t   *state = NULL;
-        fuse_private_t *priv = NULL;
-        int32_t         ret = -1;
+        struct fuse_mknod_in *fmi   = msg;
+        char                 *name  = (char *)(fmi + 1);
+        fuse_state_t         *state = NULL;
+        fuse_private_t       *priv  = NULL;
+        int32_t               ret   = -1;
 
         priv = this->private;
 #if FUSE_KERNEL_MINOR_VERSION >= 12
@@ -1477,10 +1941,19 @@ fuse_unlink_resume (fuse_state_t *state)
 static void
 fuse_unlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        char         *name = msg;
-        fuse_state_t *state = NULL;
+        char           *name  = msg;
+        fuse_state_t   *state = NULL;
+        fuse_private_t *priv  = NULL;
+        int             ret   = -1;
 
         GET_STATE (this, finh, state);
+        priv = this->private;
+
+        if (priv->aux_gfid_mount) {
+                ret = fuse_handle_gfiddir_op (this, state, msg, GF_FOP_UNLINK);
+                if (ret == 0)
+                        return;
+        }
 
         fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
@@ -1513,11 +1986,21 @@ fuse_rmdir_resume (fuse_state_t *state)
 static void
 fuse_rmdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        char         *name = msg;
-        fuse_state_t *state = NULL;
+        char           *name   = msg;
+        fuse_state_t   *state  = NULL;
+        fuse_private_t *priv   = NULL;
+        int             ret    = 0;
 
         GET_STATE (this, finh, state);
 
+        priv = this->private;
+
+        if (priv->aux_gfid_mount) {
+                ret = fuse_handle_gfiddir_op (this, state, msg, GF_FOP_RMDIR);
+                if (ret == 0)
+                        return;
+        }
+
         fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         fuse_resolve_and_resume (state, fuse_rmdir_resume);
@@ -1544,7 +2027,8 @@ fuse_symlink_resume (fuse_state_t *state)
         }
 
         if (state->loc.inode) {
-                gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
+                gf_log (state->this->name, GF_LOG_DEBUG,
+                        "inode already present");
                 inode_unref (state->loc.inode);
                 state->loc.inode = NULL;
         }
@@ -1556,7 +2040,8 @@ fuse_symlink_resume (fuse_state_t *state)
                 state->loc.path, state->name);
 
         FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_SYMLINK,
-                  symlink, state->name, &state->loc, state->umask, state->xdata);
+                  symlink, state->name, &state->loc, state->umask,
+                  state->xdata);
 }
 
 static void
@@ -1683,12 +2168,21 @@ fuse_rename_resume (fuse_state_t *state)
 static void
 fuse_rename (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_rename_in  *fri = msg;
-        char *oldname = (char *)(fri + 1);
-        char *newname = oldname + strlen (oldname) + 1;
-        fuse_state_t *state = NULL;
+        struct fuse_rename_in *fri     = msg;
+        char                  *oldname = (char *)(fri + 1);
+        char                  *newname = oldname + strlen (oldname) + 1;
+        fuse_state_t          *state   = NULL;
+        fuse_private_t        *priv    = NULL;
+        int                    ret     = -1;
 
         GET_STATE (this, finh, state);
+        priv = this->private;
+
+        if (priv->aux_gfid_mount) {
+                ret = fuse_handle_gfiddir_op (this, state, msg, GF_FOP_RENAME);
+                if (ret == 0)
+                        return;
+        }
 
         fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, oldname);
 
@@ -1732,11 +2226,20 @@ fuse_link_resume (fuse_state_t *state)
 static void
 fuse_link (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_link_in *fli = msg;
-        char         *name = (char *)(fli + 1);
-        fuse_state_t *state = NULL;
+        struct fuse_link_in *fli        = msg;
+        char                *name       = (char *)(fli + 1);
+        fuse_state_t        *state      = NULL;
+        fuse_private_t      *priv       = NULL;
+        int                  ret        = -1;
 
         GET_STATE (this, finh, state);
+        priv = this->private;
+
+        if (priv->aux_gfid_mount) {
+                ret = fuse_handle_gfiddir_op (this, state, msg, GF_FOP_LINK);
+                if (ret == 0)
+                        return;
+        }
 
         fuse_resolve_inode_init (state, &state->resolve2, fli->oldnodeid);
 
@@ -1801,7 +2304,8 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 inode_unref (linked_inode);
 
-                feo.nodeid = inode_to_fuse_nodeid (linked_inode);
+                feo.nodeid = inode_to_fuse_nodeid (this, linked_inode,
+                                                   GF_FUSE_PATH_NAMESPACE);
 
                 feo.entry_valid = calc_timeout_sec (priv->entry_timeout);
                 feo.entry_valid_nsec = calc_timeout_nsec (priv->entry_timeout);
@@ -1907,9 +2411,8 @@ fuse_create_resume (fuse_state_t *state)
                 "%"PRIu64": CREATE %s", state->finh->unique,
                 state->loc.path);
 
-        FUSE_FOP (state, fuse_create_cbk, GF_FOP_CREATE,
-                  create, &state->loc, state->flags, state->mode,
-                  state->umask, fd, state->xdata);
+        FUSE_FOP (state, fuse_create_cbk, GF_FOP_CREATE, create, &state->loc,
+                  state->flags, state->mode, state->umask, fd, state->xdata);
 
 }
 
@@ -2382,6 +2885,11 @@ fuse_opendir_resume (fuse_state_t *state)
 static void
 fuse_opendir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
+        fuse_private_t *priv = NULL;
+        int             ret  = -1;
+
+        priv = this->private;
+
         /*
         struct fuse_open_in *foi = msg;
          */
@@ -2390,6 +2898,12 @@ fuse_opendir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
+        if (priv->aux_gfid_mount) {
+                ret = fuse_handle_gfiddir_op (this, state, msg, GF_FOP_OPENDIR);
+                if (ret == 0)
+                        return;
+        }
+
         fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         fuse_resolve_and_resume (state, fuse_opendir_resume);
@@ -2535,14 +3049,14 @@ fuse_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
 		   dict_t *xdata)
 {
-	fuse_state_t *state = NULL;
-	fuse_in_header_t *finh = NULL;
-	int           size = 0;
-	char         *buf = NULL;
-	gf_dirent_t  *entry = NULL;
-	struct fuse_direntplus *fde = NULL;
-	struct fuse_entry_out *feo = NULL;
-	fuse_private_t         *priv   = NULL;
+	fuse_state_t           *state = NULL;
+	fuse_in_header_t       *finh  = NULL;
+	int                     size  = 0;
+	char                   *buf   = NULL;
+	gf_dirent_t            *entry = NULL;
+	struct fuse_direntplus *fde   = NULL;
+	struct fuse_entry_out  *feo   = NULL;
+	fuse_private_t         *priv  = NULL;
 
 	state = frame->root->state;
 	finh  = state->finh;
@@ -2597,16 +3111,23 @@ fuse_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 			continue;
 
 		entry->d_stat.ia_blksize = this->ctx->page_size;
-		gf_fuse_stat2attr (&entry->d_stat, &feo->attr, priv->enable_ino32);
+		gf_fuse_stat2attr (&entry->d_stat, &feo->attr,
+                                   priv->enable_ino32);
 
-		linked_inode = inode_link (entry->inode, state->fd->inode,
-					   entry->d_name, &entry->d_stat);
+                linked_inode = inode_link (entry->inode,
+                                           state->fd->inode,
+                                           entry->d_name,
+                                           &entry->d_stat);
 		if (!linked_inode)
 			continue;
 
 		inode_lookup (linked_inode);
 
-		feo->nodeid = inode_to_fuse_nodeid (linked_inode);
+                /* we fail opendir on /.gfid and hence there won't
+                 * be readdir in that directory.
+                 */
+		feo->nodeid = inode_to_fuse_nodeid (this, linked_inode,
+                                                    GF_FUSE_PATH_NAMESPACE);
 
 		inode_unref (linked_inode);
 
@@ -2859,7 +3380,6 @@ fuse_statfs_resume (fuse_state_t *state)
                   statfs, &state->loc, state->xdata);
 }
 
-
 static void
 fuse_statfs (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -2872,10 +3392,363 @@ fuse_statfs (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fuse_resolve_and_resume (state, fuse_statfs_resume);
 }
 
+void
+fuse_auxgfid_heal_args_free (fuse_auxgfid_heal_args_t *args)
+{
+        if (args == NULL)
+                goto out;
+
+        GF_FREE (args->bname);
+
+        mem_put (args);
+out:
+        return;
+}
+
+fuse_auxgfid_heal_args_t *
+fuse_auxgfid_heal_parse_args (xlator_t *this, data_t *data)
+{
+        fuse_auxgfid_heal_args_t *args = NULL;
+        fuse_private_t           *priv = NULL;
+        void                     *blob = NULL;
+        int                       len  = 0, blob_len;
+
+        blob = data->data;
+        blob_len = data->len;
+
+        priv = this->private;
+
+        /* bname should at least contain a character */
+        if (blob_len < (sizeof (args->gfid) + 2))
+                goto err;
+
+        args = mem_get0 (priv->auxgfid_heal_args_pool);
+        if (args == NULL)
+                goto err;
+
+        memcpy (args->gfid, blob, sizeof (args->gfid));
+        blob += sizeof (args->gfid);
+        blob_len -= sizeof (args->gfid);
+
+        len = strnlen (blob, blob_len);
+        if (len == blob_len)
+                goto err;
+
+        args->bname = GF_CALLOC (1, len + 1, gf_common_mt_char);
+        if (args->bname == NULL)
+                goto err;
+
+        memcpy (args->bname, blob, len);
+        blob_len -= (len + 1);
+
+        if (blob_len)
+                goto err;
+
+        return args;
+
+err:
+        if (args)
+                fuse_auxgfid_heal_args_free (args);
+
+        return NULL;
+}
+
+static int
+fuse_auxgfid_newentry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno,
+                           inode_t *inode, struct iatt *buf,
+                           struct iatt *preparent, struct iatt *postparent,
+                           dict_t *xdata)
+{
+        /* don't worry about inode linking and other stuff. They'll happen on
+         * the next lookup.
+         */
+        fuse_err_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+static int
+fuse_auxgfid_heal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       inode_t *inode, struct iatt *stat, dict_t *dict,
+                       struct iatt *postparent)
+{
+        /* don't worry about inode linking and other stuff. They'll happen on
+         * the next lookup.
+         */
+        fuse_err_cbk (frame, cookie, this, op_ret, op_errno, NULL);
+        return 0;
+}
+
+void
+fuse_auxgfid_newfile_args_free (fuse_auxgfid_newfile_args_t *args)
+{
+        if (args == NULL)
+                goto out;
+
+        if (args->bname)
+                GF_FREE (args->bname);
+
+        if (S_ISLNK (args->st_mode) && args->args.symlink.linkpath) {
+                GF_FREE (args->args.symlink.linkpath);
+                args->args.symlink.linkpath = NULL;
+        }
+
+        mem_put (args);
+out:
+        return;
+}
+
+fuse_auxgfid_newfile_args_t *
+fuse_auxgfid_newfile_parse_args (xlator_t *this, data_t *data)
+{
+        fuse_auxgfid_newfile_args_t *args    = NULL;
+        fuse_private_t              *priv    = NULL;
+        int                          len     = 0, blob_len = 0;
+        int                          min_len = 0;
+        void                        *blob    = NULL;
+
+        priv = this->private;
+
+        blob = data->data;
+        blob_len = data->len;
+
+        min_len = sizeof (args->uid) + sizeof (args->gid) + sizeof (args->gfid)
+                + sizeof (args->st_mode) + 2 + 2;
+        if (blob_len < min_len)
+                goto err;
+
+
+        args = mem_get0 (priv->auxgfid_newfile_args_pool);
+        if (args == NULL)
+                goto err;
+
+        args->uid = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        args->gid = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        memcpy (args->gfid, blob, sizeof (args->gfid));
+        blob += sizeof (args->gfid);
+        blob_len -= sizeof (args->gfid);
+
+        args->st_mode = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        len = strnlen (blob, blob_len);
+        if (len == blob_len)
+                goto err;
+
+        args->bname = GF_CALLOC (1, (len + 1), gf_common_mt_char);
+        if (args->bname == NULL)
+                goto err;
+
+        memcpy (args->bname, blob, (len + 1));
+        blob += (len + 1);
+        blob_len -= (len + 1);
+
+        if (S_ISDIR (args->st_mode)) {
+                if (blob_len < sizeof (uint32_t))
+                        goto err;
+                args->args.mkdir.mode = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t))
+                        goto err;
+                args->args.mkdir.umask = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+                if (blob_len < 0)
+                        goto err;
+
+        } else if (S_ISLNK (args->st_mode)) {
+                len = strnlen (blob, blob_len);
+                if (len == blob_len)
+                        goto err;
+
+                args->args.symlink.linkpath = GF_CALLOC (1, len + 1,
+                                                         gf_common_mt_char);
+                if (args->args.symlink.linkpath == NULL)
+                        goto err;
+
+                memcpy (args->args.symlink.linkpath, blob, (len + 1));
+                blob += (len + 1);
+                blob_len -= (len + 1);
+        } else {
+                if (blob_len < sizeof (uint32_t))
+                        goto err;
+                args->args.mknod.mode = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t))
+                        goto err;
+                args->args.mknod.rdev = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t))
+                        goto err;
+                args->args.mknod.umask = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+        }
+
+        if (blob_len)
+                goto err;
+
+        return args;
+
+err:
+        if (args)
+                fuse_auxgfid_newfile_args_free (args);
+
+        return NULL;
+}
+
+int
+fuse_auxgfid_loc_fill (fuse_state_t *state, char *gfid, char *bname)
+{
+        inode_t *parent = NULL;
+        int      ret    = -1;
+
+        parent = inode_ref (state->loc.inode);
+        loc_wipe (&state->loc);
+        state->loc.parent = parent;
+        state->loc.inode = inode_grep (parent->table, parent, bname);
+        if (state->loc.inode == NULL) {
+                state->loc.inode = inode_new (parent->table);
+                if (state->loc.inode == NULL)
+                        goto out;
+        }
+
+        loc_path (&state->loc, bname);
+        state->loc.name = basename (state->loc.path);
+
+        if (state->xdata == NULL)
+                state->xdata = dict_new ();
+
+        uuid_parse (gfid, state->gfid);
+        uuid_copy (state->loc.gfid, state->gfid);
+
+        ret = dict_set_static_bin (state->xdata, "gfid-req", state->gfid, 16);
+        if (ret < 0) {
+                gf_log (state->this->name, GF_LOG_WARNING,
+                        "SETXATTR %"PRId64"/%s (%s/%s) setting gfid in xdata "
+                        "failed. Hence, file created will not have the gfid "
+                        "provided (%s)", state->finh->nodeid,
+                        bname, uuid_utoa (state->loc.parent->gfid),
+                        bname, uuid_utoa (state->gfid));
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int
+fuse_auxgfid_heal (fuse_state_t *state, data_t *data)
+{
+        fuse_auxgfid_heal_args_t *args      = NULL;
+        int                       ret       = -1;
+
+        args = fuse_auxgfid_heal_parse_args (state->this, data);
+
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": LOOKUP %s", state->finh->unique,
+                state->loc.path);
+
+        ret = fuse_auxgfid_loc_fill (state, args->gfid, args->bname);
+        if (ret < 0) {
+                goto err;
+        }
+
+        FUSE_FOP (state, fuse_auxgfid_heal_cbk, GF_FOP_LOOKUP,
+                  lookup, &state->loc, state->xdata);
+
+        fuse_auxgfid_heal_args_free (args);
+
+        return 0;
+
+err:
+        fuse_auxgfid_heal_args_free (args);
+        send_fuse_err (state->this, state->finh, ENOMEM);
+        free_fuse_state (state);
+        return 0;
+}
+
+int
+fuse_auxgfid_newentry (fuse_state_t *state, data_t *data)
+{
+        int                          ret   = 0;
+        fuse_auxgfid_newfile_args_t *args  = NULL;
+
+        args = fuse_auxgfid_newfile_parse_args (state->this, data);
+        if (args == NULL) {
+                goto err;
+        }
+
+        state->finh->uid = args->uid;
+        state->finh->gid = args->gid;
+
+        ret = fuse_auxgfid_loc_fill (state, args->gfid, args->bname);
+        if (ret < 0) {
+                goto err;
+        }
+
+        if (S_ISDIR (args->st_mode)) {
+                FUSE_FOP (state, fuse_auxgfid_newentry_cbk,
+                          GF_FOP_MKDIR, mkdir, &state->loc,
+                          args->args.mkdir.mode, args->args.mkdir.umask,
+                          state->xdata);
+        } else if (S_ISLNK (args->st_mode)) {
+                FUSE_FOP (state, fuse_auxgfid_newentry_cbk,
+                          GF_FOP_SYMLINK, symlink, args->args.symlink.linkpath,
+                          &state->loc, 0, state->xdata);
+        } else {
+                if (S_ISREG (args->st_mode)) {
+                        ret = dict_set_uint32 (state->xdata,
+                                               GLUSTERFS_CREATE_MODE_KEY,
+                                               args->args.mknod.mode);
+                        if (ret < 0) {
+                                // failure
+                        }
+                        args->args.mknod.mode = IA_IFREG;
+                }
+
+                FUSE_FOP (state, fuse_auxgfid_newentry_cbk, GF_FOP_MKNOD,
+                          mknod, &state->loc, args->args.mknod.mode,
+                          args->args.mknod.rdev, args->args.mknod.umask,
+                          state->xdata);
+        }
+
+        fuse_auxgfid_newfile_args_free (args);
+        return 0;
+
+err:
+        fuse_auxgfid_newfile_args_free (args);
+        send_fuse_err (state->this, state->finh, ENOMEM);
+        free_fuse_state (state);
+        return 0;
+}
 
 void
 fuse_setxattr_resume (fuse_state_t *state)
 {
+        fuse_private_t *priv = NULL;
+        data_t         *data = NULL;
+
+        priv = state->this->private;
+
         if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                         "%"PRIu64": SETXATTR %s/%"PRIu64" (%s) "
@@ -2887,6 +3760,20 @@ fuse_setxattr_resume (fuse_state_t *state)
                 return;
         }
 
+        if (priv->aux_gfid_mount) {
+                data = dict_get (state->xattr, GF_FUSE_AUX_GFID_NEWFILE);
+                if (data != NULL) {
+                        fuse_auxgfid_newentry (state, data);
+                        return;
+                }
+
+                data = dict_get (state->xattr, GF_FUSE_AUX_GFID_HEAL);
+                if (data != NULL) {
+                        fuse_auxgfid_heal (state, data);
+                        return;
+                }
+        }
+
 #ifdef GF_TEST_FFOP
         state->fd = fd_lookup (state->loc.inode, state->finh->pid);
 #endif /* GF_TEST_FFOP */
@@ -2914,17 +3801,17 @@ fuse_setxattr_resume (fuse_state_t *state)
 static void
 fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_setxattr_in *fsi = msg;
-        char         *name = (char *)(fsi + 1);
-        char         *value = name + strlen (name) + 1;
-        struct fuse_private *priv = NULL;
-
-        fuse_state_t *state = NULL;
-        char         *dict_value = NULL;
-        int32_t       ret = -1;
-        char *newkey = NULL;
+        struct fuse_setxattr_in *fsi        = msg;
+        char                    *name       = (char *)(fsi + 1);
+        char                    *value      = name + strlen (name) + 1;
+        struct fuse_private     *priv       = NULL;
+        fuse_state_t            *state      = NULL;
+        char                    *dict_value = NULL;
+        int32_t                  ret        = -1;
+        char                    *newkey     = NULL;
 
         priv = this->private;
+        GET_STATE (this, finh, state);
 
 #ifdef GF_DARWIN_HOST_OS
         if (fsi->position) {
@@ -2938,6 +3825,13 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         }
 #endif
 
+        if (priv->aux_gfid_mount) {
+                ret = fuse_handle_gfiddir_op (this, state, msg,
+                                              GF_FOP_SETXATTR);
+                if (ret == 0)
+                        return;
+        }
+
         if (fuse_ignore_xattr_set (priv, name)) {
                 (void) send_fuse_err (this, finh, 0);
                 return;
@@ -2984,7 +3878,6 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 return;
         }
 
-        GET_STATE (this, finh, state);
         state->size = fsi->size;
 
         fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
@@ -3249,15 +4142,15 @@ fuse_getxattr_resume (fuse_state_t *state)
 static void
 fuse_getxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_getxattr_in *fgxi = msg;
-        char         *name = (char *)(fgxi + 1);
-
-        fuse_state_t *state = NULL;
-        struct fuse_private *priv = NULL;
-        int rv = 0;
-        char *newkey = NULL;
+        struct fuse_getxattr_in *fgxi     = msg;
+        char                    *name     = (char *)(fgxi + 1);
+        fuse_state_t            *state    = NULL;
+        struct fuse_private     *priv     = NULL;
+        int                      rv       = 0;
+        char                    *newkey   = NULL;
 
         priv = this->private;
+        GET_STATE (this, finh, state);
 
 #ifdef GF_DARWIN_HOST_OS
         if (fgxi->position) {
@@ -3279,6 +4172,12 @@ fuse_getxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         }
 #endif
 
+        if (priv->aux_gfid_mount) {
+                rv = fuse_handle_gfiddir_op (this, state, msg, GF_FOP_GETXATTR);
+                if (rv == 0)
+                        return;
+        }
+
         if (!priv->acl) {
                 if ((strcmp (name, "system.posix_acl_access") == 0) ||
                     (strcmp (name, "system.posix_acl_default") == 0)) {
@@ -3296,8 +4195,6 @@ fuse_getxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 }
         }
 
-        GET_STATE (this, finh, state);
-
         fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         rv = fuse_flip_xattr_ns (priv, name, &newkey);
@@ -3355,11 +4252,22 @@ fuse_listxattr_resume (fuse_state_t *state)
 static void
 fuse_listxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_getxattr_in *fgxi = msg;
-        fuse_state_t *state = NULL;
+        struct fuse_getxattr_in *fgxi  = msg;
+        fuse_state_t            *state = NULL;
+        fuse_private_t          *priv  = NULL;
+        int                      ret   = -1;
 
         GET_STATE (this, finh, state);
 
+        priv = this->private;
+
+        if (priv->aux_gfid_mount) {
+                ret = fuse_handle_gfiddir_op (this, state, msg,
+                                              GF_FOP_GETXATTR);
+                if (ret == 0)
+                        return;
+        }
+
         fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         state->size = fgxi->size;
@@ -3410,12 +4318,11 @@ fuse_removexattr_resume (fuse_state_t *state)
 static void
 fuse_removexattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        char *name = msg;
-
-        fuse_state_t *state = NULL;
-        fuse_private_t *priv = NULL;
-        int32_t       ret = -1;
-        char *newkey = NULL;
+        char           *name     = msg;
+        fuse_state_t   *state    = NULL;
+        fuse_private_t *priv     = NULL;
+        int32_t         ret      = -1;
+        char           *newkey   = NULL;
 
         if (!strcmp (GFID_XATTR_KEY, name)) {
                 send_fuse_err (this, finh, EPERM);
@@ -3427,6 +4334,13 @@ fuse_removexattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
+        if (priv->aux_gfid_mount) {
+                ret = fuse_handle_gfiddir_op (this, state, msg,
+                                              GF_FOP_REMOVEXATTR);
+                if (ret == 0)
+                        return;
+        }
+
         fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         ret = fuse_flip_xattr_ns (priv, name, &newkey);
@@ -5121,6 +6035,31 @@ init (xlator_t *this_xl)
 
         GF_OPTION_INIT ("selinux", priv->selinux, bool, cleanup_exit);
 
+
+#ifdef GF_LINUX_HOST_OS
+        GF_OPTION_INIT ("auxiliary-gfid-mount", priv->aux_gfid_mount, bool,
+                        cleanup_exit);
+#endif
+
+        if (priv->aux_gfid_mount) {
+                priv->fuse_nodeid_pool = mem_pool_new (gf_fuse_nodeid_t, 2048);
+                if (priv->fuse_nodeid_pool == NULL) {
+                        goto cleanup_exit;
+                }
+
+                priv->auxgfid_newfile_args_pool
+                        = mem_pool_new (fuse_auxgfid_newfile_args_t, 2048);
+                if (priv->auxgfid_newfile_args_pool == NULL) {
+                        goto cleanup_exit;
+                }
+
+                priv->auxgfid_heal_args_pool
+                        = mem_pool_new (fuse_auxgfid_heal_args_t, 2048);
+                if (priv->auxgfid_heal_args_pool == NULL) {
+                        goto cleanup_exit;
+                }
+        }
+
         GF_OPTION_INIT ("read-only", priv->read_only, bool, cleanup_exit);
 
         GF_OPTION_INIT ("enable-ino32", priv->enable_ino32, bool, cleanup_exit);
@@ -5284,6 +6223,16 @@ cleanup_exit:
                         close (priv->fd);
                 if (priv->fuse_dump_fd != -1)
                         close (priv->fuse_dump_fd);
+
+                if (priv->fuse_nodeid_pool != NULL)
+                        mem_pool_destroy (priv->fuse_nodeid_pool);
+
+                if (priv->auxgfid_newfile_args_pool != NULL)
+                        mem_pool_destroy (priv->auxgfid_newfile_args_pool);
+
+                if (priv->auxgfid_heal_args_pool != NULL)
+                        mem_pool_destroy (priv->auxgfid_heal_args_pool);
+
                 GF_FREE (priv);
         }
         GF_FREE (mnt_args);
@@ -5324,6 +6273,7 @@ struct xlator_fops fops;
 
 struct xlator_cbks cbks = {
         .invalidate = fuse_invalidate,
+        .forget     = fuse_xlator_forget,
 };
 
 
@@ -5410,5 +6360,13 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "no"
         },
+#ifdef GF_LINUX_HOST_OS
+        { .key  = {"auxiliary-gfid-mount"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "an option which makes the mount point allow "
+          "access to gfid directly",
+        },
+#endif
         { .key = {NULL} },
 };
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index 2626638dc..d305d013a 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -30,6 +30,7 @@
 #include "defaults.h"
 #include "common-utils.h"
 #include "statedump.h"
+#include "glfs-fuse-bridge.h"
 
 #ifdef GF_DARWIN_HOST_OS
 /* This is MacFUSE's marker for MacFUSE-specific code */
@@ -50,12 +51,26 @@
 #if defined(GF_LINUX_HOST_OS) || defined(__NetBSD__)
 #define FUSE_OP_HIGH (FUSE_READDIRPLUS + 1)
 #endif
+
 #ifdef GF_DARWIN_HOST_OS
 #define FUSE_OP_HIGH (FUSE_DESTROY + 1)
 #endif
 #define GLUSTERFS_XATTR_LEN_MAX  65536
 
 #define MAX_FUSE_PROC_DELAY 1
+#define GF_GFID_KEY "GLUSTERFS_GFID"
+#define GF_GFID_DIR ".gfid"
+#define GF_AUX_GFID 0xd
+
+typedef enum {
+        GF_FUSE_PATH_NAMESPACE,
+        GF_FUSE_GFID_NAMESPACE
+} gf_lookup_namespace_t;
+
+typedef struct {
+        inode_t *inode_path_ns;
+        inode_t *inode_gfid_ns;
+} gf_fuse_nodeid_t;
 
 typedef struct fuse_in_header fuse_in_header_t;
 typedef void (fuse_handler_t) (xlator_t *this, fuse_in_header_t *finh,
@@ -122,6 +137,14 @@ struct fuse_private {
 
         /* for using fuse-kernel readdirp*/
         gf_boolean_t use_readdirp;
+
+        gf_boolean_t aux_gfid_mount;
+        /* root inode's stbuf */
+        struct iatt root_stbuf;
+        struct iatt gfiddir_stbuf;
+        struct mem_pool *fuse_nodeid_pool;
+        struct mem_pool *auxgfid_newfile_args_pool;
+        struct mem_pool *auxgfid_heal_args_pool;
 };
 typedef struct fuse_private fuse_private_t;
 
@@ -509,7 +532,8 @@ void gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa,
                         gf_boolean_t enable_ino32);
 void gf_fuse_fill_dirent (gf_dirent_t *entry, struct fuse_dirent *fde,
                           gf_boolean_t enable_ino32);
-uint64_t inode_to_fuse_nodeid (inode_t *inode);
+uint64_t
+inode_to_fuse_nodeid (xlator_t *this, inode_t *inode, gf_lookup_namespace_t ns);
 xlator_t *fuse_active_subvol (xlator_t *fuse);
 inode_t *fuse_ino_to_inode (uint64_t ino, xlator_t *fuse);
 int send_fuse_err (xlator_t *this, fuse_in_header_t *finh, int error);
diff --git a/xlators/mount/fuse/src/fuse-helpers.c b/xlators/mount/fuse/src/fuse-helpers.c
index d4dcc2e61..4ea3094d1 100644
--- a/xlators/mount/fuse/src/fuse-helpers.c
+++ b/xlators/mount/fuse/src/fuse-helpers.c
@@ -317,12 +317,31 @@ get_call_frame_for_req (fuse_state_t *state)
         return frame;
 }
 
+inode_t *
+fuse_ino_to_inode_gfid_mount (uint64_t ino, xlator_t *fuse)
+{
+        inode_t  **ptr           = NULL, *inode = NULL;
+        xlator_t  *active_subvol = NULL;
+
+        if (ino == 1) {
+                active_subvol = fuse_active_subvol (fuse);
+                if (active_subvol)
+                        inode = active_subvol->itable->root;
+        } else {
+                ptr = (void *)ino;
+
+                if (ptr != NULL)
+                        inode = inode_ref (*ptr);
+        }
+
+        return inode;
+}
 
 inode_t *
-fuse_ino_to_inode (uint64_t ino, xlator_t *fuse)
+fuse_ino_to_inode_normal_mount (uint64_t ino, xlator_t *fuse)
 {
-        inode_t  *inode = NULL;
         xlator_t *active_subvol = NULL;
+        inode_t  *inode         = NULL;
 
         if (ino == 1) {
                 active_subvol = fuse_active_subvol (fuse);
@@ -336,17 +355,98 @@ fuse_ino_to_inode (uint64_t ino, xlator_t *fuse)
         return inode;
 }
 
-uint64_t
-inode_to_fuse_nodeid (inode_t *inode)
+inode_t *
+fuse_ino_to_inode (uint64_t ino, xlator_t *fuse)
+{
+        inode_t        *inode         = NULL;
+        fuse_private_t *priv          = NULL;
+
+        priv = fuse->private;
+
+        if (priv->aux_gfid_mount) {
+                inode = fuse_ino_to_inode_gfid_mount (ino, fuse);
+        } else {
+                inode = fuse_ino_to_inode_normal_mount (ino, fuse);
+        }
+
+        return inode;
+}
+
+inline uint64_t
+inode_to_fuse_nodeid_gfid_mount (xlator_t *this, inode_t *inode,
+                                 gf_lookup_namespace_t ns)
+{
+        inode_t          **ptr    = NULL;
+        gf_fuse_nodeid_t  *nodeid = NULL;
+        fuse_private_t    *priv   = NULL;
+        uint64_t           value  = 0;
+        int32_t            ret    = 0;
+
+        priv = this->private;
+
+        LOCK (&inode->lock);
+        {
+                __inode_ctx_get (inode, this, &value);
+                nodeid = (void *)value;
+
+                if (nodeid == NULL) {
+                        nodeid = mem_get0 (priv->fuse_nodeid_pool);
+                        if (nodeid == NULL)
+                                goto unlock;
+
+                        ret = __inode_ctx_set (inode, this, (uint64_t *)nodeid);
+                        if (ret < 0)
+                                goto unlock;
+
+                        nodeid->inode_path_ns = nodeid->inode_gfid_ns = inode;
+                }
+        }
+unlock:
+        UNLOCK (&inode->lock);
+
+        if (ret < 0) {
+                mem_put (nodeid);
+                nodeid = NULL;
+        }
+
+        if (nodeid != NULL) {
+                if (ns == GF_FUSE_GFID_NAMESPACE)
+                        ptr = &nodeid->inode_gfid_ns;
+                else
+                        ptr = &nodeid->inode_path_ns;
+        }
+
+        return (uint64_t) ptr;
+}
+
+inline uint64_t
+inode_to_fuse_nodeid_normal_mount (inode_t *inode)
 {
-        if (!inode)
-		return 0;
 	if (__is_root_gfid (inode->gfid))
                 return 1;
 
         return (unsigned long) inode;
 }
 
+uint64_t
+inode_to_fuse_nodeid (xlator_t *this, inode_t *inode, gf_lookup_namespace_t ns)
+{
+        fuse_private_t *priv = NULL;
+        uint64_t        ino  = 0;
+
+        priv = this->private;
+
+        if (!inode)
+		return 0;
+
+        if (priv->aux_gfid_mount)
+                ino = inode_to_fuse_nodeid_gfid_mount (this, inode, ns);
+        else
+                ino = inode_to_fuse_nodeid_normal_mount (inode);
+
+        return ino;
+}
+
 
 GF_MUST_CHECK int32_t
 fuse_loc_fill (loc_t *loc, fuse_state_t *state, ino_t ino,
@@ -580,7 +680,8 @@ fuse_ignore_xattr_set (fuse_private_t *priv, char *key)
               || (fnmatch ("*.glusterfs.volume-mark",
                            key, FNM_PERIOD) == 0)
               || (fnmatch ("*.glusterfs.volume-mark.*",
-                           key, FNM_PERIOD) == 0)))
+                           key, FNM_PERIOD) == 0)
+              || (fnmatch ("glusterfs.gfid.newfile", key, FNM_PERIOD) == 0)))
                 ret = -1;
 
  out:
diff --git a/xlators/mount/fuse/src/glfs-fuse-bridge.h b/xlators/mount/fuse/src/glfs-fuse-bridge.h
new file mode 100644
index 000000000..98085b83d
--- /dev/null
+++ b/xlators/mount/fuse/src/glfs-fuse-bridge.h
@@ -0,0 +1,53 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLFS_FUSE_BRIDGE_H
+#define _GLFS_FUSE_BRIDGE_H
+
+#define UUID_CANONICAL_FORM_LEN 36
+
+#define GF_FUSE_AUX_GFID_NEWFILE "glusterfs.gfid.newfile"
+#define GF_FUSE_AUX_GFID_HEAL    "glusterfs.gfid.heal"
+
+struct auxfuse_symlink_in {
+        char     *linkpath;
+} __attribute__ ((__packed__));
+
+struct auxfuse_mknod_in {
+	unsigned int mode;
+	unsigned int rdev;
+	unsigned int umask;
+} __attribute__ ((__packed__));
+
+struct auxfuse_mkdir_in {
+	unsigned int mode;
+	unsigned int umask;
+} __attribute__ ((__packed__));
+
+typedef struct {
+        unsigned int  uid;
+        unsigned int  gid;
+        char          gfid[UUID_CANONICAL_FORM_LEN + 1];
+        unsigned int  st_mode;
+        char         *bname;
+
+        union {
+                struct auxfuse_mkdir_in   mkdir;
+                struct auxfuse_mknod_in   mknod;
+                struct auxfuse_symlink_in symlink;
+        } __attribute__ ((__packed__)) args;
+} __attribute__((__packed__)) fuse_auxgfid_newfile_args_t;
+
+typedef struct {
+        char      gfid[UUID_CANONICAL_FORM_LEN + 1];
+        char     *bname; /* a null terminated basename */
+} __attribute__((__packed__)) fuse_auxgfid_heal_args_t;
+
+#endif
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index c0a2ed1a7..0f784835f 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -122,6 +122,10 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --mem-accounting");
     fi
 
+    if [ -n "$aux_gfid_mount" ]; then
+        cmd_line=$(echo "$cmd_line --aux-gfid-mount");
+    fi
+
 #options with values start here
     if [ -n "$log_level" ]; then
         cmd_line=$(echo "$cmd_line --log-level=$log_level");
@@ -268,13 +272,13 @@ mount.glusterfs --version"
 # check for recursive mounts. i.e, mounting over an existing brick
 check_recursive_mount ()
 {
-    if [ $2 = "/" ]; then
+    if [ $1 = "/" ]; then
         echo Cannot mount over root;
         exit 2;
     fi
     # GFID check first
     # remove trailing / from mount point
-    mnt_dir=${2%/};
+    mnt_dir=${1%/};
 
     export PATH;
     # check whether getfattr exists
@@ -333,7 +337,6 @@ check_recursive_mount ()
 main ()
 {
     helper=$(echo "$@" | sed -n 's/.*\--[ ]*\([^ ]*\).*/\1/p');
-
     in_opt="no"
     pos_args=0
     for opt in "$@"; do
@@ -348,6 +351,11 @@ main ()
 		    "fopen-keep-cache")	fopen_keep_cache=1 ;;
                     "enable-ino32")     enable_ino32=1 ;;
                     "mem-accounting")   mem_accounting=1;;
+                    "aux-gfid-mount")
+                        if [ `uname -s` = "Linux" ]; then
+                            aux_gfid_mount=1
+                        fi
+                        ;;
                     # "mount -t glusterfs" sends this, but it's useless.
                     "rw")       ;;
                     # these ones are interpreted during system initialization
@@ -437,7 +445,7 @@ main ()
         exit 0;
     fi
 
-    check_recursive_mount "$@";
+    check_recursive_mount "$mount_point";
 
     # Append fuse.glusterfs to PRUNEFS variable in updatedb.conf(5). updatedb(8)
     # should not index files under GlusterFS, indexing will slow down GlusteFS
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 399c94bf0..fee2ce9d7 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -842,6 +842,7 @@ posix_mknod (call_frame_t *frame, xlator_t *this,
         struct iatt           preparent = {0,};
         struct iatt           postparent = {0,};
         void *                uuid_req  = NULL;
+        mode_t                st_mode   = 0;
 
         DECLARE_OLD_FS_ID_VAR;
 
@@ -917,6 +918,19 @@ real_op:
                                 strerror (op_errno));
                         goto out;
                 }
+        } else {
+                op_ret = dict_get_uint32 (xdata, GLUSTERFS_CREATE_MODE_KEY,
+                                          &st_mode);
+
+                if (op_ret >= 0) {
+                        op_ret = chmod (real_path, st_mode);
+                        if (op_ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "chmod failed (%s)", strerror (errno));
+                        }
+
+                        dict_del (xdata, GLUSTERFS_CREATE_MODE_KEY);
+                }
         }
 
         op_ret = posix_gfid_set (this, real_path, loc, xdata);