Add GF_FOP_IPC for inter-translator communication.

Several features - e.g. encryption, erasure codes, or NSR - involve
multiple cooperating translators which sometimes need a "private" means
of communication amongst themselves.  Historically we've used virtual or
synthetic xattrs, but that's not very elegant and clutters up the
getxattr/setxattr path which must also handle real xattr requests.  This
new fop should address that.

The only argument is an int32_t "op" which should be recognized by the
target translator.  It is recommended that translators using these
feature follow some convention regarding the ops that they define, to
avoid conflicts.  Using a hash of the target translator's type string as
a base for a series of ops would probably be a good start.  Any other
information can be passed in both directions using xdata.

The default behavior for this fop, as with any other, is to pass through
to FIRST_CHILD.  That makes use of this fop "transparent" to other
translators that were written before it existed, but it also means that
it only really works with pass-through translators.  If a routing
translator (such as DHT) or a fan-out translator (such as AFR) is
involved, the IPC might not reach its intended destination unless those
translators are modified to forward IPC fops along all paths.

If an IPC gets all the way to storage/posix it is considered an error,
much like an uncaught exception.  We don't actually *do* anything in
that case, but we do flag it as an error in the log.

Change-Id: I7f37c9247ee35536f8136c7aea758e6fe04616c4
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>

21 files changed, 476 insertions, 48 deletions

diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index 326530578..37e8d22d8 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -3166,6 +3166,29 @@ out:
 }
 
 int
+glfs_ipc (struct glfs *fs, int32_t op)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+        ret = syncop_ipc (subvol, op);
+        DECODE_SYNCOP_ERR (ret);
+
+out:
+	glfs_subvol_done (fs, subvol);
+	return ret;
+}
+
+int
 glfs_chdir (struct glfs *fs, const char *path)
 {
 	int              ret = -1;
diff --git a/api/src/glfs.h b/api/src/glfs.h
index d79385792..9feaa8015 100644
--- a/api/src/glfs.h
+++ b/api/src/glfs.h
@@ -639,6 +639,8 @@ glfs_setattr (struct glfs *fs, const char *path, struct iatt *iatt,
 int
 glfs_fsetattr (struct glfs_fd *glfd, struct iatt *iatt, int valid);
 
+int glfs_ipc (struct glfs *fs, int32_t op) __THROW;
+
 
 
 
diff --git a/libglusterfs/src/call-stub.c b/libglusterfs/src/call-stub.c
index 86c2463ef..ee2e7c933 100644
--- a/libglusterfs/src/call-stub.c
+++ b/libglusterfs/src/call-stub.c
@@ -2297,6 +2297,52 @@ out:
 }
 
 
+call_stub_t *
+fop_ipc_cbk_stub (call_frame_t *frame, fop_ipc_cbk_t fn,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        call_stub_t *stub = NULL;
+
+        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+
+        stub = stub_new (frame, 0, GF_FOP_IPC);
+        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+
+        stub->fn_cbk.ipc = fn;
+
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
+
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
+out:
+        return stub;
+}
+
+call_stub_t *
+fop_ipc_stub (call_frame_t *frame, fop_ipc_t fn,
+              int32_t op, dict_t *xdata)
+{
+        call_stub_t *stub = NULL;
+
+        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
+
+        stub = stub_new (frame, 1, GF_FOP_IPC);
+        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+
+        stub->fn.ipc = fn;
+
+        stub->args.cmd = op;
+
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
+out:
+        return stub;
+
+}
+
+
 void
 call_resume_wind (call_stub_t *stub)
 {
@@ -2529,6 +2575,10 @@ call_resume_wind (call_stub_t *stub)
                                  stub->args.fd, stub->args.offset,
                                  stub->args.size, stub->args.xdata);
                 break;
+        case GF_FOP_IPC:
+                stub->fn.ipc (stub->frame, stub->frame->this,
+                              stub->args.cmd, stub->args.xdata);
+                break;
 
         default:
                 gf_log_callingfn ("call-stub", GF_LOG_ERROR,
@@ -2736,6 +2786,9 @@ call_resume_unwind (call_stub_t *stub)
                 STUB_UNWIND(stub, zerofill, &stub->args_cbk.prestat,
                             &stub->args_cbk.poststat, stub->args_cbk.xdata);
                 break;
+        case GF_FOP_IPC:
+                STUB_UNWIND (stub, ipc, stub->args_cbk.xdata);
+                break;
 
         default:
                 gf_log_callingfn ("call-stub", GF_LOG_ERROR,
diff --git a/libglusterfs/src/call-stub.h b/libglusterfs/src/call-stub.h
index ccf92cf53..e404c8dda 100644
--- a/libglusterfs/src/call-stub.h
+++ b/libglusterfs/src/call-stub.h
@@ -72,6 +72,7 @@ typedef struct {
 		fop_fallocate_t fallocate;
 		fop_discard_t discard;
                 fop_zerofill_t zerofill;
+                fop_ipc_t ipc;
 	} fn;
 
 	union {
@@ -119,6 +120,7 @@ typedef struct {
 		fop_fallocate_cbk_t fallocate;
 		fop_discard_cbk_t discard;
                 fop_zerofill_cbk_t zerofill;
+                fop_ipc_cbk_t ipc;
 	} fn_cbk;
 
 	struct {
@@ -761,6 +763,13 @@ fop_zerofill_cbk_stub(call_frame_t *frame,
                      struct iatt *statpre, struct iatt *statpost,
                      dict_t *xdata);
 
+call_stub_t *
+fop_ipc_stub (call_frame_t *frame, fop_ipc_t fn, int32_t op, dict_t *xdata);
+
+call_stub_t *
+fop_ipc_cbk_stub (call_frame_t *frame, fop_ipc_cbk_t fn,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
 void call_resume (call_stub_t *stub);
 void call_stub_destroy (call_stub_t *stub);
 void call_unwind_error (call_stub_t *stub, int op_ret, int op_errno);
diff --git a/libglusterfs/src/defaults.c b/libglusterfs/src/defaults.c
index f5fb2aa9d..07c7d3de4 100644
--- a/libglusterfs/src/defaults.c
+++ b/libglusterfs/src/defaults.c
@@ -837,6 +837,16 @@ default_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
+
+int32_t
+default_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (ipc, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
 /* RESUME */
 
 int32_t
@@ -1268,6 +1278,17 @@ default_zerofill_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
 }
 
 
+int32_t
+default_ipc_resume (call_frame_t *frame, xlator_t *this, int32_t op,
+                    dict_t *xdata)
+{
+        STACK_WIND (frame, default_ipc_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->ipc,
+                    op, xdata);
+        return 0;
+}
+
+
 /* FOPS */
 
 int32_t
@@ -1704,6 +1725,16 @@ default_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 
 int32_t
+default_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+        STACK_WIND_TAIL (frame,
+                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->ipc,
+                         op, xdata);
+        return 0;
+}
+
+
+int32_t
 default_forget (xlator_t *this, inode_t *inode)
 {
         gf_log_callingfn (this->name, GF_LOG_WARNING, "xlator does not "
diff --git a/libglusterfs/src/defaults.h b/libglusterfs/src/defaults.h
index f0b786ddf..6b3c90719 100644
--- a/libglusterfs/src/defaults.h
+++ b/libglusterfs/src/defaults.h
@@ -261,6 +261,9 @@ int32_t default_zerofill(call_frame_t *frame,
                         off_t offset,
                         off_t len, dict_t *xdata);
 
+int32_t default_ipc (call_frame_t *frame, xlator_t *this, int32_t op,
+                     dict_t *xdata);
+
 
 /* Resume */
 int32_t default_getspec_resume (call_frame_t *frame,
@@ -490,6 +493,9 @@ int32_t default_zerofill_resume(call_frame_t *frame,
                                off_t offset,
                                off_t len, dict_t *xdata);
 
+int32_t default_ipc_resume (call_frame_t *frame, xlator_t *this,
+                            int32_t op, dict_t *xdata);
+
 
 /* _cbk */
 
@@ -713,6 +719,9 @@ int32_t default_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                             int32_t op_ret, int32_t op_errno, struct iatt *pre,
                             struct iatt *post, dict_t *xdata);
 
+int32_t default_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
 int32_t
 default_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, char *spec_data);
diff --git a/libglusterfs/src/globals.c b/libglusterfs/src/globals.c
index 259c5c885..0bb5ea9f4 100644
--- a/libglusterfs/src/globals.c
+++ b/libglusterfs/src/globals.c
@@ -70,7 +70,8 @@ const char *gf_fop_list[GF_FOP_MAXVALUE] = {
         [GF_FOP_FREMOVEXATTR]= "FREMOVEXATTR",
 	[GF_FOP_FALLOCATE]   = "FALLOCATE",
 	[GF_FOP_DISCARD]     = "DISCARD",
-        [GF_FOP_ZEROFILL]     = "ZEROFILL",
+        [GF_FOP_ZEROFILL]    = "ZEROFILL",
+        [GF_FOP_IPC]         = "IPC",
 };
 /* THIS */
 
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index f537110d7..0d4a4564f 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -237,6 +237,7 @@ typedef enum {
 	GF_FOP_FALLOCATE,
 	GF_FOP_DISCARD,
         GF_FOP_ZEROFILL,
+        GF_FOP_IPC,
         GF_FOP_MAXVALUE,
 } glusterfs_fop_t;
 
diff --git a/libglusterfs/src/syncop.c b/libglusterfs/src/syncop.c
index 552a26355..a46a6603f 100644
--- a/libglusterfs/src/syncop.c
+++ b/libglusterfs/src/syncop.c
@@ -2481,6 +2481,36 @@ syncop_zerofill(xlator_t *subvol, fd_t *fd, off_t offset, off_t len)
 
 
 int
+syncop_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, dict_t *xdata)
+{
+        struct syncargs *args = NULL;
+
+        args = cookie;
+
+        args->op_ret   = op_ret;
+        args->op_errno = op_errno;
+
+        __wake (args);
+
+        return 0;
+}
+
+int
+syncop_ipc (xlator_t *subvol, int32_t op)
+{
+        struct syncargs args = {0, };
+
+        SYNCOP (subvol, (&args), syncop_ipc_cbk, subvol->fops->ipc,
+                op, NULL);
+
+        if (args.op_ret < 0)
+                return -args.op_errno;
+        return args.op_ret;
+}
+
+
+int
 syncop_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 	       int op_ret, int op_errno, struct gf_flock *flock,
 	       dict_t *xdata)
diff --git a/libglusterfs/src/syncop.h b/libglusterfs/src/syncop.h
index fb867a108..8760f4b68 100644
--- a/libglusterfs/src/syncop.h
+++ b/libglusterfs/src/syncop.h
@@ -450,4 +450,6 @@ int syncop_rename_with_xdata (xlator_t *subvol, loc_t *oldloc, loc_t *newloc, di
 
 int syncop_lk (xlator_t *subvol, fd_t *fd, int cmd, struct gf_flock *flock);
 
+int syncop_ipc (xlator_t *subvol, int op);
+
 #endif /* _SYNCOP_H */
diff --git a/libglusterfs/src/xlator.c b/libglusterfs/src/xlator.c
index f3df8e2ae..da62db0a5 100644
--- a/libglusterfs/src/xlator.c
+++ b/libglusterfs/src/xlator.c
@@ -82,6 +82,7 @@ fill_defaults (xlator_t *xl)
 	SET_DEFAULT_FOP (fallocate);
 	SET_DEFAULT_FOP (discard);
         SET_DEFAULT_FOP (zerofill);
+        SET_DEFAULT_FOP (ipc);
 
         SET_DEFAULT_FOP (getspec);
 
@@ -876,4 +877,3 @@ glusterd_check_log_level (const char *value)
 
         return log_level;
 }
-
diff --git a/libglusterfs/src/xlator.h b/libglusterfs/src/xlator.h
index 1daa06ec2..ec5c0c5c1 100644
--- a/libglusterfs/src/xlator.h
+++ b/libglusterfs/src/xlator.h
@@ -442,6 +442,10 @@ typedef int32_t (*fop_zerofill_cbk_t) (call_frame_t *frame,
                                       struct iatt *preop_stbuf,
                                       struct iatt *postop_stbuf, dict_t *xdata);
 
+typedef int32_t (*fop_ipc_cbk_t) (call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *xdata);
+
 typedef int32_t (*fop_lookup_t) (call_frame_t *frame,
                                  xlator_t *this,
                                  loc_t *loc,
@@ -673,6 +677,7 @@ typedef int32_t (*fop_discard_t) (call_frame_t *frame,
 				  off_t offset,
 				  size_t len,
                                   dict_t *xdata);
+
 typedef int32_t (*fop_zerofill_t) (call_frame_t *frame,
                                   xlator_t *this,
                                   fd_t *fd,
@@ -680,6 +685,9 @@ typedef int32_t (*fop_zerofill_t) (call_frame_t *frame,
                                   off_t len,
                                   dict_t *xdata);
 
+typedef int32_t (*fop_ipc_t) (call_frame_t *frame, xlator_t *this, int32_t op,
+                              dict_t *xdata);
+
 struct xlator_fops {
         fop_lookup_t         lookup;
         fop_stat_t           stat;
@@ -726,6 +734,7 @@ struct xlator_fops {
 	fop_fallocate_t	     fallocate;
 	fop_discard_t	     discard;
         fop_zerofill_t       zerofill;
+        fop_ipc_t            ipc;
 
         /* these entries are used for a typechecking hack in STACK_WIND _only_ */
         fop_lookup_cbk_t         lookup_cbk;
@@ -773,6 +782,7 @@ struct xlator_fops {
 	fop_fallocate_cbk_t	 fallocate_cbk;
 	fop_discard_cbk_t	 discard_cbk;
         fop_zerofill_cbk_t       zerofill_cbk;
+        fop_ipc_cbk_t            ipc_cbk;
 };
 
 typedef int32_t (*cbk_forget_t) (xlator_t *this,
diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h
index 6d28ed90e..a31a0926b 100644
--- a/rpc/rpc-lib/src/protocol-common.h
+++ b/rpc/rpc-lib/src/protocol-common.h
@@ -59,6 +59,7 @@ enum gf_fop_procnum {
 	GFS3_OP_FALLOCATE,
 	GFS3_OP_DISCARD,
         GFS3_OP_ZEROFILL,
+        GFS3_OP_IPC,
         GFS3_OP_MAXVALUE,
 } ;
 
diff --git a/rpc/xdr/src/glusterfs3-xdr.c b/rpc/xdr/src/glusterfs3-xdr.c
index 3205c551e..6b59f4040 100644
--- a/rpc/xdr/src/glusterfs3-xdr.c
+++ b/rpc/xdr/src/glusterfs3-xdr.c
@@ -1587,44 +1587,41 @@ xdr_gfs3_discard_rsp (XDR *xdrs, gfs3_discard_rsp *objp)
 bool_t
 xdr_gfs3_zerofill_req (XDR *xdrs, gfs3_zerofill_req *objp)
 {
-        register int32_t *buf;
+	register int32_t *buf;
         buf = NULL;
 
-        if (!xdr_opaque (xdrs, objp->gfid, 16))
-                return FALSE;
-        if (!xdr_quad_t (xdrs, &objp->fd))
-                return FALSE;
-        if (!xdr_u_quad_t (xdrs, &objp->offset))
-                return FALSE;
-        if (!xdr_u_quad_t (xdrs, &objp->size))
-                return FALSE;
-        if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val,
-             (u_int *) &objp->xdata.xdata_len, ~0))
-                return FALSE;
-        return TRUE;
+	 if (!xdr_opaque (xdrs, objp->gfid, 16))
+		 return FALSE;
+	 if (!xdr_quad_t (xdrs, &objp->fd))
+		 return FALSE;
+	 if (!xdr_u_quad_t (xdrs, &objp->offset))
+		 return FALSE;
+	 if (!xdr_u_quad_t (xdrs, &objp->size))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
+		 return FALSE;
+	return TRUE;
 }
 
 bool_t
 xdr_gfs3_zerofill_rsp (XDR *xdrs, gfs3_zerofill_rsp *objp)
 {
-        register int32_t *buf;
+	register int32_t *buf;
         buf = NULL;
 
-        if (!xdr_int (xdrs, &objp->op_ret))
-                return FALSE;
-        if (!xdr_int (xdrs, &objp->op_errno))
-                return FALSE;
-        if (!xdr_gf_iatt (xdrs, &objp->statpre))
-                return FALSE;
-        if (!xdr_gf_iatt (xdrs, &objp->statpost))
-                return FALSE;
-        if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val,
-             (u_int *) &objp->xdata.xdata_len, ~0))
-                return FALSE;
-        return TRUE;
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	 if (!xdr_gf_iatt (xdrs, &objp->statpre))
+		 return FALSE;
+	 if (!xdr_gf_iatt (xdrs, &objp->statpost))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
+		 return FALSE;
+	return TRUE;
 }
 
-
 bool_t
 xdr_gfs3_rchecksum_req (XDR *xdrs, gfs3_rchecksum_req *objp)
 {
@@ -1705,6 +1702,19 @@ xdr_gfs3_rchecksum_rsp (XDR *xdrs, gfs3_rchecksum_rsp *objp)
 }
 
 bool_t
+xdr_gfs3_ipc_req (XDR *xdrs, gfs3_ipc_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_int (xdrs, &objp->op))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
 xdr_gf_setvolume_req (XDR *xdrs, gf_setvolume_req *objp)
 {
 	register int32_t *buf;
diff --git a/rpc/xdr/src/glusterfs3-xdr.h b/rpc/xdr/src/glusterfs3-xdr.h
index 13566e694..c44a48680 100644
--- a/rpc/xdr/src/glusterfs3-xdr.h
+++ b/rpc/xdr/src/glusterfs3-xdr.h
@@ -937,30 +937,29 @@ struct gfs3_discard_rsp {
 typedef struct gfs3_discard_rsp gfs3_discard_rsp;
 
 struct gfs3_zerofill_req {
-        char gfid[16];
-        quad_t fd;
-        u_quad_t offset;
-        u_quad_t size;
-        struct {
-                u_int xdata_len;
-                char *xdata_val;
-        } xdata;
+	char gfid[16];
+	quad_t fd;
+	u_quad_t offset;
+	u_quad_t size;
+	struct {
+		u_int xdata_len;
+		char *xdata_val;
+	} xdata;
 };
 typedef struct gfs3_zerofill_req gfs3_zerofill_req;
 
 struct gfs3_zerofill_rsp {
-        int op_ret;
-        int op_errno;
-        struct gf_iatt statpre;
-        struct gf_iatt statpost;
-        struct {
-                u_int xdata_len;
-                char *xdata_val;
-        } xdata;
+	int op_ret;
+	int op_errno;
+	struct gf_iatt statpre;
+	struct gf_iatt statpost;
+	struct {
+		u_int xdata_len;
+		char *xdata_val;
+	} xdata;
 };
 typedef struct gfs3_zerofill_rsp gfs3_zerofill_rsp;
 
-
 struct gfs3_rchecksum_req {
 	quad_t fd;
 	u_quad_t offset;
@@ -987,6 +986,15 @@ struct gfs3_rchecksum_rsp {
 };
 typedef struct gfs3_rchecksum_rsp gfs3_rchecksum_rsp;
 
+struct gfs3_ipc_req {
+	int op;
+	struct {
+		u_int xdata_len;
+		char *xdata_val;
+	} xdata;
+};
+typedef struct gfs3_ipc_req gfs3_ipc_req;
+
 struct gf_setvolume_req {
 	struct {
 		u_int dict_len;
@@ -1264,6 +1272,7 @@ extern  bool_t xdr_gfs3_zerofill_req (XDR *, gfs3_zerofill_req*);
 extern  bool_t xdr_gfs3_zerofill_rsp (XDR *, gfs3_zerofill_rsp*);
 extern  bool_t xdr_gfs3_rchecksum_req (XDR *, gfs3_rchecksum_req*);
 extern  bool_t xdr_gfs3_rchecksum_rsp (XDR *, gfs3_rchecksum_rsp*);
+extern  bool_t xdr_gfs3_ipc_req (XDR *, gfs3_ipc_req*);
 extern  bool_t xdr_gf_setvolume_req (XDR *, gf_setvolume_req*);
 extern  bool_t xdr_gf_setvolume_rsp (XDR *, gf_setvolume_rsp*);
 extern  bool_t xdr_gf_getspec_req (XDR *, gf_getspec_req*);
@@ -1364,6 +1373,7 @@ extern bool_t xdr_gfs3_zerofill_req ();
 extern bool_t xdr_gfs3_zerofill_rsp ();
 extern bool_t xdr_gfs3_rchecksum_req ();
 extern bool_t xdr_gfs3_rchecksum_rsp ();
+extern bool_t xdr_gfs3_ipc_req ();
 extern bool_t xdr_gf_setvolume_req ();
 extern bool_t xdr_gf_setvolume_rsp ();
 extern bool_t xdr_gf_getspec_req ();
diff --git a/rpc/xdr/src/glusterfs3-xdr.x b/rpc/xdr/src/glusterfs3-xdr.x
index 1edbda3ad..9fdc8f909 100644
--- a/rpc/xdr/src/glusterfs3-xdr.x
+++ b/rpc/xdr/src/glusterfs3-xdr.x
@@ -630,6 +630,11 @@ struct gfs3_fstat_req {
         opaque   xdata<>; /* Extra data */
 }  ;
 
+struct gfs3_ipc_req {
+	int	op;
+	opaque	xdata<>;
+};
+
 
  struct gf_setvolume_req {
         opaque dict<>;
@@ -744,6 +749,7 @@ struct gf_set_lk_ver_req {
        int lk_ver;
 };
 
+
 struct gf_event_notify_req {
 	int op;
 	opaque dict<>;
diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c
index 8201a95a4..55bcfd16e 100644
--- a/xlators/performance/io-threads/src/io-threads.c
+++ b/xlators/performance/io-threads/src/io-threads.c
@@ -341,6 +341,7 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
         case GF_FOP_RELEASE:
         case GF_FOP_RELEASEDIR:
         case GF_FOP_GETSPEC:
+        case GF_FOP_IPC:
         case GF_FOP_MAXVALUE:
                 //fail compilation on missing fop
                 //new fop must choose priority.
diff --git a/xlators/protocol/client/src/client-rpc-fops.c b/xlators/protocol/client/src/client-rpc-fops.c
index 99a6f6d74..55d62dd39 100644
--- a/xlators/protocol/client/src/client-rpc-fops.c
+++ b/xlators/protocol/client/src/client-rpc-fops.c
@@ -2104,6 +2104,55 @@ out:
 }
 
 int
+client3_3_ipc_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+        call_frame_t    *frame         = NULL;
+        gfs3_zerofill_rsp rsp          = {0,};
+        int              ret           = 0;
+        xlator_t *this                 = NULL;
+        dict_t  *xdata                 = NULL;
+
+        this = THIS;
+
+        frame = myframe;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = ENOTCONN;
+                goto out;
+        }
+        ret = xdr_to_generic(*iov, &rsp, (xdrproc_t) xdr_gf_common_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
+out:
+        if (rsp.op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "remote operation failed: %s",
+                        strerror (gf_error_to_errno (rsp.op_errno)));
+        }
+        CLIENT_STACK_UNWIND (ipc, frame,
+                             rsp.op_ret, gf_error_to_errno (rsp.op_errno),
+                             xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
+        return 0;
+}
+
+int
 client3_3_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
@@ -6093,6 +6142,45 @@ unwind:
         return 0;
 }
 
+int32_t
+client3_3_ipc (call_frame_t *frame, xlator_t *this, void *data)
+{
+        clnt_args_t       *args        = NULL;
+        clnt_conf_t       *conf        = NULL;
+        gfs3_ipc_req       req        = {0,};
+        int                op_errno    = ESTALE;
+        int                ret         = 0;
+
+        GF_ASSERT (frame);
+
+        if (!this || !data)
+                goto unwind;
+
+        args = data;
+        conf = this->private;
+
+        req.op = args->cmd;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
+        ret = client_submit_request(this, &req, frame, conf->fops,
+                                    GFS3_OP_IPC, client3_3_ipc_cbk,
+                                    NULL, NULL, 0, NULL, 0, NULL,
+                                    (xdrproc_t) xdr_gfs3_ipc_req);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+unwind:
+        CLIENT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+}
+
 /* Table Specific to FOPS */
 
 
@@ -6144,7 +6232,8 @@ rpc_clnt_procedure_t clnt3_3_fop_actors[GF_FOP_MAXVALUE] = {
         [GF_FOP_RELEASE]     = { "RELEASE",     client3_3_release },
         [GF_FOP_RELEASEDIR]  = { "RELEASEDIR",  client3_3_releasedir },
         [GF_FOP_GETSPEC]     = { "GETSPEC",     client3_getspec },
-        [GF_FOP_FREMOVEXATTR] = { "FREMOVEXATTR", client3_3_fremovexattr },
+        [GF_FOP_FREMOVEXATTR]= { "FREMOVEXATTR",client3_3_fremovexattr },
+        [GF_FOP_IPC]         = { "IPC",         client3_3_ipc },
 };
 
 /* Used From RPC-CLNT library to log proper name of procedure based on number */
@@ -6196,6 +6285,7 @@ char *clnt3_3_fop_names[GFS3_OP_MAXVALUE] = {
 	[GFS3_OP_FALLOCATE]   = "FALLOCATE",
 	[GFS3_OP_DISCARD]     = "DISCARD",
         [GFS3_OP_ZEROFILL]    = "ZEROFILL",
+        [GFS3_OP_IPC]         = "IPC",
 
 };
 
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index b0a71d3f9..72ff35a04 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -2068,6 +2068,38 @@ out:
 
 
 int32_t
+client_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+        int          ret              = -1;
+        clnt_conf_t *conf             = NULL;
+        rpc_clnt_procedure_t *proc    = NULL;
+        clnt_args_t  args             = {0,};
+
+        conf = this->private;
+        if (!conf || !conf->fops)
+                goto out;
+
+        args.cmd = op;
+        args.xdata = xdata;
+
+        proc = &conf->fops->proctable[GF_FOP_IPC];
+        if (!proc) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "rpc procedure not found for %s",
+                        gf_fop_list[GF_FOP_IPC]);
+                goto out;
+        }
+        if (proc->fn)
+                ret = proc->fn (frame, this, &args);
+out:
+        if (ret)
+                STACK_UNWIND_STRICT(ipc, frame, -1, ENOTCONN, NULL);
+
+        return 0;
+}
+
+
+int32_t
 client_getspec (call_frame_t *frame, xlator_t *this, const char *key,
                 int32_t flags)
 {
@@ -2788,6 +2820,7 @@ struct xlator_fops fops = {
 	.discard     = client_discard,
         .zerofill    = client_zerofill,
         .getspec     = client_getspec,
+        .ipc         = client_ipc,
 };
 
 
diff --git a/xlators/protocol/server/src/server-rpc-fops.c b/xlators/protocol/server/src/server-rpc-fops.c
index 8bdadec6b..bf674271c 100644
--- a/xlators/protocol/server/src/server-rpc-fops.c
+++ b/xlators/protocol/server/src/server-rpc-fops.c
@@ -2040,6 +2040,42 @@ out:
 }
 
 
+int
+server_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp      rsp   = {0,};
+        server_state_t    *state = NULL;
+        rpcsvc_request_t  *req   = NULL;
+
+        req = frame->local;
+        state  = CALL_STATE (frame);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, (&rsp.xdata.xdata_val),
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": IPC%"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        server_submit_reply(frame, req, &rsp, NULL, 0, NULL,
+                            (xdrproc_t) xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
 /* Resume function section */
 
 int
@@ -3088,7 +3124,6 @@ err:
 }
 
 
-
 /* Fop section */
 
 int
@@ -3450,6 +3485,61 @@ out:
 }
 
 int
+server3_3_ipc (rpcsvc_request_t *req)
+{
+        call_frame_t    *frame          = NULL;
+        gfs3_ipc_req     args           = {0,};
+        int              ret            = -1;
+        int              op_errno       = 0;
+        dict_t          *xdata          = NULL;
+        xlator_t        *bound_xl       = NULL;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_ipc_req);
+        if (ret < 0) {
+                /*failed to decode msg*/;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                /* something wrong, mostly insufficient memory*/
+                req->rpc_err = GARBAGE_ARGS; /* TODO */
+                goto out;
+        }
+        frame->root->op = GF_FOP_IPC;
+
+        bound_xl = frame->root->client->bound_xl;
+        if (!bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (bound_xl, xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len,
+                                      ret, op_errno, out);
+
+        ret = 0;
+        STACK_WIND (frame, server_ipc_cbk, bound_xl, bound_xl->fops->ipc,
+                    args.op, xdata);
+        free(xdata);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                req->rpc_err = GARBAGE_ARGS;
+
+        return ret;
+}
+
+int
 server3_3_readlink (rpcsvc_request_t *req)
 {
         server_state_t    *state                 = NULL;
@@ -6170,6 +6260,7 @@ rpcsvc_actor_t glusterfs3_3_fop_actors[] = {
         [GFS3_OP_FALLOCATE]    = {"FALLOCATE",    GFS3_OP_FALLOCATE,    server3_3_fallocate,    NULL, 0, DRC_NA},
         [GFS3_OP_DISCARD]      = {"DISCARD",      GFS3_OP_DISCARD,      server3_3_discard,      NULL, 0, DRC_NA},
         [GFS3_OP_ZEROFILL]    =  {"ZEROFILL",     GFS3_OP_ZEROFILL,     server3_3_zerofill,     NULL, 0, DRC_NA},
+        [GFS3_OP_IPC]         =  {"IPC",          GFS3_OP_IPC,          server3_3_ipc,          NULL, 0, DRC_NA},
 };
 
 
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 588079d92..2997a07dd 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -870,6 +870,20 @@ err:
 
 }
 
+static int32_t
+posix_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+        /*
+         * IPC is for inter-translator communication.  If one gets here, it
+         * means somebody sent one that nobody else recognized, which is an
+         * error much like an uncaught exception.
+         */
+        gf_log (this->name, GF_LOG_ERROR, "GF_LOG_IPC(%d) not handled", op);
+        STACK_UNWIND_STRICT (ipc, frame, -1, -EOPNOTSUPP, NULL);
+        return 0;
+
+}
+
 int32_t
 posix_opendir (call_frame_t *frame, xlator_t *this,
                loc_t *loc, fd_t *fd, dict_t *xdata)
@@ -5680,6 +5694,7 @@ struct xlator_fops fops = {
 	.fallocate   = _posix_fallocate,
 	.discard     = posix_discard,
         .zerofill    = posix_zerofill,
+        .ipc         = posix_ipc,
 };
 
 struct xlator_cbks cbks = {