cluster/afr: Make [f]xattrop metadata transaction

Problem:
When xlators above afr do [f]xattrop when one of the bricks is down, after the
brick comes backup, the metadata is not healed because [f]xattrop is not
considered a transaction.

Fix:
Treat [f]xattrop as transaction so that changes done by xlators above afr are
marked for heal when some of the bricks were down at the time of [f]xattrop.

Change-Id: Iea180f9a456509847c3cd8d5d59a0cdc2712d334
BUG: 1248887
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/11809
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Ravishankar N <ravishankar@redhat.com>
Tested-by: NetBSD Build System <jenkins@build.gluster.org>

5 files changed, 272 insertions, 183 deletions

diff --git a/tests/bugs/shard/bug-1248887.t b/tests/bugs/shard/bug-1248887.t
new file mode 100644
index 00000000000..4528e9b9ac2
--- /dev/null
+++ b/tests/bugs/shard/bug-1248887.t
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume set $V0 features.shard on
+TEST $CLI volume start $V0
+
+TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0
+#Create a file.
+TEST touch $M0/foo
+#Write some data into it.
+TEST `echo "abc" > $M0/foo`
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/foo
+EXPECT "0000000000000004000000000000000000000000000000010000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/foo
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/foo
+EXPECT "0000000000000004000000000000000000000000000000010000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/foo
+TEST kill_brick $V0 $H0 $B0/${V0}0
+TEST `echo "abc" >> $M0/foo`
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/foo
+EXPECT "0000000000000008000000000000000000000000000000010000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/foo
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1
+TEST $CLI volume heal $V0
+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}0/foo
+EXPECT "0000000000000008000000000000000000000000000000010000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}0/foo
+EXPECT "0000000000400000" get_hex_xattr trusted.glusterfs.shard.block-size $B0/${V0}1/foo
+EXPECT "0000000000000008000000000000000000000000000000010000000000000000" get_hex_xattr trusted.glusterfs.shard.file-size $B0/${V0}1/foo
+
+cleanup;
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index e9a7c8c3649..f74d530a7aa 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1221,6 +1221,11 @@ afr_replies_wipe (struct afr_reply *replies, int count)
                         dict_unref (replies[i].xdata);
                         replies[i].xdata = NULL;
                 }
+
+                if (replies[i].xattr) {
+                        dict_unref (replies[i].xattr);
+                        replies[i].xattr = NULL;
+                }
         }
 }
 
@@ -1273,6 +1278,9 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         if (local->xattr_req)
                 dict_unref (local->xattr_req);
 
+        if (local->xattr_rsp)
+                dict_unref (local->xattr_rsp);
+
         if (local->dict)
                 dict_unref (local->dict);
 
@@ -1350,10 +1358,6 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
                 if (local->cont.xattrop.xattr)
                         dict_unref (local->cont.xattrop.xattr);
         }
-        { /* fxattrop */
-                if (local->cont.fxattrop.xattr)
-                        dict_unref (local->cont.fxattrop.xattr);
-        }
         { /* symlink */
                 GF_FREE (local->cont.symlink.linkpath);
         }
@@ -2905,166 +2909,6 @@ out:
 
 /* }}} */
 
-/* {{{ xattrop */
-
-int32_t
-afr_xattrop_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 dict_t *xattr, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        if (!local->cont.xattrop.xattr)
-                                local->cont.xattrop.xattr = dict_ref (xattr);
-
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-
-                        local->op_ret = 0;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                AFR_STACK_UNWIND (xattrop, frame, local->op_ret, local->op_errno,
-                local->cont.xattrop.xattr, local->xdata_rsp);
-
-        return 0;
-}
-
-
-int32_t
-afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
-{
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = ENOMEM;
-
-        priv = this->private;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_xattrop_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->xattrop,
-                                    loc, optype, xattr, xdata);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-	return 0;
-out:
-	AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
-}
-
-/* }}} */
-
-/* {{{ fxattrop */
-
-int32_t
-afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *xattr, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        if (!local->cont.fxattrop.xattr)
-                                local->cont.fxattrop.xattr = dict_ref (xattr);
-
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-                        local->op_ret = 0;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fxattrop, frame, local->op_ret, local->op_errno,
-                                  local->cont.fxattrop.xattr, local->xdata_rsp);
-
-        return 0;
-}
-
-
-int32_t
-afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
-{
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        priv = this->private;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-                goto out;
-
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fxattrop_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fxattrop,
-                                    fd, optype, xattr, xdata);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-	return 0;
-out:
-	AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
-}
-
-/* }}} */
-
 int32_t
 afr_unlock_partial_inodelk_cbk (call_frame_t *frame, void *cookie,
                                 xlator_t *this, int32_t op_ret,
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index a1bf5b65491..5dd52cf9c0d 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -92,6 +92,12 @@ __afr_inode_write_finalize (call_frame_t *frame, xlator_t *this)
 				local->xdata_rsp =
 					dict_ref (local->replies[i].xdata);
 			}
+			if (local->replies[i].xattr) {
+				if (local->xattr_rsp)
+					dict_unref (local->xattr_rsp);
+				local->xattr_rsp =
+					dict_ref (local->replies[i].xattr);
+			}
 		}
 	}
 
@@ -102,7 +108,8 @@ __afr_inode_write_finalize (call_frame_t *frame, xlator_t *this)
 static void
 __afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
 			int op_ret, int op_errno,
-			struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+			struct iatt *prebuf, struct iatt *postbuf,
+			dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
 
@@ -117,6 +124,8 @@ __afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
 			local->replies[child_index].prestat = *prebuf;
 		if (postbuf)
 			local->replies[child_index].poststat = *postbuf;
+		if (xattr)
+			local->replies[child_index].xattr = dict_ref (xattr);
 		if (xdata)
 			local->replies[child_index].xdata = dict_ref (xdata);
 	} else {
@@ -130,7 +139,7 @@ __afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
 static int
 __afr_inode_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
+                       struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int child_index = (long) cookie;
@@ -141,7 +150,8 @@ __afr_inode_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         LOCK (&frame->lock);
         {
                 __afr_inode_write_fill (frame, this, child_index, op_ret,
-					op_errno, prebuf, postbuf, xdata);
+					op_errno, prebuf, postbuf, xattr,
+					xdata);
         }
         UNLOCK (&frame->lock);
 
@@ -250,7 +260,7 @@ afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         LOCK (&frame->lock);
         {
                 __afr_inode_write_fill (frame, this, child_index, op_ret,
-					op_errno, prebuf, postbuf, xdata);
+					op_errno, prebuf, postbuf, NULL, xdata);
 		if (op_ret == -1 || !xdata)
 			goto unlock;
 
@@ -504,7 +514,7 @@ afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		local->stable_write = _gf_false;
 
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+				      prebuf, postbuf, NULL, xdata);
 }
 
 
@@ -623,7 +633,7 @@ afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		local->stable_write = _gf_false;
 
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+				      prebuf, postbuf, NULL, xdata);
 }
 
 
@@ -735,7 +745,7 @@ afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      preop, postop, xdata);
+				      preop, postop, NULL, xdata);
 }
 
 
@@ -840,7 +850,7 @@ afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      preop, postop, xdata);
+				      preop, postop, NULL, xdata);
 }
 
 
@@ -947,7 +957,7 @@ afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xdata);
+				      NULL, NULL, NULL, xdata);
 }
 
 
@@ -1494,7 +1504,7 @@ afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xdata);
+				      NULL, NULL, NULL, xdata);
 }
 
 
@@ -1608,7 +1618,7 @@ afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xdata);
+				      NULL, NULL, NULL, xdata);
 }
 
 
@@ -1716,7 +1726,7 @@ afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xdata);
+				      NULL, NULL, NULL, xdata);
 }
 
 
@@ -1826,7 +1836,7 @@ afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         struct iatt *postbuf, dict_t *xdata)
 {
         return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+				      prebuf, postbuf, NULL, xdata);
 }
 
 
@@ -1940,7 +1950,7 @@ afr_discard_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       struct iatt *postbuf, dict_t *xdata)
 {
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+				      prebuf, postbuf, NULL, xdata);
 }
 
 
@@ -2050,7 +2060,7 @@ afr_zerofill_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       struct iatt *postbuf, dict_t *xdata)
 {
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+				      prebuf, postbuf, NULL, xdata);
 }
 
 
@@ -2132,3 +2142,192 @@ out:
 }
 
 /* }}} */
+
+int32_t
+afr_xattrop_wind_cbk (call_frame_t *frame, void *cookie,
+                      xlator_t *this, int32_t op_ret, int32_t op_errno,
+                      dict_t *xattr, dict_t *xdata)
+{
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xattr, xdata);
+}
+
+int
+afr_xattrop_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_xattrop_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->xattrop,
+			   &local->loc, local->cont.xattrop.optype,
+			   local->cont.xattrop.xattr, local->xdata_req);
+        return 0;
+}
+
+int
+afr_xattrop_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        call_frame_t *main_frame = NULL;
+
+        local = frame->local;
+
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
+
+	AFR_STACK_UNWIND (xattrop, main_frame, local->op_ret, local->op_errno,
+			  local->xattr_rsp, local->xdata_rsp);
+        return 0;
+}
+
+int32_t
+afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
+
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
+
+        local->cont.xattrop.xattr = dict_ref (xattr);
+        local->cont.xattrop.optype = optype;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
+
+        local->transaction.wind   = afr_xattrop_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_xattrop_unwind;
+
+        loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+
+	local->op = GF_FOP_XATTROP;
+
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
+
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
+        }
+
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+int32_t
+afr_fxattrop_wind_cbk (call_frame_t *frame, void *cookie,
+                       xlator_t *this, int32_t op_ret, int32_t op_errno,
+                       dict_t *xattr, dict_t *xdata)
+{
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xattr, xdata);
+}
+
+int
+afr_fxattrop_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_fxattrop_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fxattrop,
+			   local->fd, local->cont.xattrop.optype,
+			   local->cont.xattrop.xattr, local->xdata_req);
+        return 0;
+}
+
+int
+afr_fxattrop_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        call_frame_t *main_frame = NULL;
+
+        local = frame->local;
+
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
+
+	AFR_STACK_UNWIND (fxattrop, main_frame, local->op_ret, local->op_errno,
+			  local->xattr_rsp, local->xdata_rsp);
+        return 0;
+}
+
+int32_t
+afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
+
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
+
+        local->cont.xattrop.xattr = dict_ref (xattr);
+        local->cont.xattrop.optype = optype;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
+
+        local->transaction.wind   = afr_fxattrop_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_fxattrop_unwind;
+
+	local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	local->op = GF_FOP_FXATTROP;
+
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
+
+        ret = afr_transaction (transaction_frame, this,
+                               AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
+        }
+
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index 7b1fc552880..e174cc2d610 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -79,4 +79,12 @@ afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
 int
 afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
              off_t len, dict_t *xdata);
+
+int32_t
+afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+int32_t
+afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 7f995c731d7..3a4a8794698 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -257,6 +257,7 @@ struct afr_reply {
 	int	valid;
 	int32_t	op_ret;
 	int32_t	op_errno;
+	dict_t *xattr;/*For xattrop*/
 	dict_t *xdata;
 	struct iatt poststat;
 	struct iatt postparent;
@@ -563,12 +564,9 @@ typedef struct _afr_local {
 
                 struct {
                         dict_t *xattr;
+                        gf_xattrop_flags_t optype;
                 } xattrop;
 
-                struct {
-                        dict_t *xattr;
-                } fxattrop;
-
                 /* dir write */
 
                 struct {
@@ -729,6 +727,8 @@ typedef struct _afr_local {
         dict_t         *xdata_req;
         dict_t         *xdata_rsp;
 
+        dict_t         *xattr_rsp; /*for [f]xattrop*/
+
         mode_t          umask;
         int             xflag;
         gf_boolean_t    do_discovery;