feature/bitrot: remove internal xattrs from lookup cbk

Problem:
afr requests all xattrs in lookup via the list-xattr key. If bitrot is
enabled and later disabled, or if the bitrot xattrs were present due to
an older version of bitrot which used to create the xattrs without
enabling the feature, the xattrs (trusted.bit-rot.version in particular)
was not getting filtered and ended up reaching the client stack. AFR, on
noticing different values of the xattr across bricks of the replica,
started triggering spurious metadata heals.

Fix:
Filter all internal xattrs in bitrot xlator before unwinding lookup,
(f)getxattr.

Thanks to Kotresh for the help in RCA'ing.

Change-Id: I5bc70e4b901359c3daefc67b8e4fa6ddb47f046c
BUG: 1524365
Signed-off-by: Ravishankar N <ravishankar@redhat.com>

2 files changed, 21 insertions, 7 deletions

diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub.c b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
index f2ee1950002..76a4c3f6962 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub.c
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
@@ -1598,6 +1598,7 @@ br_stub_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         int32_t              ret          = 0;
         size_t               totallen     = 0;
         size_t               signaturelen = 0;
+        br_stub_private_t   *priv         = NULL;
         br_version_t        *obuf         = NULL;
         br_signature_t      *sbuf         = NULL;
         br_isignature_out_t *sign         = NULL;
@@ -1605,9 +1606,15 @@ br_stub_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         br_stub_local_t     *local        = NULL;
         inode_t             *inode        = NULL;
         gf_boolean_t         bad_object   = _gf_false;
+        gf_boolean_t         ver_enabled  = _gf_false;
+
+        BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+        priv = this->private;
 
         if (op_ret < 0)
                 goto unwind;
+        BR_STUB_VER_COND_GOTO (priv, (!ver_enabled), delkeys);
+
         if (cookie != (void *) BR_STUB_REQUEST_COOKIE)
                 goto unwind;
 
@@ -1753,8 +1760,7 @@ br_stub_getxattr (call_frame_t *frame, xlator_t *this,
                 goto unwind;
 
         priv = this->private;
-        if (!priv->do_versioning)
-                goto wind;
+        BR_STUB_VER_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
 
         /**
          * If xattr is node-uuid and the inode is marked bad, return EIO.
@@ -1775,6 +1781,7 @@ br_stub_getxattr (call_frame_t *frame, xlator_t *this,
                           strlen (GLUSTERFS_GET_BR_STUB_INIT_TIME)) == 0)
             && ((gf_uuid_compare (loc->gfid, rootgfid) == 0)
                 || (gf_uuid_compare (loc->inode->gfid, rootgfid) == 0))) {
+                BR_STUB_RESET_LOCAL_NULL (frame);
                 br_stub_send_stub_init_time (frame, this);
                 return 0;
         }
@@ -1805,6 +1812,7 @@ br_stub_getxattr (call_frame_t *frame, xlator_t *this,
                        FIRST_CHILD (this)->fops->getxattr, loc, name, xdata);
         return 0;
 unwind:
+        BR_STUB_RESET_LOCAL_NULL (frame);
         STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, NULL, NULL);
                 return 0;
 }
@@ -1822,6 +1830,7 @@ br_stub_fgetxattr (call_frame_t *frame, xlator_t *this,
         br_stub_private_t  *priv = NULL;
 
         rootgfid[15] = 1;
+        priv = this->private;
 
         if (!name) {
                 cbk = br_stub_listxattr_cbk;
@@ -1831,9 +1840,7 @@ br_stub_fgetxattr (call_frame_t *frame, xlator_t *this,
         if (br_stub_is_internal_xattr (name))
                 goto unwind;
 
-        priv = this->private;
-        if (!priv->do_versioning)
-                goto wind;
+        BR_STUB_VER_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
 
         /**
          * If xattr is node-uuid and the inode is marked bad, return EIO.
@@ -1853,6 +1860,7 @@ br_stub_fgetxattr (call_frame_t *frame, xlator_t *this,
             && (strncmp (name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
                          strlen (GLUSTERFS_GET_BR_STUB_INIT_TIME)) == 0)
             && (gf_uuid_compare (fd->inode->gfid, rootgfid) == 0)) {
+                BR_STUB_RESET_LOCAL_NULL (frame);
                 br_stub_send_stub_init_time (frame, this);
                 return 0;
         }
@@ -1883,6 +1891,7 @@ br_stub_fgetxattr (call_frame_t *frame, xlator_t *this,
                        FIRST_CHILD (this)->fops->fgetxattr, fd, name, xdata);
         return 0;
 unwind:
+        BR_STUB_RESET_LOCAL_NULL (frame);
         STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, NULL, NULL);
         return 0;
 }
@@ -2880,13 +2889,14 @@ br_stub_lookup_cbk (call_frame_t *frame, void *cookie,
 
         BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
         priv = this->private;
-        BR_STUB_VER_COND_GOTO (priv, (!ver_enabled), unwind);
 
         if (op_ret < 0) {
                 (void) br_stub_handle_lookup_error (this, inode, op_errno);
                 goto unwind;
         }
 
+        BR_STUB_VER_COND_GOTO (priv, (!ver_enabled), delkey);
+
         if (!IA_ISREG (stbuf->ia_type))
                 goto unwind;
 
@@ -2905,7 +2915,6 @@ br_stub_lookup_cbk (call_frame_t *frame, void *cookie,
                         op_errno = EIO;
                         goto unwind;
                 }
-
                 goto delkey;
         }
 
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub.h b/xlators/features/bit-rot/src/stub/bit-rot-stub.h
index 27a84e3218d..41b6cb9d2ba 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub.h
@@ -58,6 +58,11 @@
                         frame->local = NULL;                \
         } while (0)
 
+#define BR_STUB_RESET_LOCAL_NULL(frame) do {           \
+                if (frame->local == (void *)0x1)      \
+                        frame->local = NULL;           \
+        } while (0)
+
 typedef int (br_stub_version_cbk) (call_frame_t *, void *,
                                    xlator_t *, int32_t, int32_t, dict_t *);