features/bit-rot-stub: handle REOPEN_WAIT on forgotten inodes

Change-Id: Ia8706ec9b66d78c4e33e7b7faf69f0d113ba68a4
BUG: 1245981
Signed-off-by: Raghavendra Bhat <raghavendra@redhat.com>
Reviewed-on: http://review.gluster.org/11729
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Venky Shankar <vshankar@redhat.com>

2 files changed, 98 insertions, 1 deletions

diff --git a/tests/bugs/bitrot/bug-1245981.t b/tests/bugs/bitrot/bug-1245981.t
new file mode 100644
index 00000000000..2bed4d980fa
--- /dev/null
+++ b/tests/bugs/bitrot/bug-1245981.t
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+## Test case for bitrot
+## Tunable object signing waiting time value for bitrot.
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../cluster.rc
+
+SLEEP_TIME=5
+
+cleanup;
+## Start glusterd
+TEST glusterd;
+TEST pidof glusterd;
+
+## Lets create and start the volume
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1
+TEST $CLI volume start $V0
+TEST $CLI volume set $V0 network.inode-lru-limit 1
+## Enable bitrot on volume $V0
+TEST $CLI volume bitrot $V0 enable
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" get_bitd_count
+
+# wait a bit for oneshot crawler to finish
+sleep 2;
+
+## Set object expiry time value
+TEST $CLI volume bitrot $V0 signing-time $SLEEP_TIME
+
+## Mount the volume
+TEST $GFS --volfile-server=$H0 --volfile-id=$V0 $M0;
+
+# create and check object signature
+fname="$M0/filezero"
+echo "ZZZ" > $fname
+echo "123" > $M0/new_file;
+
+touch $M0/1
+touch $M0/2
+touch $M0/3
+touch $M0/4
+touch $M0/5
+
+# wait till the object is signed
+sleep `expr $SLEEP_TIME \* 2`
+
+backpath=$(get_backend_paths $fname)
+TEST getfattr -m . -n trusted.bit-rot.signature $backpath
+
+backpath=$(get_backend_paths $M0/new_file)
+TEST getfattr -m . -n trusted.bit-rot.signature $backpath
+
+cleanup;
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub.c b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
index 9fa16c36a9e..f30b5a74307 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub.c
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
@@ -846,6 +846,40 @@ br_stub_fsetxattr_resume (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
+/**
+ * Handles object reopens. Object reopens can be of 3 types. 2 are from
+ * oneshot crawler and 1 from the regular signer.
+ * ONESHOT CRAWLER:
+ * For those objects which were created before bitrot was enabled. oneshow
+ * crawler crawls the namespace and signs all the objects. It has to do
+ * the versioning before making bit-rot-stub send a sign notification.
+ * So it sends fsetxattr with BR_OBJECT_REOPEN as the value. And bit-rot-stub
+ * upon getting BR_OBJECT_REOPEN value checks if the version has to be
+ * increased or not. By default the version will be increased. But if the
+ * object is modified before BR_OBJECT_REOPEN from oneshot crawler, then
+ * versioning need not be done. In that case simply a success is returned.
+ * SIGNER:
+ * Signer wait for 2 minutes upon getting the notification from bit-rot-stub
+ * and then it sends a dummy write (in reality a fsetxattr) call, to change
+ * the state of the inode from REOPEN_WAIT to SIGN_QUICK. The funny part here
+ * is though the inode's state is REOPEN_WAIT, the call sent by signer is
+ * BR_OBJECT_RESIGN. Once the state is changed to SIGN_QUICK, then yet another
+ * notification is sent upon release (RESIGN would have happened via fsetxattr,
+ * so a fd is needed) and the object is signed truly this time.
+ * There is a challenge in the above RESIGN method by signer. After sending
+ * the 1st notification, the inode could be forgotten before RESIGN request
+ * is received. In that case, the inode's context (the newly looked up inode)
+ * would not indicate the inode as being modified (it would be in the default
+ * state) and because of this, a SIGN_QUICK notification to truly sign the
+ * object would not be sent. So, this is how its handled.
+ * if (request == RESIGN) {
+ *    if (inode->sign_info == NORMAL) {
+ *        mark_inode_non_dirty;
+ *        mark_inode_modified;
+ *    }
+ *    GOBACK (means unwind without doing versioning)
+ * }
+ */
 static void
 br_stub_handle_object_reopen (call_frame_t *frame,
                               xlator_t *this, fd_t *fd, uint32_t val)
@@ -858,6 +892,7 @@ br_stub_handle_object_reopen (call_frame_t *frame,
         gf_boolean_t         modified    = _gf_false;
         br_stub_inode_ctx_t *ctx         = NULL;
         br_stub_local_t     *local       = NULL;
+        gf_boolean_t         goback      = _gf_true;
 
         ret = br_stub_need_versioning (this, fd, &inc_version, &modified, &ctx);
         if (ret)
@@ -865,11 +900,18 @@ br_stub_handle_object_reopen (call_frame_t *frame,
 
         LOCK (&fd->inode->lock);
         {
+                if ((val == BR_OBJECT_REOPEN) && inc_version)
+                        goback = _gf_false;
+                if (val == BR_OBJECT_RESIGN &&
+                    ctx->info_sign == BR_SIGN_NORMAL) {
+                        __br_stub_mark_inode_synced (ctx);
+                        __br_stub_set_inode_modified (ctx);
+                }
                 (void) __br_stub_inode_sign_state (ctx, GF_FOP_FSETXATTR, fd);
         }
         UNLOCK (&fd->inode->lock);
 
-        if ((val == BR_OBJECT_RESIGN) || !inc_version) {
+        if (goback) {
                 op_ret = op_errno = 0;
                 goto unwind;
         }