cluster/afr: Ensure directory contents are in sync during opendir.

The problem: If some files on the first subvolume disappeared
without leaving a trace in the entry changelog (this can happen,
for example, when an fsck has deleted files or when a hard drive
is replaced), those files would never be self-healed even though
they would be present on the second subvolume. This is because
readdir is sent only to the first subvolume, and since the files
don't appear in the directory listing, no lookup would ever be
sent on them.

This patch fixes this problem by doing a readdir on all the subvolumes
during the first opendir on a directory inode. If a discrepancy in the
contents is detected, entry self-heal in a special "force merge" mode
is triggered on that directory.

Signed-off-by: Vikas Gorur <vikas@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 249 (Self heal of a file that does not exist on the first subvolume)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=249

1 files changed, 63 insertions, 4 deletions

diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index c56ce314a07..a0441bfb0bd 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -55,9 +55,9 @@
 
 #include "afr-self-heal.h"
 
-
-#define AFR_ICTX_SPLIT_BRAIN_MASK    0x0000000100000000ULL
-#define AFR_ICTX_READ_CHILD_MASK     0x00000000FFFFFFFFULL
+#define AFR_ICTX_OPENDIR_DONE_MASK     0x0000000200000000ULL
+#define AFR_ICTX_SPLIT_BRAIN_MASK      0x0000000100000000ULL
+#define AFR_ICTX_READ_CHILD_MASK       0x00000000FFFFFFFFULL
 
 
 uint64_t
@@ -115,6 +115,60 @@ out:
 
 
 uint64_t
+afr_is_opendir_done (xlator_t *this, inode_t *inode)
+{
+        int ret = 0;
+
+        uint64_t ctx          = 0;
+        uint64_t opendir_done = 0;
+
+        VALIDATE_OR_GOTO (inode, out);
+
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_get (inode, this, &ctx);
+
+                if (ret < 0)
+                        goto unlock;
+                
+                opendir_done = ctx & AFR_ICTX_OPENDIR_DONE_MASK;
+        }
+unlock:
+        UNLOCK (&inode->lock);
+
+out:
+        return opendir_done;
+}
+
+
+void
+afr_set_opendir_done (xlator_t *this, inode_t *inode, int32_t opendir_done)
+{
+        uint64_t ctx = 0;
+        int      ret = 0;
+
+        VALIDATE_OR_GOTO (inode, out);
+
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_get (inode, this, &ctx);
+
+                if (ret < 0) {
+                        ctx = 0;
+                }
+
+                ctx = (~AFR_ICTX_OPENDIR_DONE_MASK & ctx)
+                        | (opendir_done & AFR_ICTX_OPENDIR_DONE_MASK);
+
+                __inode_ctx_put (inode, this, ctx);
+        }
+        UNLOCK (&inode->lock);
+out:
+        return;
+}
+
+
+uint64_t
 afr_read_child (xlator_t *this, inode_t *inode)
 {
         int ret = 0;
@@ -320,6 +374,11 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
 	{ /* symlink */
 		FREE (local->cont.symlink.linkpath);
 	}
+
+        { /* opendir */
+                if (local->cont.opendir.checksum)
+                        FREE (local->cont.opendir.checksum);
+        }
 }
 
 
@@ -642,7 +701,7 @@ unlock:
 					lookup_buf->st_mode;
 			}
 
-			afr_self_heal (frame, this, afr_self_heal_cbk);
+			afr_self_heal (frame, this, NULL);
 		} else {
 			AFR_STACK_UNWIND (lookup, frame, local->op_ret,
 					  local->op_errno,