cluster/afr: refactor

- Remove client side self-healing completely (opendir, openfd, lookup)
- Re-work readdir-failover to work reliably in case of NFS
- Remove unused/dead lock recovery code
- Consistently use xdata in both calls and callbacks in all FOPs
- Per-inode event generation, used to force inode ctx refresh
- Implement dirty flag support (in place of pending counts)
- Eliminate inode ctx structure, use read subvol bits + event_generation
- Implement inode ctx refreshing based on event generation
- Provide backward compatibility in transactions
- remove unused variables and functions
- make code more consistent in style and pattern
- regularize and clean up inode-write transaction code
- regularize and clean up dir-write transaction code
- regularize and clean up common FOPs
- reorganize transaction framework code
- skip setting xattrs in pending dict if nothing is pending
- re-write self-healing code using syncops
- re-write simpler self-heal-daemon

Change-Id: I1e4080c9796c8a2815c2dab4be3073f389d614a8
BUG: 1021686
Signed-off-by: Anand Avati <avati@redhat.com>
Reviewed-on: http://review.gluster.org/6010
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

2 files changed, 71 insertions, 341 deletions

diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c
index 5edfeda8f30..5c1c65fbd21 100644
--- a/xlators/features/index/src/index.c
+++ b/xlators/features/index/src/index.c
@@ -15,11 +15,9 @@
 #include "index.h"
 #include "options.h"
 #include "glusterfs3-xdr.h"
-#include "syncop.h"
 #include "syscall.h"
 
 #define XATTROP_SUBDIR "xattrop"
-#define BASE_INDICES_HOLDER_SUBDIR "base_indices_holder"
 
 call_stub_t *
 __index_dequeue (struct list_head *callstubs)
@@ -245,40 +243,20 @@ check_delete_stale_index_file (xlator_t *this, char *filename)
 {
         int             ret = 0;
         struct stat     st = {0};
-        struct stat     base_index_st = {0};
         char            filepath[PATH_MAX] = {0};
-        char            filepath_under_base_indices_holder[PATH_MAX] = {0};
         index_priv_t    *priv = NULL;
 
         priv = this->private;
-        if (priv->to_be_healed_states != synced_state)
-                return;
-
         make_file_path (priv->index_basepath, XATTROP_SUBDIR,
                         filename, filepath, sizeof (filepath));
-
-        make_file_path (priv->index_basepath, BASE_INDICES_HOLDER_SUBDIR,
-                        filename, filepath_under_base_indices_holder,
-                        sizeof (filepath_under_base_indices_holder));
-
-
-        ret = stat (filepath_under_base_indices_holder, &base_index_st);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Base index is not created"
-                        " under index/base_indices_holder");
-                return;
-        }
-
         ret = stat (filepath, &st);
-        if (!ret && st.st_nlink == 2) {
+        if (!ret && st.st_nlink == 1)
                 unlink (filepath);
-                unlink (filepath_under_base_indices_holder);
-        }
 }
 
 static int
 index_fill_readdir (fd_t *fd, DIR *dir, off_t off,
-                    size_t size, gf_dirent_t *entries, readdir_directory type)
+                    size_t size, gf_dirent_t *entries)
 {
         off_t     in_case = -1;
         size_t    filled = 0;
@@ -321,8 +299,7 @@ index_fill_readdir (fd_t *fd, DIR *dir, off_t off,
                 }
 
                 if (!strncmp (entry->d_name, XATTROP_SUBDIR"-",
-                              strlen (XATTROP_SUBDIR"-")) &&
-                              (type == INDEX_XATTROP)) {
+                              strlen (XATTROP_SUBDIR"-"))) {
                         check_delete_stale_index_file (this, entry->d_name);
                         continue;
                 }
@@ -361,192 +338,16 @@ out:
 }
 
 int
-sync_base_indices (void *index_priv)
-{
-        index_priv_t    *priv = NULL;
-        DIR             *dir_base_holder  = NULL;
-        DIR             *xattrop_dir = NULL;
-        struct dirent   *entry = NULL;
-        char            base_indices_holder[PATH_MAX] = {0};
-        char            xattrop_directory[PATH_MAX] = {0};
-        char            base_index_path[PATH_MAX] = {0};
-        char            xattrop_index_path[PATH_MAX] = {0};
-        int32_t         op_errno = 0;
-        int             ret = 0;
-
-        priv = index_priv;
-
-        snprintf (base_indices_holder, PATH_MAX, "%s/%s", priv->index_basepath,
-                  BASE_INDICES_HOLDER_SUBDIR);
-        snprintf (xattrop_directory, PATH_MAX, "%s/%s", priv->index_basepath,
-                  XATTROP_SUBDIR);
-
-        if ((dir_base_holder = opendir(base_indices_holder)) == NULL) {
-                op_errno = errno;
-                ret = -1;
-                goto out;
-        }
-        if ((xattrop_dir = opendir (xattrop_directory)) == NULL) {
-                op_errno = errno;
-                ret = -1;
-                (void) closedir (dir_base_holder);
-                goto out;
-        }
-
-        priv->to_be_healed_states = sync_started;
-        while ((entry = readdir(xattrop_dir)) != NULL) {
-                if (!strcmp (entry->d_name, ".") ||
-                    !strcmp (entry->d_name, "..")) {
-                        continue;
-                }
-                if (strncmp (entry->d_name, XATTROP_SUBDIR"-",
-                    strlen (XATTROP_SUBDIR"-"))) {
-                        continue;
-                }
-                if (!strncmp (entry->d_name, XATTROP_SUBDIR"-",
-                    strlen (XATTROP_SUBDIR"-"))) {
-
-                    snprintf (xattrop_index_path, PATH_MAX, "%s/%s",
-                              xattrop_directory, entry->d_name);
-
-                    snprintf (base_index_path, PATH_MAX, "%s/%s",
-                              base_indices_holder, entry->d_name);
-
-                    ret = sys_link (xattrop_index_path, base_index_path);
-
-                    if (ret && errno != EEXIST) {
-                        op_errno = errno;
-                        (void) closedir (dir_base_holder);
-                        (void) closedir (xattrop_dir);
-                        goto out;
-                    }
-
-                }
-        }
-        ret = closedir (xattrop_dir);
-        if (ret) {
-                op_errno = errno;
-                (void) closedir (dir_base_holder);
-                goto out;
-        }
-        ret = closedir (dir_base_holder);
-        if (ret) {
-                op_errno = errno;
-                goto out;
-        }
-
-        ret = 0;
-out:
-        errno = op_errno;
-        return ret;
-
-}
-
-int
-base_indices_syncing_done (int ret, call_frame_t *frame, void *data)
-{
-        index_priv_t         *priv = NULL;
-        priv = data;
-
-        if (!priv)
-                goto out;
-
-        if (ret) {
-                priv->to_be_healed_states = sync_not_started;
-        } else {
-                priv->to_be_healed_states = synced_state;
-        }
-
-        STACK_DESTROY (frame->root);
-
-out:
-        return 0;
-}
-
-int
-sync_base_indices_from_xattrop (xlator_t *this)
-{
-
-        index_priv_t          *priv = NULL;
-        char                  base_indices_holder[PATH_MAX] =  {0};
-        int                   ret = 0;
-        struct stat           st = {0};
-        DIR                   *dir = NULL;
-        struct dirent         *entry = NULL;
-        call_frame_t          *frame = NULL;
-
-        priv = this->private;
-
-        if (priv->to_be_healed_states != sync_not_started) {
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (base_indices_holder, PATH_MAX, "%s/%s", priv->index_basepath,
-                  BASE_INDICES_HOLDER_SUBDIR);
-
-        ret = stat (base_indices_holder, &st);
-
-        if (ret && (errno != ENOENT)) {
-                goto out;
-        } else if (errno == ENOENT) {
-                ret = index_dir_create (this, BASE_INDICES_HOLDER_SUBDIR);
-                if (ret)
-                        goto out;
-        } else {
-                if ((dir = opendir (base_indices_holder)) == NULL) {
-                        ret = -1;
-                        goto out;
-                }
-                while ((entry = readdir (dir)) != NULL) {
-                        if (!strcmp (entry->d_name, ".") ||
-                            !strcmp (entry->d_name,"..")) {
-                                continue;
-                        }
-                        ret = unlink (entry->d_name);
-                        if (ret) {
-                                closedir (dir);
-                                goto out;
-                        }
-                }
-                closedir (dir);
-        }
-
-        /*At this point of time we have index/base_indicies_holder directory
-         *is with no entries*/
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-        set_lk_owner_from_ptr (&frame->root->lk_owner, frame->root);
-
-        frame->root->pid = LOW_PRIO_PROC_PID;
-
-        ret = synctask_new (this->ctx->env, sync_base_indices,
-                            base_indices_syncing_done,frame, priv);
-
-
-
-out:
-        return ret;
-
-}
-
-int
 index_add (xlator_t *this, uuid_t gfid, const char *subdir)
 {
         int32_t           op_errno = 0;
         char              gfid_path[PATH_MAX] = {0};
         char              index_path[PATH_MAX] = {0};
-        char              base_path[PATH_MAX] = {0};
         int               ret = 0;
         uuid_t            index = {0};
         index_priv_t      *priv = NULL;
         struct stat       st = {0};
         int               fd = 0;
-        int               index_created = 0;
 
         priv = this->private;
         GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !uuid_is_null (gfid),
@@ -561,15 +362,12 @@ index_add (xlator_t *this, uuid_t gfid, const char *subdir)
         index_get_index (priv, index);
         make_index_path (priv->index_basepath, subdir,
                          index, index_path, sizeof (index_path));
-
         ret = sys_link (index_path, gfid_path);
         if (!ret || (errno == EEXIST))  {
                 ret = 0;
-                index_created = 1;
                 goto out;
         }
 
-
         op_errno = errno;
         if (op_errno == ENOENT) {
                 ret = index_dir_create (this, subdir);
@@ -601,36 +399,10 @@ index_add (xlator_t *this, uuid_t gfid, const char *subdir)
                         "add to index (%s)", uuid_utoa (gfid),
                         strerror (errno));
                 goto out;
-        } else {
-                index_created = 1;
-        }
-
-        if (priv->to_be_healed_states != sync_not_started) {
-                 make_index_path (priv->index_basepath,
-                                  GF_BASE_INDICES_HOLDER_GFID,
-                                  index, base_path, sizeof (base_path));
-                 ret = sys_link (index_path, base_path);
-                 if (ret)
-                         goto out;
         }
 
         ret = 0;
 out:
-        /*If base_indices_holder is not created: create and sync
-         *If directory is present: delete contents and start syncing
-         *If syncing is in progress :No need to do any thing
-         *If syncing is done: No need to do anything*/
-        if (!ret) {
-                switch (priv->to_be_healed_states) {
-                        case sync_not_started:
-                                ret = sync_base_indices_from_xattrop (this);
-                                break;
-                        case sync_started:
-                        case synced_state:
-                                /*No need to do anything*/
-                                break;
-                }
-        }
         return ret;
 }
 
@@ -966,6 +738,41 @@ out:
         return 0;
 }
 
+uint64_t
+index_entry_count (xlator_t *this, char *subdir)
+{
+	index_priv_t *priv = NULL;
+	char index_dir[PATH_MAX];
+	DIR *dirp = NULL;
+	uint64_t count = 0;
+	struct dirent buf;
+	struct dirent *entry = NULL;
+
+	priv = this->private;
+
+	make_index_dir_path (priv->index_basepath, subdir,
+			     index_dir, sizeof (index_dir));
+
+	dirp = opendir (index_dir);
+	if (!dirp)
+		return 0;
+
+	while (readdir_r (dirp, &buf, &entry) == 0) {
+		if (!entry)
+			break;
+		if (!strcmp (entry->d_name, ".") ||
+		    !strcmp (entry->d_name, ".."))
+			continue;
+                if (!strncmp (entry->d_name, subdir, strlen (subdir)))
+			continue;
+		count++;
+	}
+	closedir (dirp);
+
+	return count;
+}
+
+
 int32_t
 index_getxattr_wrapper (call_frame_t *frame, xlator_t *this,
                         loc_t *loc, const char *name, dict_t *xdata)
@@ -973,6 +780,7 @@ index_getxattr_wrapper (call_frame_t *frame, xlator_t *this,
         index_priv_t    *priv = NULL;
         dict_t          *xattr = NULL;
         int             ret = 0;
+	uint64_t        count = 0;
 
         priv = this->private;
 
@@ -982,24 +790,26 @@ index_getxattr_wrapper (call_frame_t *frame, xlator_t *this,
                 goto done;
         }
 
-        if (!strcmp (name, GF_XATTROP_INDEX_GFID)) {
-
-                ret = dict_set_static_bin (xattr, (char*)name,
-                                           priv->xattrop_vgfid,
-                                           sizeof (priv->xattrop_vgfid));
-
-        } else if (!strcmp (name, GF_BASE_INDICES_HOLDER_GFID)) {
-
-                ret = dict_set_static_bin (xattr, (char*)name,
-                                           priv->base_indices_holder_vgfid,
-                                      sizeof (priv->base_indices_holder_vgfid));
-        }
-        if (ret) {
-                ret = -ENOMEM;
-                gf_log (THIS->name, GF_LOG_ERROR, "xattrop index "
-                        "gfid set failed");
-                goto done;
-        }
+	if (strcmp (name, GF_XATTROP_INDEX_GFID) == 0) {
+		ret = dict_set_static_bin (xattr, (char*)name, priv->xattrop_vgfid,
+					   sizeof (priv->xattrop_vgfid));
+		if (ret) {
+			ret = -ENOMEM;
+			gf_log (this->name, GF_LOG_ERROR, "xattrop index "
+				"gfid set failed");
+			goto done;
+		}
+	} else if (strcmp (name, GF_XATTROP_INDEX_COUNT) == 0) {
+		count = index_entry_count (this, XATTROP_SUBDIR);
+
+		ret = dict_set_uint64 (xattr, (char *)name, count);
+		if (ret) {
+			ret = -ENOMEM;
+			gf_log (this->name, GF_LOG_ERROR, "xattrop index "
+				"count set failed");
+			goto done;
+		}
+	}
 done:
         if (ret)
                 STACK_UNWIND_STRICT (getxattr, frame, -1, -ret, xattr, xdata);
@@ -1037,15 +847,6 @@ index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
         } else if (!uuid_compare (loc->pargfid, priv->xattrop_vgfid)) {
                 make_file_path (priv->index_basepath, XATTROP_SUBDIR,
                                 loc->name, path, sizeof (path));
-        } else if (!uuid_compare (loc->gfid,priv->base_indices_holder_vgfid)){
-                make_index_dir_path (priv->index_basepath,
-                                     BASE_INDICES_HOLDER_SUBDIR, path,
-                                    sizeof (path));
-                is_dir = _gf_true;
-        } else if (!uuid_compare (loc->pargfid, priv->base_indices_holder_vgfid)) {
-                make_file_path (priv->index_basepath,
-                                BASE_INDICES_HOLDER_SUBDIR,loc->name, path,
-                                sizeof (path));
         }
 
         ret = lstat (path, &lstatbuf);
@@ -1067,14 +868,10 @@ index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
         }
 
         iatt_from_stat (&stbuf, &lstatbuf);
-        if (is_dir && !uuid_compare (loc->gfid, priv->xattrop_vgfid)) {
+        if (is_dir)
                 uuid_copy (stbuf.ia_gfid, priv->xattrop_vgfid);
-        } else if (is_dir &&
-                !uuid_compare (loc->gfid, priv->base_indices_holder_vgfid)) {
-                uuid_copy (stbuf.ia_gfid, priv->base_indices_holder_vgfid);
-        } else {
+        else
                 uuid_generate (stbuf.ia_gfid);
-        }
         stbuf.ia_ino = -1;
         op_ret = 0;
 done:
@@ -1086,44 +883,6 @@ done:
 }
 
 int32_t
-base_indices_readdir_wrapper (call_frame_t *frame, xlator_t *this,
-                              fd_t *fd, size_t size, off_t off, dict_t *xdata)
-{
-        index_priv_t    *priv = NULL;
-        char            base_indices_holder[PATH_MAX] = {0};
-        DIR             *dir = NULL;
-        int32_t         op_ret = -1;
-        int32_t         op_errno = 0;
-        int             count = 0;
-        gf_dirent_t     entries;
-
-        priv = this->private;
-
-        make_index_dir_path (priv->index_basepath, BASE_INDICES_HOLDER_SUBDIR,
-                             base_indices_holder, sizeof (base_indices_holder));
-
-        dir = opendir (base_indices_holder);
-        if (!dir) {
-                op_errno = EINVAL;
-                goto done;
-        }
-
-
-        INIT_LIST_HEAD (&entries.list);
-
-        count = index_fill_readdir (fd, dir, off, size, &entries,
-                                    BASE_INDICES_HOLDER);
-        /* pick ENOENT to indicate EOF */
-        op_errno = errno;
-        op_ret = count;
-        closedir (dir);
-done:
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
-        gf_dirent_free (&entries);
-        return 0;
-}
-
-int32_t
 index_readdir_wrapper (call_frame_t *frame, xlator_t *this,
                        fd_t *fd, size_t size, off_t off, dict_t *xdata)
 {
@@ -1154,8 +913,7 @@ index_readdir_wrapper (call_frame_t *frame, xlator_t *this,
                 goto done;
         }
 
-        count = index_fill_readdir (fd, dir, off, size, &entries,
-                                    INDEX_XATTROP);
+        count = index_fill_readdir (fd, dir, off, size, &entries);
 
         /* pick ENOENT to indicate EOF */
         op_errno = errno;
@@ -1221,11 +979,12 @@ index_getxattr (call_frame_t *frame, xlator_t *this,
                 loc_t *loc, const char *name, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
 
-        if (!name)
-                goto out;
-        if (strcmp (GF_XATTROP_INDEX_GFID, name) &&
-            strcmp (GF_BASE_INDICES_HOLDER_GFID, name))
+        priv = this->private;
+
+        if (!name || (strcmp (GF_XATTROP_INDEX_GFID, name) &&
+		      strcmp (GF_XATTROP_INDEX_COUNT, name)))
                 goto out;
 
         stub = fop_getxattr_stub (frame, index_getxattr_wrapper, loc, name,
@@ -1252,9 +1011,7 @@ index_lookup (call_frame_t *frame, xlator_t *this,
         priv = this->private;
 
         if (uuid_compare (loc->gfid, priv->xattrop_vgfid) &&
-            uuid_compare (loc->pargfid, priv->xattrop_vgfid) &&
-            uuid_compare (loc->gfid, priv->base_indices_holder_vgfid) &&
-            uuid_compare (loc->pargfid, priv->base_indices_holder_vgfid))
+            uuid_compare (loc->pargfid, priv->xattrop_vgfid))
                 goto normal;
 
         stub = fop_lookup_stub (frame, index_lookup_wrapper, loc, xattr_req);
@@ -1280,19 +1037,10 @@ index_readdir (call_frame_t *frame, xlator_t *this,
         index_priv_t    *priv = NULL;
 
         priv = this->private;
-        if (uuid_compare (fd->inode->gfid, priv->xattrop_vgfid) &&
-            uuid_compare (fd->inode->gfid, priv->base_indices_holder_vgfid))
+        if (uuid_compare (fd->inode->gfid, priv->xattrop_vgfid))
                 goto out;
-
-        if (!uuid_compare (fd->inode->gfid, priv->xattrop_vgfid)) {
-                stub = fop_readdir_stub (frame, index_readdir_wrapper, fd, size, 
-                                         off, xdata);
-        } else if (!uuid_compare (fd->inode->gfid,
-                                  priv->base_indices_holder_vgfid)) {
-                stub = fop_readdir_stub (frame, base_indices_readdir_wrapper,
-                                         fd, size, off, xdata);
-        }
-
+        stub = fop_readdir_stub (frame, index_readdir_wrapper, fd, size, off,
+                                 xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
@@ -1396,9 +1144,6 @@ init (xlator_t *this)
         GF_OPTION_INIT ("index-base", priv->index_basepath, path, out);
         uuid_generate (priv->index);
         uuid_generate (priv->xattrop_vgfid);
-        /*base_indices_holder is a directory which contains hard links to
-         * all base indices inside indices/xattrop directory*/
-        uuid_generate (priv->base_indices_holder_vgfid);
         INIT_LIST_HEAD (&priv->callstubs);
 
         this->private = priv;
@@ -1415,7 +1160,6 @@ init (xlator_t *this)
         }
 
         ret = 0;
-
 out:
         if (ret) {
                 if (cond_inited)
diff --git a/xlators/features/index/src/index.h b/xlators/features/index/src/index.h
index d6dcb1c23b4..661dcdbc417 100644
--- a/xlators/features/index/src/index.h
+++ b/xlators/features/index/src/index.h
@@ -36,28 +36,14 @@ typedef struct index_fd_ctx {
         DIR *dir;
 } index_fd_ctx_t;
 
-typedef enum {
-        sync_not_started,
-        sync_started,
-        synced_state,
-} to_be_healed_states_t;
-
-typedef enum {
-        INDEX_XATTROP,
-        BASE_INDICES_HOLDER,
-} readdir_directory;
-
 typedef struct index_priv {
         char *index_basepath;
         uuid_t index;
         gf_lock_t lock;
         uuid_t xattrop_vgfid;//virtual gfid of the xattrop index dir
-        uuid_t base_indices_holder_vgfid; //virtual gfid of the
-                                          //to_be_healed_xattrop directory
         struct list_head callstubs;
         pthread_mutex_t mutex;
         pthread_cond_t  cond;
-        to_be_healed_states_t to_be_healed_states;
 } index_priv_t;
 
 #define INDEX_STACK_UNWIND(fop, frame, params ...)      \