readdir-ahead: Prefetch xattrs needed by md-cache

Problem:
Negative cache feature implementation in md-cache requires xattrs
returned by posix to be intercepted for every call that can possibly
return xattrs. This includes readdirp(). This is crucial to treat
missing keys in cache as a case of negative entry (returns ENODATA)

md-cache puts names of xattrs that it wants to cache in xdata and
passes it down to posix which returns the specified xattrs in the
callback. This is done in lookup() and readdirp(). Hence, a xattr
that is cached can be invalidated during readdirp_cbk too.

This is based on the assumption that readdirp() will always return
all xattrs that md-cache is interested in. However, this is not the
case when readdirp() call is served from readdir-ahead's cache.
readdir-ahead xlator will pre-fetch dentries during opendir_cbk
and readdirp. These internal readdirp() calls made by readdir-ahead
xlator does not set xdata in it's requests. Hence, no xattrs are
fetched and stored in it's internal cache.

This causes metadata loss in gluster-swift. md-cache returns ENODATA
during getxattr() call even though the xattr for that object exists on
the brick. On receiving ENODATA, gluster-swift will create new metadata
and do setxattr(). This results in loss of information stored in
existing xattr.

Fix:
During opendir, md-cache will communicate to readdir-ahead asking it
to store the names of xattrs it's interested in so that readdir-ahead
can fetch those in all subsequent internal readdirp() calls issued by
it. This stored names of xattrs is invalidated/updated on the next
real readdirp() call issued by application. This readdirp() call will
have xdata set correctly by md-cache xlator.

> Reviewed-on: http://review.gluster.org/14214
> Tested-by: Prashanth Pai <ppai@redhat.com>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
> Tested-by: Gluster Build System <jenkins@build.gluster.com>
> Smoke: Gluster Build System <jenkins@build.gluster.com>
> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
> Tested-by: Raghavendra G <rgowdapp@redhat.com>

BUG: 1334700
Change-Id: I32d46f93a99d4ec34c741f3c52b0646d141614f9
(cherry picked from commit 0c73e7050c4d30ace0c39cc9b9634e9c1b448cfb)
Reviewed-on: http://review.gluster.org/14282
Tested-by: Prashanth Pai <ppai@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Smoke: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>

1 files changed, 80 insertions, 0 deletions

diff --git a/xlators/performance/md-cache/src/md-cache.c b/xlators/performance/md-cache/src/md-cache.c
index b94dade5ebc..74aafdbd8d0 100644
--- a/xlators/performance/md-cache/src/md-cache.c
+++ b/xlators/performance/md-cache/src/md-cache.c
@@ -14,6 +14,7 @@
 #endif
 
 #include "glusterfs.h"
+#include "defaults.h"
 #include "logging.h"
 #include "dict.h"
 #include "xlator.h"
@@ -743,6 +744,50 @@ mdc_load_reqs (xlator_t *this, dict_t *dict)
 }
 
 
+static char*
+mdc_serialize_loaded_key_names (xlator_t *this)
+{
+        int             max_len = 0;
+        int             len = 0;
+        int             i = 0;
+        char           *mdc_key_names = NULL;
+        const char     *mdc_key = NULL;
+        gf_boolean_t    at_least_one_key_loaded = _gf_false;
+
+        for (mdc_key = mdc_keys[i].name; (mdc_key = mdc_keys[i].name); i++) {
+                max_len += (strlen(mdc_keys[i].name) + 1);
+                if (mdc_keys[i].load)
+                        at_least_one_key_loaded = _gf_true;
+        }
+
+        if (!at_least_one_key_loaded)
+                goto out;
+
+        mdc_key_names = GF_CALLOC (1, max_len + 1, gf_common_mt_char);
+        if (!mdc_key_names)
+                goto out;
+
+        i = 0;
+        for (mdc_key = mdc_keys[i].name; (mdc_key = mdc_keys[i].name); i++) {
+                if (!mdc_keys[i].load)
+                        continue;
+                strcat (mdc_key_names, mdc_keys[i].name);
+                strcat (mdc_key_names, " ");
+        }
+
+        len = strlen (mdc_key_names);
+        if (len > 0) {
+                mdc_key_names[len - 1] = '\0';
+        } else {
+                GF_FREE (mdc_key_names);
+                mdc_key_names = NULL;
+        }
+
+out:
+        return mdc_key_names;
+}
+
+
 struct checkpair {
 	int  ret;
 	dict_t *rsp;
@@ -1980,6 +2025,40 @@ mdc_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 
 int
+mdc_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            fd_t *fd, dict_t *xdata)
+{
+        int          ret = -1;
+        char        *mdc_key_names = NULL;
+        dict_t      *xattr_alloc = NULL;
+
+        if (!xdata)
+                xdata = xattr_alloc = dict_new ();
+
+        if (xdata) {
+                /* Tell readdir-ahead to include these keys in xdata when it
+                 * internally issues readdirp() in it's opendir_cbk */
+                mdc_key_names = mdc_serialize_loaded_key_names(this);
+                if (!mdc_key_names)
+                        goto wind;
+                ret = dict_set_dynstr (xdata, GF_MDC_LOADED_KEY_NAMES,
+                                       mdc_key_names);
+                if (ret)
+                        goto wind;
+        }
+
+wind:
+        STACK_WIND (frame, default_opendir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+
+        if (xattr_alloc)
+                dict_unref (xattr_alloc);
+
+        return 0;
+}
+
+
+int
 mdc_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		  int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
@@ -2322,6 +2401,7 @@ struct xlator_fops fops = {
         .fgetxattr   = mdc_fgetxattr,
 	.removexattr = mdc_removexattr,
 	.fremovexattr= mdc_fremovexattr,
+        .opendir     = mdc_opendir,
 	.readdirp    = mdc_readdirp,
 	.readdir     = mdc_readdir,
 	.fallocate   = mdc_fallocate,