md-cache: Register the list of xattrs with cache-invalidation

Issue:
md-cache caches a specified list of xattrs, and when cache invalidation
is enabled, it makes sense to recieve invalidation only when those xattrs
are modified by other clients. But the current implementation of upcall
is that, it will send invalidation when any of the on-disk xattrs is modified.

Solution:
md-cache sends a list of xattrs that it is interested in, to upcall by
issuing an ipc(). The challenge here is to make sure everytime a brick
goes offline and comes back up, the ipc() needs to be issued to the
bricks. Hence ipc() is sent from md-cache every time there is a
CHILD_UP/CHILD_MODIFIED event.

TODO:
There will be patches following, in cluster xlators, to implement ipc fop.

Change-Id: I6efcf3df474f5ce6eabd3d6694c00c7bd89bc25d
BUG: 1211863
Signed-off-by: Poornima G <pgurusid@redhat.com>
Reviewed-on: http://review.gluster.org/15002
Smoke: Gluster Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Prashanth Pai <ppai@redhat.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>

3 files changed, 170 insertions, 6 deletions

diff --git a/xlators/performance/md-cache/src/md-cache-mem-types.h b/xlators/performance/md-cache/src/md-cache-mem-types.h
index 6634cf962a5..5cfc68e13c1 100644
--- a/xlators/performance/md-cache/src/md-cache-mem-types.h
+++ b/xlators/performance/md-cache/src/md-cache-mem-types.h
@@ -18,6 +18,7 @@ enum gf_mdc_mem_types_ {
         gf_mdc_mt_mdc_local_t   = gf_common_mt_end + 1,
 	gf_mdc_mt_md_cache_t,
 	gf_mdc_mt_mdc_conf_t,
+        gf_mdc_mt_mdc_ipc,
         gf_mdc_mt_end
 };
 #endif
diff --git a/xlators/performance/md-cache/src/md-cache-messages.h b/xlators/performance/md-cache/src/md-cache-messages.h
index 1fe26ccc8b2..4aea7cd0a9d 100644
--- a/xlators/performance/md-cache/src/md-cache-messages.h
+++ b/xlators/performance/md-cache/src/md-cache-messages.h
@@ -40,7 +40,7 @@
  */
 
 #define GLFS_MD_CACHE_BASE                   GLFS_MSGID_COMP_MD_CACHE
-#define GLFS_MD_CACHE_NUM_MESSAGES           3
+#define GLFS_MD_CACHE_NUM_MESSAGES           5
 #define GLFS_MSGID_END  (GLFS_MD_CACHE_BASE + GLFS_MD_CACHE_NUM_MESSAGES + 1)
 
 /* Messages with message IDs */
@@ -67,8 +67,33 @@
 
 #define MD_CACHE_MSG_DISCARD_UPDATE    (GLFS_MD_CACHE_BASE + 2)
 
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction  None
+ *
+ */
+
 #define MD_CACHE_MSG_CACHE_UPDATE      (GLFS_MD_CACHE_BASE + 3)
 
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction  None
+ *
+ */
+
+#define MD_CACHE_MSG_IPC_UPCALL_FAILED  (GLFS_MD_CACHE_BASE + 4)
+
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction  None
+ *
+ */
+
+#define MD_CACHE_MSG_NO_XATTR_CACHE     (GLFS_MD_CACHE_BASE + 5)
+
 /*------------*/
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 
diff --git a/xlators/performance/md-cache/src/md-cache.c b/xlators/performance/md-cache/src/md-cache.c
index bb2310e8c82..e0e7ee68e3b 100644
--- a/xlators/performance/md-cache/src/md-cache.c
+++ b/xlators/performance/md-cache/src/md-cache.c
@@ -13,6 +13,7 @@
 #include "logging.h"
 #include "dict.h"
 #include "xlator.h"
+#include "syncop.h"
 #include "md-cache-mem-types.h"
 #include "compat-errno.h"
 #include "glusterfs-acl.h"
@@ -2455,6 +2456,18 @@ is_strpfx (const char *str1, const char *str2)
 }
 
 
+static int
+mdc_key_unload_all (struct mdc_key *keys)
+{
+        struct mdc_key *key = NULL;
+
+        for (key = keys; key->name; key++) {
+                key->load = 0;
+        }
+
+        return 0;
+}
+
 int
 mdc_key_load_set (struct mdc_key *keys, char *pattern, gf_boolean_t val)
 {
@@ -2545,12 +2558,129 @@ out:
         return ret;
 }
 
+struct mdc_ipc {
+       xlator_t *this;
+       dict_t   *xattr;
+};
+
+static int
+mdc_send_xattrs_cbk (int ret, call_frame_t *frame, void *data)
+{
+        struct mdc_ipc *tmp = data;
+
+        if (ret < 0) {
+                mdc_key_unload_all (mdc_keys);
+                gf_msg ("md-cache", GF_LOG_INFO, 0, MD_CACHE_MSG_NO_XATTR_CACHE,
+                        "Disabled cache for all xattrs, as registering for "
+                        "xattr cache invalidation failed");
+        }
+        STACK_DESTROY (frame->root);
+        dict_unref (tmp->xattr);
+        GF_FREE (tmp);
+
+        return 0;
+}
+
+static int
+mdc_send_xattrs (void *data)
+{
+        int             ret = 0;
+        struct mdc_ipc *tmp = data;
+
+        ret = syncop_ipc (FIRST_CHILD (tmp->this), GF_IPC_TARGET_UPCALL,
+                          tmp->xattr, NULL);
+        DECODE_SYNCOP_ERR (ret);
+        if (ret < 0) {
+                gf_msg (tmp->this->name, GF_LOG_WARNING, errno,
+                        MD_CACHE_MSG_IPC_UPCALL_FAILED, "Registering the list "
+                        "of xattrs that needs invalidaton, with upcall, failed");
+        }
+
+        return ret;
+}
+
+
+static int
+mdc_register_xattr_inval (xlator_t *this)
+{
+        dict_t          *xattr = NULL;
+        int              ret   = 0;
+        struct mdc_conf *conf  = NULL;
+        call_frame_t    *frame = NULL;
+        struct mdc_ipc  *data  = NULL;
+
+        conf = this->private;
+
+        LOCK (&conf->lock);
+        {
+                if (!conf->mdc_invalidation) {
+                        UNLOCK (&conf->lock);
+                        goto out;
+                }
+        }
+        UNLOCK (&conf->lock);
+
+        xattr = dict_new ();
+        if (!xattr) {
+                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
+                        MD_CACHE_MSG_NO_MEMORY, "dict_new failed");
+                ret = -1;
+                goto out;
+        }
+
+        mdc_load_reqs (this, xattr);
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
+                        MD_CACHE_MSG_NO_MEMORY,
+                        "failed to create the frame");
+                ret = -1;
+                goto out;
+        }
+
+        data = GF_CALLOC (1, sizeof (struct mdc_ipc), gf_mdc_mt_mdc_ipc);
+        if (!data) {
+                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
+                        MD_CACHE_MSG_NO_MEMORY,
+                        "failed to allocate memory");
+                ret = -1;
+                goto out;
+        }
+
+        data->this = this;
+        data->xattr = xattr;
+        ret = synctask_new (this->ctx->env, mdc_send_xattrs, mdc_send_xattrs_cbk,
+                            frame, data);
+        if (ret < 0) {
+                gf_msg (this->name, GF_LOG_WARNING, errno,
+                        MD_CACHE_MSG_IPC_UPCALL_FAILED, "Registering the list "
+                        "of xattrs that needs invalidaton, with upcall, failed");
+        }
+
+out:
+        if (ret < 0) {
+                mdc_key_unload_all (mdc_keys);
+                if (xattr)
+                        dict_unref (xattr);
+                if (frame)
+                        STACK_DESTROY (frame->root);
+                GF_FREE (data);
+                gf_msg (this->name, GF_LOG_INFO, 0, MD_CACHE_MSG_NO_XATTR_CACHE,
+                        "Disabled cache for all xattrs, as registering for "
+                        "xattr cache invalidation failed");
+        }
+
+        return ret;
+}
+
 
 int
 reconfigure (xlator_t *this, dict_t *options)
 {
 	struct mdc_conf *conf = NULL;
         int    timeout = 0;
+        int    ret     = 0;
 
 	conf = this->private;
 
@@ -2589,6 +2719,8 @@ reconfigure (xlator_t *this, dict_t *options)
                         goto out;
         }
         conf->timeout = timeout;
+
+        ret = mdc_register_xattr_inval (this);
 out:
 	return 0;
 }
@@ -2686,22 +2818,28 @@ notify (xlator_t *this, int event, void *data, ...)
         switch (event) {
         case GF_EVENT_CHILD_DOWN:
         case GF_EVENT_SOME_CHILD_DOWN:
-        case GF_EVENT_CHILD_MODIFIED:
                 time (&now);
                 mdc_update_child_down_time (this, &now);
-                ret = default_notify (this, event, data);
                 break;
         case GF_EVENT_UPCALL:
                 if (conf->mdc_invalidation)
                         ret = mdc_invalidate (this, data);
-                if (default_notify (this, event, data) != 0)
-                        ret = -1;
+                break;
+        case GF_EVENT_CHILD_MODIFIED:
+                time (&now);
+                mdc_update_child_down_time (this, &now);
+                ret = mdc_register_xattr_inval (this);
+                break;
+        case GF_EVENT_CHILD_UP:
+                ret = mdc_register_xattr_inval (this);
                 break;
         default:
-                ret = default_notify (this, event, data);
                 break;
         }
 
+        if (default_notify (this, event, data) != 0)
+                ret = -1;
+
         return ret;
 }