Upcall: Added xlator options to enable cache-invalidation

Added two xlator options to enable cache-invalidation and set
cache-invalidation-timeout.

In addition, made few minor changes in the upcall processing code
in gfapi.

Change-Id: Ie0b32ca8348e34e3fe4f1e7df30cc925fa4aad31
BUG: 1200271
Signed-off-by: Soumya Koduri <skoduri@redhat.com>
Reviewed-on: http://review.gluster.org/9975
Reviewed-by: Meghana M <mmadhusu@redhat.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>

4 files changed, 161 insertions, 53 deletions

diff --git a/xlators/features/upcall/src/upcall-cache-invalidation.h b/xlators/features/upcall/src/upcall-cache-invalidation.h
index 758ddf1dae8..138595cecb6 100644
--- a/xlators/features/upcall/src/upcall-cache-invalidation.h
+++ b/xlators/features/upcall/src/upcall-cache-invalidation.h
@@ -16,13 +16,9 @@
 #include "config.h"
 #endif
 
-/* TODO: Below macros have to be replaced with
- * xlator options - Bug1200271 */
-#define ON_CACHE_INVALIDATION 0 /* disable by default */
-
 /* The time period for which a client will be notified of cache_invalidation
  * events post its last access */
-#define CACHE_INVALIDATION_PERIOD 60
+#define CACHE_INVALIDATION_TIMEOUT "60"
 
 /* Flags sent for cache_invalidation */
 #define UP_NLINK   0x00000001   /* update nlink */
@@ -58,13 +54,19 @@
 #define UP_NLINK_FLAGS          (UP_NLINK | UP_TIMES)
 
 #define CACHE_INVALIDATE(frame, this, client, inode, p_flags) do {      \
- if (ON_CACHE_INVALIDATION) {                                             \
-        (void)upcall_cache_invalidate (frame, this, client, inode, p_flags);  \
- }                                                                      \
+                                                                        \
+        if (!is_cache_invalidation_enabled(this))                       \
+                break;                                                  \
+                                                                        \
+        (void)upcall_cache_invalidate (frame, this, client,             \
+                                       inode, p_flags);                 \
 } while (0)
 
-#define CACHE_INVALIDATE_DIR(frame, this, client, inode_p, p_flags) do {      \
- if (ON_CACHE_INVALIDATION) {                                             \
+#define CACHE_INVALIDATE_DIR(frame, this, client, inode_p, p_flags) do {\
+                                                                        \
+        if (!is_cache_invalidation_enabled(this))                       \
+                break;                                                  \
+                                                                        \
         dentry_t *dentry;                                               \
         dentry_t *dentry_tmp;                                           \
         list_for_each_entry_safe (dentry, dentry_tmp,                   \
@@ -73,7 +75,10 @@
                 (void)upcall_cache_invalidate (frame, this, client,     \
                                                dentry->inode, p_flags); \
         }                                                               \
- }                                                                      \
 } while (0)
 
+/* xlator options */
+gf_boolean_t is_cache_invalidation_enabled(xlator_t *this);
+int32_t get_cache_invalidation_timeout(xlator_t *this);
+
 #endif /* __UPCALL_CACHE_INVALIDATION_H__ */
diff --git a/xlators/features/upcall/src/upcall-internal.c b/xlators/features/upcall/src/upcall-internal.c
index 26473e2a7bd..ef7743c9083 100644
--- a/xlators/features/upcall/src/upcall-internal.c
+++ b/xlators/features/upcall/src/upcall-internal.c
@@ -34,6 +34,62 @@
 #include "defaults.h"
 
 /*
+ * Check if any of the upcall options are enabled:
+ *     - cache_invalidation
+ *     - XXX: lease_lk
+ */
+gf_boolean_t
+is_upcall_enabled(xlator_t *this) {
+        upcall_private_t *priv      = NULL;
+        gf_boolean_t     is_enabled = _gf_false;
+
+        if (this->private) {
+                priv = (upcall_private_t *)this->private;
+
+                if (priv->cache_invalidation_enabled) {
+                        is_enabled = _gf_true;
+                }
+        }
+
+        return is_enabled;
+}
+
+/*
+ * Check if any of cache_invalidation is enabled
+ */
+gf_boolean_t
+is_cache_invalidation_enabled(xlator_t *this) {
+        upcall_private_t *priv      = NULL;
+        gf_boolean_t     is_enabled = _gf_false;
+
+        if (this->private) {
+                priv = (upcall_private_t *)this->private;
+
+                if (priv->cache_invalidation_enabled) {
+                        is_enabled = _gf_true;
+                }
+        }
+
+        return is_enabled;
+}
+
+/*
+ * Get the cache_invalidation_timeout
+ */
+int32_t
+get_cache_invalidation_timeout(xlator_t *this) {
+        upcall_private_t *priv      = NULL;
+        int32_t          timeout    = 0;
+
+        if (this->private) {
+                priv = (upcall_private_t *)this->private;
+                timeout = priv->cache_invalidation_timeout;
+        }
+
+        return timeout;
+}
+
+/*
  * Allocate and add a new client entry to the given upcall entry
  */
 upcall_client_t*
@@ -73,7 +129,8 @@ __add_upcall_client (call_frame_t *frame, uuid_t gfid,
         INIT_LIST_HEAD (&up_client_entry->client_list);
         up_client_entry->client_uid = gf_strdup(client->client_uid);
         up_client_entry->access_time = time(NULL);
-        up_client_entry->expire_time_attr = CACHE_INVALIDATION_PERIOD;
+        up_client_entry->expire_time_attr =
+                        get_cache_invalidation_timeout(frame->this);
 
         list_add_tail (&up_client_entry->client_list,
                        &up_inode_ctx->client_list);
@@ -349,7 +406,7 @@ upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client,
 
 /*
  * If the upcall_client_t has recently accessed the file (i.e, within
- * CACHE_INVALIDATION_PERIOD), send a upcall notification.
+ * priv->cache_invalidation_timeout), send a upcall notification.
  */
 void
 upcall_client_cache_invalidate (xlator_t *this, uuid_t gfid,
@@ -357,9 +414,12 @@ upcall_client_cache_invalidate (xlator_t *this, uuid_t gfid,
                                 uint32_t flags)
 {
         notify_event_data_t n_event_data;
+        time_t timeout   = 0;
         time_t t_expired = time(NULL) - up_client_entry->access_time;
 
-        if (t_expired < CACHE_INVALIDATION_PERIOD) {
+        timeout = get_cache_invalidation_timeout(this);
+
+        if (t_expired < timeout) {
                 /* Send notify call */
                 uuid_copy(n_event_data.gfid, gfid);
                 n_event_data.client_entry = up_client_entry;
@@ -374,7 +434,7 @@ upcall_client_cache_invalidate (xlator_t *this, uuid_t gfid,
                         up_client_entry->client_uid);
 
         } else {
-                if (t_expired > (2*CACHE_INVALIDATION_PERIOD)) {
+                if (t_expired > (2*timeout)) {
                         /* Cleanup the entry */
                         __upcall_cleanup_client_entry (up_client_entry);
                 }
diff --git a/xlators/features/upcall/src/upcall.c b/xlators/features/upcall/src/upcall.c
index b7f2e975bba..45114b5f29f 100644
--- a/xlators/features/upcall/src/upcall.c
+++ b/xlators/features/upcall/src/upcall.c
@@ -42,7 +42,7 @@ up_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         uint32_t         flags          = 0;
         upcall_local_t   *local         = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         client = frame->root->client;
         local = frame->local;
@@ -67,7 +67,7 @@ up_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         int32_t          op_errno        = -1;
         upcall_local_t   *local          = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         local = upcall_local_init (frame, this, fd->inode);
         if (!local) {
@@ -123,7 +123,7 @@ up_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         int32_t          op_errno        = -1;
         upcall_local_t   *local          = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         local = upcall_local_init (frame, this, fd->inode);
         if (!local) {
@@ -157,7 +157,7 @@ up_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         uint32_t         flags          = 0;
         upcall_local_t   *local         = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         client = frame->root->client;
         local = frame->local;
@@ -183,7 +183,7 @@ up_readv (call_frame_t *frame, xlator_t *this,
         int32_t          op_errno        = -1;
         upcall_local_t   *local          = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         local = upcall_local_init (frame, this, fd->inode);
         if (!local) {
@@ -215,7 +215,7 @@ up_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         uint32_t         flags          = 0;
         upcall_local_t   *local         = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         client = frame->root->client;
         local = frame->local;
@@ -239,7 +239,7 @@ up_lk (call_frame_t *frame, xlator_t *this,
         int32_t          op_errno        = -1;
         upcall_local_t   *local          = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         local = upcall_local_init (frame, this, fd->inode);
         if (!local) {
@@ -269,7 +269,7 @@ up_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         uint32_t         flags          = 0;
         upcall_local_t   *local         = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         client = frame->root->client;
         local = frame->local;
@@ -294,7 +294,7 @@ up_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
         int32_t          op_errno        = -1;
         upcall_local_t   *local          = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         local = upcall_local_init (frame, this, loc->inode);
         if (!local) {
@@ -325,7 +325,7 @@ up_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         uint32_t         flags          = 0;
         upcall_local_t   *local         = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         client = frame->root->client;
         local = frame->local;
@@ -355,7 +355,7 @@ up_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         int32_t          op_errno        = -1;
         upcall_local_t   *local          = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         local = upcall_local_init (frame, this, loc->inode);
         if (!local) {
@@ -389,7 +389,7 @@ up_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         uint32_t         flags          = 0;
         upcall_local_t   *local         = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         client = frame->root->client;
         local = frame->local;
@@ -423,7 +423,7 @@ up_rename (call_frame_t *frame, xlator_t *this,
         int32_t          op_errno        = -1;
         upcall_local_t   *local          = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         local = upcall_local_init (frame, this, oldloc->inode);
         if (!local) {
@@ -455,7 +455,7 @@ up_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         uint32_t         flags          = 0;
         upcall_local_t   *local         = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         client = frame->root->client;
         local = frame->local;
@@ -484,7 +484,7 @@ up_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
         int32_t          op_errno        = -1;
         upcall_local_t   *local          = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         local = upcall_local_init (frame, this, loc->inode);
         if (!local) {
@@ -515,7 +515,7 @@ up_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         uint32_t         flags          = 0;
         upcall_local_t   *local         = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         client = frame->root->client;
         local = frame->local;
@@ -541,7 +541,7 @@ up_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
         int32_t          op_errno        = -1;
         upcall_local_t   *local          = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         local = upcall_local_init (frame, this, oldloc->inode);
         if (!local) {
@@ -573,7 +573,7 @@ up_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         uint32_t         flags          = 0;
         upcall_local_t   *local         = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         client = frame->root->client;
         local = frame->local;
@@ -602,7 +602,7 @@ up_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
         int32_t          op_errno        = -1;
         upcall_local_t   *local          = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         local = upcall_local_init (frame, this, loc->inode);
         if (!local) {
@@ -634,7 +634,7 @@ up_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         uint32_t         flags          = 0;
         upcall_local_t   *local         = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         client = frame->root->client;
         local = frame->local;
@@ -663,7 +663,7 @@ up_mkdir (call_frame_t *frame, xlator_t *this,
         int32_t          op_errno        = -1;
         upcall_local_t   *local          = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         local = upcall_local_init (frame, this, loc->inode);
         if (!local) {
@@ -696,7 +696,7 @@ up_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         uint32_t         flags          = 0;
         upcall_local_t   *local         = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         client = frame->root->client;
         local = frame->local;
@@ -725,7 +725,7 @@ up_create (call_frame_t *frame, xlator_t *this,
         int32_t          op_errno        = -1;
         upcall_local_t   *local          = NULL;
 
-        EXIT_IF_UPCALL_OFF (out);
+        EXIT_IF_UPCALL_OFF (this, out);
 
         local = upcall_local_init (frame, this, loc->inode);
 
@@ -800,10 +800,29 @@ out:
 }
 
 int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        upcall_private_t *priv                   = NULL;
+        int              ret                    = -1;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_OPTION_RECONF ("cache-invalidation", priv->cache_invalidation_enabled,
+                          options, bool, out);
+        GF_OPTION_RECONF ("cache-invalidation-timeout", priv->cache_invalidation_timeout,
+                          options, int32, out);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
 init (xlator_t *this)
 {
         int                       ret        = -1;
-        upcalls_private_t        *priv       = NULL;
+        upcall_private_t         *priv       = NULL;
 
         priv = GF_CALLOC (1, sizeof (*priv),
                           gf_upcall_mt_private_t);
@@ -814,6 +833,11 @@ init (xlator_t *this)
                 goto out;
         }
 
+        GF_OPTION_INIT ("cache-invalidation", priv->cache_invalidation_enabled,
+                        bool, out);
+        GF_OPTION_INIT ("cache-invalidation-timeout",
+                        priv->cache_invalidation_timeout, int32, out);
+
         this->private = priv;
         this->local_pool = mem_pool_new (upcall_local_t, 512);
         ret = 0;
@@ -829,13 +853,14 @@ out:
 int
 fini (xlator_t *this)
 {
-        upcalls_private_t *priv = NULL;
+        upcall_private_t *priv = NULL;
 
         priv = this->private;
         if (!priv) {
                 return 0;
         }
         this->private = NULL;
+
         GF_FREE (priv);
 
         return 0;
@@ -952,5 +977,18 @@ struct xlator_cbks cbks = {
 };
 
 struct volume_options options[] = {
+        { .key  = {"cache-invalidation"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "When \"on\", sends cache-invalidation"
+                         " notifications."
+        },
+        { .key  = {"cache-invalidation-timeout"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = CACHE_INVALIDATION_TIMEOUT,
+          .description = "After 'timeout' seconds since the time"
+                         " client accessed any file, cache-invalidation"
+                         " notifications are no longer sent to that client."
+        },
         { .key = {NULL} },
 };
diff --git a/xlators/features/upcall/src/upcall.h b/xlators/features/upcall/src/upcall.h
index a5aff9d091e..7cbec22bbf2 100644
--- a/xlators/features/upcall/src/upcall.h
+++ b/xlators/features/upcall/src/upcall.h
@@ -26,9 +26,9 @@
 #include "upcall-messages.h"
 #include "upcall-cache-invalidation.h"
 
-#define EXIT_IF_UPCALL_OFF(label) do {                     \
-        if (!(ON_CACHE_INVALIDATION))                      \
-                goto label;                                \
+#define EXIT_IF_UPCALL_OFF(this, label) do {                       \
+        if (!is_upcall_enabled(this))                              \
+                goto label;                                        \
 } while (0)
 
 #define UPCALL_STACK_UNWIND(fop, frame, params ...) do {        \
@@ -38,10 +38,10 @@
                         __xl         = frame->this;             \
                         __local      = frame->local;            \
                         frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                upcall_local_wipe (__xl, __local);              \
-        } while (0)
+        }                                                       \
+        STACK_UNWIND_STRICT (fop, frame, params);               \
+        upcall_local_wipe (__xl, __local);                      \
+} while (0)
 
 #define UPCALL_STACK_DESTROY(frame) do {                   \
                 upcall_local_t *__local = NULL;            \
@@ -51,12 +51,13 @@
                 frame->local         = NULL;               \
                 STACK_DESTROY (frame->root);               \
                 upcall_local_wipe (__xl, __local);         \
-        } while (0)
+} while (0)
 
-struct _upcalls_private_t {
-        int client_id; /* Not sure if reqd */
+struct _upcall_private_t {
+        gf_boolean_t     cache_invalidation_enabled;
+        int32_t          cache_invalidation_timeout;
 };
-typedef struct _upcalls_private_t upcalls_private_t;
+typedef struct _upcall_private_t upcall_private_t;
 
 enum _upcall_event_type_t {
         EVENT_NULL,
@@ -122,13 +123,17 @@ int __upcall_inode_ctx_set (inode_t *inode, xlator_t *this);
 upcall_inode_ctx_t *__upcall_inode_ctx_get (inode_t *inode, xlator_t *this);
 upcall_inode_ctx_t *upcall_inode_ctx_get (inode_t *inode, xlator_t *this);
 int upcall_cleanup_inode_ctx (xlator_t *this, inode_t *inode);
+void upcall_cache_forget (xlator_t *this, inode_t *inode,
+                          upcall_inode_ctx_t *up_inode_ctx);
 
+/* Xlator options */
+gf_boolean_t is_upcall_enabled(xlator_t *this);
+
+/* Cache invalidation specific */
 void upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client,
                               inode_t *inode, uint32_t flags);
 void upcall_client_cache_invalidate (xlator_t *xl, uuid_t gfid,
                                      upcall_client_t *up_client_entry,
                                      uint32_t flags);
-void upcall_cache_forget (xlator_t *this, inode_t *inode,
-                          upcall_inode_ctx_t *up_inode_ctx);
 
 #endif /* __UPCALL_H__ */