diff options
| -rw-r--r-- | configure.ac | 2 | ||||
| -rw-r--r-- | libglusterfs/src/defaults.c | 15 | ||||
| -rw-r--r-- | libglusterfs/src/glfs-message-id.h | 4 | ||||
| -rw-r--r-- | libglusterfs/src/glusterfs.h | 9 | ||||
| -rw-r--r-- | rpc/rpc-lib/src/protocol-common.h | 3 | ||||
| -rw-r--r-- | rpc/xdr/src/glusterfs3-xdr.x | 9 | ||||
| -rw-r--r-- | rpc/xdr/src/glusterfs3.h | 12 | ||||
| -rw-r--r-- | xlators/features/Makefile.am | 3 | ||||
| -rw-r--r-- | xlators/features/upcall/Makefile.am | 3 | ||||
| -rw-r--r-- | xlators/features/upcall/src/Makefile.am | 21 | ||||
| -rw-r--r-- | xlators/features/upcall/src/upcall-cache-invalidation.h | 79 | ||||
| -rw-r--r-- | xlators/features/upcall/src/upcall-internal.c | 424 | ||||
| -rw-r--r-- | xlators/features/upcall/src/upcall-mem-types.h | 24 | ||||
| -rw-r--r-- | xlators/features/upcall/src/upcall-messages.h | 62 | ||||
| -rw-r--r-- | xlators/features/upcall/src/upcall.c | 956 | ||||
| -rw-r--r-- | xlators/features/upcall/src/upcall.h | 134 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-volgen.c | 25 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-volgen.h | 1 | ||||
| -rw-r--r-- | xlators/protocol/server/src/server.c | 66 |
19 files changed, 1845 insertions, 7 deletions
diff --git a/configure.ac b/configure.ac index f4de4462aec..ab47dcae7a3 100644 --- a/configure.ac +++ b/configure.ac @@ -151,6 +151,8 @@ AC_CONFIG_FILES([Makefile xlators/features/snapview-server/src/Makefile xlators/features/snapview-client/Makefile xlators/features/snapview-client/src/Makefile + xlators/features/upcall/Makefile + xlators/features/upcall/src/Makefile xlators/playground/Makefile xlators/playground/template/Makefile xlators/playground/template/src/Makefile diff --git a/libglusterfs/src/defaults.c b/libglusterfs/src/defaults.c index 599f9477dca..a4f8f924b17 100644 --- a/libglusterfs/src/defaults.c +++ b/libglusterfs/src/defaults.c @@ -2287,6 +2287,21 @@ default_notify (xlator_t *this, int32_t event, void *data, ...) } } break; + case GF_EVENT_UPCALL: + { + xlator_list_t *parent = this->parents; + + if (!parent && this->ctx && this->ctx->master) + xlator_notify (this->ctx->master, event, data, NULL); + + while (parent) { + if (parent->xlator->init_succeeded) + xlator_notify (parent->xlator, event, + data, NULL); + parent = parent->next; + } + } + break; default: { xlator_list_t *parent = this->parents; diff --git a/libglusterfs/src/glfs-message-id.h b/libglusterfs/src/glfs-message-id.h index 35abe472aab..c977206c44d 100644 --- a/libglusterfs/src/glfs-message-id.h +++ b/libglusterfs/src/glfs-message-id.h @@ -72,6 +72,10 @@ #define GLFS_MSGID_COMP_COMMON_END (GLFS_MSGID_COMP_COMMON +\ GLFS_MSGID_SEGMENT) +#define GLFS_MSGID_COMP_UPCALL GLFS_MSGID_COMP_COMMON_END +#define GLFS_MSGID_COMP_UPCALL_END (GLFS_MSGID_COMP_UPCALL +\ + GLFS_MSGID_SEGMENT) + /* --- new segments for messages goes above this line --- */ #endif /* !_GLFS_MESSAGE_ID_H_ */ diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h index c482b3d2242..d5a604d0341 100644 --- a/libglusterfs/src/glusterfs.h +++ b/libglusterfs/src/glusterfs.h @@ -562,6 +562,7 @@ typedef enum { GF_EVENT_VOLUME_DEFRAG, GF_EVENT_PARENT_DOWN, GF_EVENT_VOLUME_BARRIER_OP, + GF_EVENT_UPCALL, GF_EVENT_MAXVAL, } glusterfs_event_t; @@ -574,6 +575,14 @@ struct gf_flock { gf_lkowner_t l_owner; }; +struct gf_upcall { + char *client_uid; + char gfid[16]; + u_int event_type; + u_int flags; + u_int expire_time_attr; +}; + #define GF_MUST_CHECK __attribute__((warn_unused_result)) /* * Some macros (e.g. ALLOC_OR_GOTO) set variables in function scope, but the diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h index 8ab6078aaf5..3d8cf11fa5f 100644 --- a/rpc/rpc-lib/src/protocol-common.h +++ b/rpc/rpc-lib/src/protocol-common.h @@ -131,6 +131,9 @@ enum gf_cbk_procnum { GF_CBK_INO_FLUSH, GF_CBK_EVENT_NOTIFY, GF_CBK_GET_SNAPS, + /* XXX: Have separate events for each + * UPCALL event - BZ 1200268) */ + GF_CBK_UPCALL, GF_CBK_MAXVALUE, }; diff --git a/rpc/xdr/src/glusterfs3-xdr.x b/rpc/xdr/src/glusterfs3-xdr.x index a68fcae7e1c..81b0f201f89 100644 --- a/rpc/xdr/src/glusterfs3-xdr.x +++ b/rpc/xdr/src/glusterfs3-xdr.x @@ -43,6 +43,15 @@ struct gf_iatt { unsigned int ia_ctime_nsec; }; + +struct gfs3_upcall_req { + opaque gfid[16]; + unsigned int event_type; /* Upcall event type */ + unsigned int flags; /* or mask of events incase of inotify */ + unsigned int expire_time_attr; /* the amount of time which client + * can cache this entry */ +}; + struct gfs3_stat_req { opaque gfid[16]; opaque xdata<>; /* Extra data */ diff --git a/rpc/xdr/src/glusterfs3.h b/rpc/xdr/src/glusterfs3.h index 798413e3145..b3ee267b64d 100644 --- a/rpc/xdr/src/glusterfs3.h +++ b/rpc/xdr/src/glusterfs3.h @@ -267,4 +267,16 @@ gf_stat_from_iatt (struct gf_iatt *gf_stat, struct iatt *iatt) gf_stat->ia_ctime_nsec = iatt->ia_ctime_nsec ; } +static inline void +gf_proto_upcall_from_upcall (gfs3_upcall_req *gf_up_req, + struct gf_upcall *gf_up_data) +{ + if (!gf_up_req || !gf_up_data) + return; + + memcpy (gf_up_req->gfid, gf_up_data->gfid, 16); + gf_up_req->event_type = gf_up_data->event_type; + gf_up_req->flags = gf_up_data->flags; + gf_up_req->expire_time_attr = gf_up_data->expire_time_attr; +} #endif /* !_GLUSTERFS3_H */ diff --git a/xlators/features/Makefile.am b/xlators/features/Makefile.am index dcb3cc8e5a7..096f0e04bb2 100644 --- a/xlators/features/Makefile.am +++ b/xlators/features/Makefile.am @@ -1,4 +1,5 @@ SUBDIRS = locks quota read-only mac-compat quiesce marker index barrier \ - protect compress changelog gfid-access $(GLUPY_SUBDIR) qemu-block snapview-client snapview-server trash # path-converter # filter + protect compress changelog gfid-access $(GLUPY_SUBDIR) qemu-block \ + upcall snapview-client snapview-server trash # path-converter # filter CLEANFILES = diff --git a/xlators/features/upcall/Makefile.am b/xlators/features/upcall/Makefile.am new file mode 100644 index 00000000000..a985f42a877 --- /dev/null +++ b/xlators/features/upcall/Makefile.am @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = diff --git a/xlators/features/upcall/src/Makefile.am b/xlators/features/upcall/src/Makefile.am new file mode 100644 index 00000000000..2ac09551476 --- /dev/null +++ b/xlators/features/upcall/src/Makefile.am @@ -0,0 +1,21 @@ +xlator_LTLIBRARIES = upcall.la +xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features + +upcall_la_LDFLAGS = -module -avoid-version + +upcall_la_SOURCES = upcall.c upcall-internal.c + +upcall_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \ + $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \ + $(top_builddir)/rpc/xdr/src/libgfxdr.la + +noinst_HEADERS = upcall.h upcall-mem-types.h upcall-messages.h \ + upcall-cache-invalidation.h + +AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ + -I$(top_srcdir)/rpc/rpc-lib/src \ + -I$(top_srcdir)/rpc/xdr/src + +AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS) + +CLEANFILES = diff --git a/xlators/features/upcall/src/upcall-cache-invalidation.h b/xlators/features/upcall/src/upcall-cache-invalidation.h new file mode 100644 index 00000000000..758ddf1dae8 --- /dev/null +++ b/xlators/features/upcall/src/upcall-cache-invalidation.h @@ -0,0 +1,79 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __UPCALL_CACHE_INVALIDATION_H__ +#define __UPCALL_CACHE_INVALIDATION_H__ + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +/* TODO: Below macros have to be replaced with + * xlator options - Bug1200271 */ +#define ON_CACHE_INVALIDATION 0 /* disable by default */ + +/* The time period for which a client will be notified of cache_invalidation + * events post its last access */ +#define CACHE_INVALIDATION_PERIOD 60 + +/* Flags sent for cache_invalidation */ +#define UP_NLINK 0x00000001 /* update nlink */ +#define UP_MODE 0x00000002 /* update mode and ctime */ +#define UP_OWN 0x00000004 /* update mode,uid,gid and ctime */ +#define UP_SIZE 0x00000008 /* update fsize */ +#define UP_TIMES 0x00000010 /* update all times */ +#define UP_ATIME 0x00000020 /* update atime only */ +#define UP_PERM 0x00000040 /* update fields needed for + permission checking */ +#define UP_RENAME 0x00000080 /* this is a rename op - + delete the cache entry */ +#define UP_FORGET 0x00000100 /* inode_forget on server side - + invalidate the cache entry */ + +/* for fops - open, read, lk, */ +#define UP_IDEMPOTENT_FLAGS (UP_ATIME) + +/* for fop - write, truncate */ +#define UP_WRITE_FLAGS (UP_SIZE | UP_TIMES) + +/* for fop - setattr */ +#define UP_ATTR_FLAGS (UP_SIZE | UP_TIMES | UP_OWN | \ + UP_MODE | UP_PERM) +/* for fop - rename */ +#define UP_RENAME_FLAGS (UP_RENAME) + +/* to invalidate parent directory entries for fops -rename, unlink, + * rmdir, mkdir, create */ +#define UP_PARENT_DENTRY_FLAGS (UP_TIMES) + +/* for fop - unlink, link, rmdir, mkdir */ +#define UP_NLINK_FLAGS (UP_NLINK | UP_TIMES) + +#define CACHE_INVALIDATE(frame, this, client, inode, p_flags) do { \ + if (ON_CACHE_INVALIDATION) { \ + (void)upcall_cache_invalidate (frame, this, client, inode, p_flags); \ + } \ +} while (0) + +#define CACHE_INVALIDATE_DIR(frame, this, client, inode_p, p_flags) do { \ + if (ON_CACHE_INVALIDATION) { \ + dentry_t *dentry; \ + dentry_t *dentry_tmp; \ + list_for_each_entry_safe (dentry, dentry_tmp, \ + &inode_p->dentry_list, \ + inode_list) { \ + (void)upcall_cache_invalidate (frame, this, client, \ + dentry->inode, p_flags); \ + } \ + } \ +} while (0) + +#endif /* __UPCALL_CACHE_INVALIDATION_H__ */ diff --git a/xlators/features/upcall/src/upcall-internal.c b/xlators/features/upcall/src/upcall-internal.c new file mode 100644 index 00000000000..26473e2a7bd --- /dev/null +++ b/xlators/features/upcall/src/upcall-internal.c @@ -0,0 +1,424 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <unistd.h> +#include <fcntl.h> +#include <limits.h> + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include "glusterfs.h" +#include "compat.h" +#include "xlator.h" +#include "inode.h" +#include "logging.h" +#include "common-utils.h" + +#include "statedump.h" +#include "syncop.h" + +#include "upcall.h" +#include "upcall-mem-types.h" +#include "glusterfs3-xdr.h" +#include "protocol-common.h" +#include "defaults.h" + +/* + * Allocate and add a new client entry to the given upcall entry + */ +upcall_client_t* +add_upcall_client (call_frame_t *frame, uuid_t gfid, + client_t *client, + upcall_inode_ctx_t *up_inode_ctx) +{ + upcall_client_t *up_client_entry = NULL; + + pthread_mutex_lock (&up_inode_ctx->client_list_lock); + { + up_client_entry = __add_upcall_client (frame, + gfid, + client, + up_inode_ctx); + } + pthread_mutex_unlock (&up_inode_ctx->client_list_lock); + + return up_client_entry; +} + +upcall_client_t* +__add_upcall_client (call_frame_t *frame, uuid_t gfid, + client_t *client, + upcall_inode_ctx_t *up_inode_ctx) +{ + upcall_client_t *up_client_entry = NULL; + + up_client_entry = GF_CALLOC (1, sizeof(*up_client_entry), + gf_upcall_mt_upcall_client_entry_t); + if (!up_client_entry) { + gf_msg ("upcall", GF_LOG_WARNING, 0, + UPCALL_MSG_NO_MEMORY, + "Memory allocation failed"); + return NULL; + } + INIT_LIST_HEAD (&up_client_entry->client_list); + up_client_entry->client_uid = gf_strdup(client->client_uid); + up_client_entry->access_time = time(NULL); + up_client_entry->expire_time_attr = CACHE_INVALIDATION_PERIOD; + + list_add_tail (&up_client_entry->client_list, + &up_inode_ctx->client_list); + + gf_log (THIS->name, GF_LOG_DEBUG, "upcall_entry_t client added - %s", + up_client_entry->client_uid); + + return up_client_entry; +} + +/* + * Given gfid and client->uid, retrieve the corresponding upcall client entry. + * If none found, create a new entry. + */ +upcall_client_t* +__get_upcall_client (call_frame_t *frame, uuid_t gfid, client_t *client, + upcall_inode_ctx_t *up_inode_ctx) +{ + upcall_client_t *up_client_entry = NULL; + upcall_client_t *up_client = NULL; + upcall_client_t *tmp = NULL; + gf_boolean_t found_client = _gf_false; + + list_for_each_entry_safe (up_client_entry, tmp, + &up_inode_ctx->client_list, + client_list) { + if (strcmp(client->client_uid, + up_client_entry->client_uid) == 0) { + /* found client entry. Update the access_time */ + up_client_entry->access_time = time(NULL); + found_client = _gf_true; + gf_log (THIS->name, GF_LOG_DEBUG, + "upcall_entry_t client found - %s", + up_client_entry->client_uid); + break; + } + } + + if (!found_client) { /* create one */ + up_client_entry = __add_upcall_client (frame, gfid, client, + up_inode_ctx); + } + + return up_client_entry; +} + +int +__upcall_inode_ctx_set (inode_t *inode, xlator_t *this) +{ + upcall_inode_ctx_t *inode_ctx = NULL; + int ret = -1; + uint64_t ctx = 0; + + ret = __inode_ctx_get (inode, this, &ctx); + + if (!ret) + goto out; + + inode_ctx = GF_CALLOC (1, sizeof (upcall_inode_ctx_t), + gf_upcall_mt_upcall_inode_ctx_t); + + if (!inode_ctx) { + ret = -ENOMEM; + goto out; + } + + pthread_mutex_init (&inode_ctx->client_list_lock, NULL); + INIT_LIST_HEAD (&inode_ctx->client_list); + + ret = __inode_ctx_set (inode, this, (uint64_t *) inode_ctx); + if (ret) + gf_log (this->name, GF_LOG_DEBUG, + "failed to set inode ctx (%p)", inode); +out: + return ret; +} + +upcall_inode_ctx_t * +__upcall_inode_ctx_get (inode_t *inode, xlator_t *this) +{ + upcall_inode_ctx_t *inode_ctx = NULL; + uint64_t ctx = 0; + int ret = 0; + + ret = __inode_ctx_get (inode, this, &ctx); + + if (ret < 0) { + ret = __upcall_inode_ctx_set (inode, this); + if (ret < 0) + goto out; + + ret = __inode_ctx_get (inode, this, &ctx); + if (ret < 0) + goto out; + } + + inode_ctx = (upcall_inode_ctx_t *)(long) ctx; + +out: + return inode_ctx; +} + +upcall_inode_ctx_t * +upcall_inode_ctx_get (inode_t *inode, xlator_t *this) +{ + upcall_inode_ctx_t *inode_ctx = NULL; + + LOCK (&inode->lock); + { + inode_ctx = __upcall_inode_ctx_get (inode, this); + } + UNLOCK (&inode->lock); + + return inode_ctx; +} + +int +__upcall_cleanup_client_entry (upcall_client_t *up_client) +{ + list_del_init (&up_client->client_list); + + GF_FREE (up_client->client_uid); + GF_FREE (up_client); + + return 0; +} + +/* + * Free Upcall inode_ctx client list + */ +int +__upcall_cleanup_inode_ctx_client_list (upcall_inode_ctx_t *inode_ctx) +{ + upcall_client_t *up_client = NULL; + upcall_client_t *tmp = NULL; + + list_for_each_entry_safe (up_client, tmp, + &inode_ctx->client_list, + client_list) { + __upcall_cleanup_client_entry (up_client); + } + + return 0; +} + +/* + * Free upcall_inode_ctx + */ +int +upcall_cleanup_inode_ctx (xlator_t *this, inode_t *inode) +{ + uint64_t ctx = 0; + upcall_inode_ctx_t *inode_ctx = NULL; + int ret = 0; + + ret = inode_ctx_get (inode, this, &ctx); + + if (ret < 0) { + gf_log (THIS->name, GF_LOG_TRACE, + "Failed to get upcall_inode_ctx (%p)", + inode); + goto out; + } + + /* Invalidate all the upcall cache entries */ + upcall_cache_forget (this, inode, inode_ctx); + + /* Set inode context to NULL */ + ret = __inode_ctx_set (inode, this, NULL); + + if (!ret) { + gf_log (this->name, GF_LOG_WARNING, + "_inode_ctx_set to NULL failed (%p)", + inode); + } + inode_ctx = (upcall_inode_ctx_t *)(long) ctx; + + if (inode_ctx) { + /* do we really need lock? */ + pthread_mutex_lock (&inode_ctx->client_list_lock); + { + if (!list_empty (&inode_ctx->client_list)) { + __upcall_cleanup_inode_ctx_client_list (inode_ctx); + } + } + pthread_mutex_unlock (&inode_ctx->client_list_lock); + + pthread_mutex_destroy (&inode_ctx->client_list_lock); + + GF_FREE (inode_ctx); + } + +out: + return ret; +} + +/* + * Given a gfid, client, first fetch upcall_entry_t based on gfid. + * Later traverse through the client list of that upcall entry. If this client + * is not present in the list, create one client entry with this client info. + * Also check if there are other clients which need to be notified of this + * op. If yes send notify calls to them. + * + * Since sending notifications for cache_invalidation is a best effort, + * any errors during the process are logged and ignored. + */ +void +upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client, + inode_t *inode, uint32_t flags) +{ + upcall_client_t *up_client = NULL; + upcall_client_t *up_client_entry = NULL; + upcall_client_t *tmp = NULL; + upcall_inode_ctx_t *up_inode_ctx = NULL; + gf_boolean_t found = _gf_false; + + up_inode_ctx = ((upcall_local_t *)frame->local)->upcall_inode_ctx; + + if (!up_inode_ctx) + up_inode_ctx = upcall_inode_ctx_get (inode, this); + + if (!up_inode_ctx) { + gf_log (this->name, GF_LOG_WARNING, + "upcall_inode_ctx_get failed (%p)", + inode); + return; + } + + pthread_mutex_lock (&up_inode_ctx->client_list_lock); + { + list_for_each_entry_safe (up_client_entry, tmp, + &up_inode_ctx->client_list, + client_list) { + + if (!strcmp(client->client_uid, + up_client_entry->client_uid)) { + up_client_entry->access_time = time(NULL); + found = _gf_true; + } + + /* + * Ignore sending notifications in case of only UP_ATIME + */ + if (!(flags & ~(UP_ATIME))) { + if (found) + break; + else /* we still need to find current client entry*/ + continue; + } + + /* any other client */ + + /* XXX: Send notifications asynchrounously + * instead of in the I/O path - BZ 1200264 + * Also if the file is frequently accessed, set + * expire_time_attr to 0. + */ + upcall_client_cache_invalidate(this, + inode->gfid, + up_client_entry, + flags); + } + + if (!found) { + up_client_entry = __add_upcall_client (frame, + inode->gfid, + client, + up_inode_ctx); + } + } + pthread_mutex_unlock (&up_inode_ctx->client_list_lock); +} + +/* + * If the upcall_client_t has recently accessed the file (i.e, within + * CACHE_INVALIDATION_PERIOD), send a upcall notification. + */ +void +upcall_client_cache_invalidate (xlator_t *this, uuid_t gfid, + upcall_client_t *up_client_entry, + uint32_t flags) +{ + notify_event_data_t n_event_data; + time_t t_expired = time(NULL) - up_client_entry->access_time; + + if (t_expired < CACHE_INVALIDATION_PERIOD) { + /* Send notify call */ + uuid_copy(n_event_data.gfid, gfid); + n_event_data.client_entry = up_client_entry; + n_event_data.event_type = CACHE_INVALIDATION; + n_event_data.invalidate_flags = flags; + + /* Need to send inode flags */ + this->notify (this, GF_EVENT_UPCALL, &n_event_data); + + gf_log (THIS->name, GF_LOG_TRACE, + "Cache invalidation notification sent to %s", + up_client_entry->client_uid); + + } else { + if (t_expired > (2*CACHE_INVALIDATION_PERIOD)) { + /* Cleanup the entry */ + __upcall_cleanup_client_entry (up_client_entry); + } + + gf_log (THIS->name, GF_LOG_TRACE, + "Cache invalidation notification NOT sent to %s", + up_client_entry->client_uid); + } +} + +/* + * This is called during upcall_inode_ctx cleanup incase of 'inode_forget'. + * Send "UP_FORGET" to all the clients so that they invalidate their cache + * entry and do a fresh lookup next time when any I/O comes in. + */ +void +upcall_cache_forget (xlator_t *this, inode_t *inode, upcall_inode_ctx_t *up_inode_ctx) +{ + upcall_client_t *up_client = NULL; + upcall_client_t *up_client_entry = NULL; + upcall_client_t *tmp = NULL; + uint32_t flags = 0; + + if (!up_inode_ctx) { + return; + } + + pthread_mutex_lock (&up_inode_ctx->client_list_lock); + { + list_for_each_entry_safe (up_client_entry, tmp, + &up_inode_ctx->client_list, + client_list) { + flags = UP_FORGET; + + /* Set the access time to time(NULL) + * to send notify */ + up_client_entry->access_time = time(NULL); + + upcall_client_cache_invalidate(this, + inode->gfid, + up_client_entry, + flags); + } + + } + pthread_mutex_unlock (&up_inode_ctx->client_list_lock); +} diff --git a/xlators/features/upcall/src/upcall-mem-types.h b/xlators/features/upcall/src/upcall-mem-types.h new file mode 100644 index 00000000000..55793ec65ca --- /dev/null +++ b/xlators/features/upcall/src/upcall-mem-types.h @@ -0,0 +1,24 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __UPCALL_MEM_TYPES_H__ +#define __UPCALL_MEM_TYPES_H__ + +#include "mem-types.h" + +enum gf_upcall_mem_types_ { + gf_upcall_mt_conf_t = gf_common_mt_end + 1, + gf_upcall_mt_private_t, + gf_upcall_mt_upcall_inode_ctx_t, + gf_upcall_mt_upcall_client_entry_t, + gf_upcall_mt_end +}; +#endif + diff --git a/xlators/features/upcall/src/upcall-messages.h b/xlators/features/upcall/src/upcall-messages.h new file mode 100644 index 00000000000..c8483f16528 --- /dev/null +++ b/xlators/features/upcall/src/upcall-messages.h @@ -0,0 +1,62 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. + */ + +#ifndef _UPCALL_MESSAGES_H_ +#define _UPCALL_MESSAGES_H_ + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include "glfs-message-id.h" + +/*! \file upcall-messages.h + * \brief UPCALL log-message IDs and their descriptions. + */ + +/* NOTE: Rules for message additions + * 1) Each instance of a message is _better_ left with a unique message ID, even + * if the message format is the same. Reasoning is that, if the message + * format needs to change in one instance, the other instances are not + * impacted or the new change does not change the ID of the instance being + * modified. + * 2) Addition of a message, + * - Should increment the GLFS_NUM_MESSAGES + * - Append to the list of messages defined, towards the end + * - Retain macro naming as glfs_msg_X (for redability across developers) + * NOTE: Rules for message format modifications + * 3) Check across the code if the message ID macro in question is reused + * anywhere. If reused then then the modifications should ensure correctness + * everywhere, or needs a new message ID as (1) above was not adhered to. If + * not used anywhere, proceed with the required modification. + * NOTE: Rules for message deletion + * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used + * anywhere, then can be deleted, but will leave a hole by design, as + * addition rules specify modification to the end of the list and not filling + * holes. + */ + +#define GLFS_COMP_BASE_UPCALL GLFS_MSGID_COMP_UPCALL +#define GLFS_NUM_MESSAGES 1 +#define GLFS_MSGID_END (GLFS_COMP_BASE_UPCALL + GLFS_NUM_MESSAGES + 1) + +#define glfs_msg_start_x GLFS_COMP_BASE_UPCALL, "Invalid: Start of messages" + +/*! + * @messageid 110001 + * @diagnosis Out of Memory + * @recommendedaction None + */ +#define UPCALL_MSG_NO_MEMORY (GLFS_COMP_BASE_UPCALL + 1) + +#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages" + +#endif /* !_UPCALL_MESSAGES_H_ */ diff --git a/xlators/features/upcall/src/upcall.c b/xlators/features/upcall/src/upcall.c new file mode 100644 index 00000000000..b7f2e975bba --- /dev/null +++ b/xlators/features/upcall/src/upcall.c @@ -0,0 +1,956 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <unistd.h> +#include <fcntl.h> +#include <limits.h> +#include <pthread.h> + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include "glusterfs.h" +#include "compat.h" +#include "xlator.h" +#include "inode.h" +#include "logging.h" +#include "common-utils.h" + +#include "statedump.h" +#include "syncop.h" + +#include "upcall.h" +#include "upcall-mem-types.h" +#include "glusterfs3-xdr.h" +#include "protocol-common.h" +#include "defaults.h" + +int +up_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata) +{ + client_t *client = NULL; + uint32_t flags = 0; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + client = frame->root->client; + local = frame->local; + + if ((op_ret < 0) || !local) { + goto out; + } + flags = UP_IDEMPOTENT_FLAGS; + CACHE_INVALIDATE (frame, this, client, local->inode, flags); + +out: + UPCALL_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata); + + return 0; +} + + +int +up_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, + fd_t *fd, dict_t *xdata) +{ + int32_t op_errno = -1; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + local = upcall_local_init (frame, this, fd->inode); + if (!local) { + op_errno = ENOMEM; + goto err; + } + +out: + STACK_WIND (frame, up_open_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->open, + loc, flags, fd, xdata); + + return 0; + +err: + op_errno = (op_errno == -1) ? errno : op_errno; + UPCALL_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL); + + return 0; +} + +int +up_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) +{ + client_t *client = NULL; + uint32_t flags = 0; + upcall_local_t *local = NULL; + + client = frame->root->client; + local = frame->local; + + if ((op_ret < 0) || !local) { + goto out; + } + flags = UP_WRITE_FLAGS; + CACHE_INVALIDATE (frame, this, client, local->inode, flags); + +out: + UPCALL_STACK_UNWIND (writev, frame, op_ret, op_errno, + prebuf, postbuf, xdata); + + return 0; +} + + +int +up_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, + struct iovec *vector, int count, off_t off, uint32_t flags, + struct iobref *iobref, dict_t *xdata) +{ + int32_t op_errno = -1; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + local = upcall_local_init (frame, this, fd->inode); + if (!local) { + op_errno = ENOMEM; + goto err; + } + +out: + STACK_WIND (frame, up_writev_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev, + fd, vector, count, off, flags, iobref, xdata); + + return 0; + +err: + op_errno = (op_errno == -1) ? errno : op_errno; + op_errno = (op_errno == -1) ? errno : op_errno; + UPCALL_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL); + + return 0; +} + + +int +up_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, + struct iovec *vector, int count, struct iatt *stbuf, + struct iobref *iobref, dict_t *xdata) +{ + client_t *client = NULL; + uint32_t flags = 0; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + client = frame->root->client; + local = frame->local; + + if ((op_ret < 0) || !local) { + goto out; + } + flags = UP_IDEMPOTENT_FLAGS; + CACHE_INVALIDATE (frame, this, client, local->inode, flags); + +out: + UPCALL_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, + count, stbuf, iobref, xdata); + + return 0; +} + +int +up_readv (call_frame_t *frame, xlator_t *this, + fd_t *fd, size_t size, off_t offset, + uint32_t flags, dict_t *xdata) +{ + int32_t op_errno = -1; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + local = upcall_local_init (frame, this, fd->inode); + if (!local) { + op_errno = ENOMEM; + goto err; + } + +out: + STACK_WIND (frame, up_readv_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv, + fd, size, offset, flags, xdata); + + return 0; + +err: + op_errno = (op_errno == -1) ? errno : op_errno; + UPCALL_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, + NULL, NULL, NULL); + + return 0; +} + +int32_t +up_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct gf_flock *lock, + dict_t *xdata) +{ + client_t *client = NULL; + uint32_t flags = 0; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + client = frame->root->client; + local = frame->local; + + if ((op_ret < 0) || !local) { + goto out; + } + flags = UP_IDEMPOTENT_FLAGS; + CACHE_INVALIDATE (frame, this, client, local->inode, flags); + +out: + UPCALL_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata); + + return 0; +} + +int +up_lk (call_frame_t *frame, xlator_t *this, + fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata) +{ + int32_t op_errno = -1; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + local = upcall_local_init (frame, this, fd->inode); + if (!local) { + op_errno = ENOMEM; + goto err; + } + +out: + STACK_WIND (frame, up_lk_cbk, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->lk, + fd, cmd, flock, xdata); + return 0; + +err: + op_errno = (op_errno == -1) ? errno : op_errno; + UPCALL_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL); + + return 0; +} + +int +up_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) +{ + client_t *client = NULL; + uint32_t flags = 0; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + client = frame->root->client; + local = frame->local; + + if ((op_ret < 0) || !local) { + goto out; + } + flags = UP_WRITE_FLAGS; + CACHE_INVALIDATE (frame, this, client, local->inode, flags); + +out: + UPCALL_STACK_UNWIND (truncate, frame, op_ret, op_errno, + prebuf, postbuf, xdata); + + return 0; +} + +int +up_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, + dict_t *xdata) +{ + int32_t op_errno = -1; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + local = upcall_local_init (frame, this, loc->inode); + if (!local) { + op_errno = ENOMEM; + goto err; + } + +out: + STACK_WIND (frame, up_truncate_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate, + loc, offset, xdata); + + return 0; + +err: + op_errno = (op_errno == -1) ? errno : op_errno; + UPCALL_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL); + + return 0; +} + +int +up_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, struct iatt *statpre, + struct iatt *statpost, dict_t *xdata) +{ + client_t *client = NULL; + uint32_t flags = 0; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + client = frame->root->client; + local = frame->local; + + if ((op_ret < 0) || !local) { + goto out; + } + /* XXX: setattr -> UP_SIZE or UP_OWN or UP_MODE or UP_TIMES + * or INODE_UPDATE (or UP_PERM esp incase of ACLs -> INODE_INVALIDATE) + * Need to check what attr is changed and accordingly pass UP_FLAGS. + * Bug1200271. + */ + flags = UP_ATTR_FLAGS; + CACHE_INVALIDATE (frame, this, client, local->inode, flags); + +out: + UPCALL_STACK_UNWIND (setattr, frame, op_ret, op_errno, + statpre, statpost, xdata); + + return 0; +} + +int +up_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, + struct iatt *stbuf, int32_t valid, dict_t *xdata) +{ + int32_t op_errno = -1; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + local = upcall_local_init (frame, this, loc->inode); + if (!local) { + op_errno = ENOMEM; + goto err; + } + +out: + STACK_WIND (frame, up_setattr_cbk, + FIRST_CHILD(this), + FIRST_CHILD(this)->fops->setattr, + loc, stbuf, valid, xdata); + + return 0; + +err: + op_errno = (op_errno == -1) ? errno : op_errno; + UPCALL_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL); + + return 0; +} + +int +up_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *stbuf, + struct iatt *preoldparent, struct iatt *postoldparent, + struct iatt *prenewparent, struct iatt *postnewparent, + dict_t *xdata) +{ + client_t *client = NULL; + uint32_t flags = 0; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + client = frame->root->client; + local = frame->local; + + if ((op_ret < 0) || !local) { + goto out; + } + flags = UP_RENAME_FLAGS; + CACHE_INVALIDATE (frame, this, client, local->inode, flags); + + /* Need to invalidate old and new parent entries as well */ + flags = UP_PARENT_DENTRY_FLAGS; + CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags); + + /* XXX: notify oldparent as well */ +/* if (uuid_compare (preoldparent->ia_gfid, prenewparent->ia_gfid)) + CACHE_INVALIDATE (frame, this, client, prenewparent->ia_gfid, flags);*/ + +out: + UPCALL_STACK_UNWIND (rename, frame, op_ret, op_errno, + stbuf, preoldparent, postoldparent, + prenewparent, postnewparent, xdata); + + return 0; +} + +int +up_rename (call_frame_t *frame, xlator_t *this, + loc_t *oldloc, loc_t *newloc, dict_t *xdata) +{ + int32_t op_errno = -1; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + local = upcall_local_init (frame, this, oldloc->inode); + if (!local) { + op_errno = ENOMEM; + goto err; + } + +out: + STACK_WIND (frame, up_rename_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename, + oldloc, newloc, xdata); + + return 0; + +err: + op_errno = (op_errno == -1) ? errno : op_errno; + UPCALL_STACK_UNWIND (rename, frame, -1, op_errno, NULL, + NULL, NULL, NULL, NULL, NULL); + + return 0; +} + +int +up_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) +{ + client_t *client = NULL; + uint32_t flags = 0; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + client = frame->root->client; + local = frame->local; + + if ((op_ret < 0) || !local) { + goto out; + } + flags = UP_NLINK_FLAGS; + CACHE_INVALIDATE (frame, this, client, local->inode, flags); + + flags = UP_PARENT_DENTRY_FLAGS; + /* invalidate parent's entry too */ + CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags); + +out: + UPCALL_STACK_UNWIND (unlink, frame, op_ret, op_errno, + preparent, postparent, xdata); + + return 0; +} + +int +up_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag, + dict_t *xdata) +{ + int32_t op_errno = -1; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + local = upcall_local_init (frame, this, loc->inode); + if (!local) { + op_errno = ENOMEM; + goto err; + } + +out: + STACK_WIND (frame, up_unlink_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink, + loc, xflag, xdata); + + return 0; + +err: + op_errno = (op_errno == -1) ? errno : op_errno; + UPCALL_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL); + + return 0; +} + +int +up_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf, + struct iatt *preparent, struct iatt *postparent, dict_t *xdata) +{ + client_t *client = NULL; + uint32_t flags = 0; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + client = frame->root->client; + local = frame->local; + + if ((op_ret < 0) || !local) { + goto out; + } + flags = UP_NLINK_FLAGS; + CACHE_INVALIDATE (frame, this, client, local->inode, flags); + + /* do we need to update parent as well?? */ +out: + UPCALL_STACK_UNWIND (link, frame, op_ret, op_errno, + inode, stbuf, preparent, postparent, xdata); + + return 0; +} + +int +up_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, + loc_t *newloc, dict_t *xdata) +{ + int32_t op_errno = -1; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + local = upcall_local_init (frame, this, oldloc->inode); + if (!local) { + op_errno = ENOMEM; + goto err; + } + +out: + STACK_WIND (frame, up_link_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->link, + oldloc, newloc, xdata); + + return 0; + +err: + op_errno = (op_errno == -1) ? errno : op_errno; + UPCALL_STACK_UNWIND (link, frame, -1, op_errno, NULL, + NULL, NULL, NULL, NULL); + + return 0; +} + +int +up_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) +{ + client_t *client = NULL; + uint32_t flags = 0; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + client = frame->root->client; + local = frame->local; + + if ((op_ret < 0) || !local) { + goto out; + } + flags = UP_NLINK_FLAGS; + CACHE_INVALIDATE (frame, this, client, local->inode, flags); + + /* invalidate parent's entry too */ + flags = UP_PARENT_DENTRY_FLAGS; + CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags); + +out: + UPCALL_STACK_UNWIND (rmdir, frame, op_ret, op_errno, + preparent, postparent, xdata); + + return 0; +} + +int +up_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, + dict_t *xdata) +{ + int32_t op_errno = -1; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + local = upcall_local_init (frame, this, loc->inode); + if (!local) { + op_errno = ENOMEM; + goto err; + } + +out: + STACK_WIND (frame, up_rmdir_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir, + loc, flags, xdata); + + return 0; + +err: + op_errno = (op_errno == -1) ? errno : op_errno; + UPCALL_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL); + + return 0; +} + +int +up_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, inode_t *inode, + struct iatt *stbuf, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) +{ + client_t *client = NULL; + uint32_t flags = 0; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + client = frame->root->client; + local = frame->local; + + if ((op_ret < 0) || !local) { + goto out; + } + flags = UP_NLINK_FLAGS; + CACHE_INVALIDATE (frame, this, client, local->inode, flags); + + /* invalidate parent's entry too */ + flags = UP_PARENT_DENTRY_FLAGS; + CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags); + +out: + UPCALL_STACK_UNWIND (mkdir, frame, op_ret, op_errno, + inode, stbuf, preparent, postparent, xdata); + + return 0; +} + +int +up_mkdir (call_frame_t *frame, xlator_t *this, + loc_t *loc, mode_t mode, mode_t umask, dict_t *params) +{ + int32_t op_errno = -1; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + local = upcall_local_init (frame, this, loc->inode); + if (!local) { + op_errno = ENOMEM; + goto err; + } + +out: + STACK_WIND (frame, up_mkdir_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir, + loc, mode, umask, params); + + return 0; + +err: + op_errno = (op_errno == -1) ? errno : op_errno; + UPCALL_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, + NULL, NULL, NULL, NULL); + + return 0; +} + +int +up_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, fd_t *fd, inode_t *inode, + struct iatt *stbuf, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) +{ + client_t *client = NULL; + uint32_t flags = 0; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + client = frame->root->client; + local = frame->local; + + if ((op_ret < 0) || !local) { + goto out; + } + + /* As its a new file create, no need of sending notification */ + /* However invalidate parent's entry */ + flags = UP_PARENT_DENTRY_FLAGS; + CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags); + +out: + UPCALL_STACK_UNWIND (create, frame, op_ret, op_errno, fd, + inode, stbuf, preparent, postparent, xdata); + + return 0; +} + +int +up_create (call_frame_t *frame, xlator_t *this, + loc_t *loc, int32_t flags, mode_t mode, + mode_t umask, fd_t *fd, dict_t *params) +{ + int32_t op_errno = -1; + upcall_local_t *local = NULL; + + EXIT_IF_UPCALL_OFF (out); + + local = upcall_local_init (frame, this, loc->inode); + + if (!local) { + op_errno = ENOMEM; + goto err; + } + +out: + STACK_WIND (frame, up_create_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->create, + loc, flags, mode, umask, fd, params); + + return 0; + +err: + op_errno = (op_errno == -1) ? errno : op_errno; + UPCALL_STACK_UNWIND (create, frame, -1, op_errno, NULL, + NULL, NULL, NULL, NULL, NULL); + + return 0; +} + +int32_t +mem_acct_init (xlator_t *this) +{ + int ret = -1; + + if (!this) + return ret; + + ret = xlator_mem_acct_init (this, gf_upcall_mt_end + 1); + + if (ret != 0) { + gf_msg ("upcall", GF_LOG_WARNING, 0, + UPCALL_MSG_NO_MEMORY, + "Memory allocation failed"); + return ret; + } + + return ret; +} + +void +upcall_local_wipe (xlator_t *this, upcall_local_t *local) +{ + if (local) { + inode_unref (local->inode); + mem_put (local); + } +} + +upcall_local_t * +upcall_local_init (call_frame_t *frame, xlator_t *this, inode_t *inode) +{ + upcall_local_t *local = NULL; + + local = mem_get0 (THIS->local_pool); + + if (!local) + goto out; + + local->inode = inode_ref (inode); + + /* Shall we get inode_ctx and store it here itself? */ + local->upcall_inode_ctx = upcall_inode_ctx_get (inode, this); + + frame->local = local; + +out: + return local; +} + +int +init (xlator_t *this) +{ + int ret = -1; + upcalls_private_t *priv = NULL; + + priv = GF_CALLOC (1, sizeof (*priv), + gf_upcall_mt_private_t); + if (!priv) { + gf_msg ("upcall", GF_LOG_WARNING, 0, + UPCALL_MSG_NO_MEMORY, + "Memory allocation failed"); + goto out; + } + + this->private = priv; + this->local_pool = mem_pool_new (upcall_local_t, 512); + ret = 0; + +out: + if (ret) { + GF_FREE (priv); + } + + return ret; +} + +int +fini (xlator_t *this) +{ + upcalls_private_t *priv = NULL; + + priv = this->private; + if (!priv) { + return 0; + } + this->private = NULL; + GF_FREE (priv); + + return 0; +} + +int +upcall_forget (xlator_t *this, inode_t *inode) +{ + upcall_cleanup_inode_ctx (this, inode); + return 0; +} + +int +upcall_release (xlator_t *this, fd_t *fd) +{ + return 0; +} + +int +notify (xlator_t *this, int32_t event, void *data, ...) +{ + int ret = -1; + int32_t val = 0; + notify_event_data_t *notify_event = NULL; + struct gf_upcall up_req = {0,}; + upcall_client_t *up_client_entry = NULL; + + switch (event) { + case GF_EVENT_UPCALL: + { + gf_log (this->name, GF_LOG_DEBUG, "Upcall Notify event = %d", + event); + + notify_event = (notify_event_data_t *) data; + up_client_entry = notify_event->client_entry; + + if (!up_client_entry) { + goto out; + } + + up_req.client_uid = up_client_entry->client_uid; + + memcpy (up_req.gfid, notify_event->gfid, 16); + gf_log (this->name, GF_LOG_DEBUG, + "Sending notify to the client- %s, gfid - %s", + up_client_entry->client_uid, up_req.gfid); + + switch (notify_event->event_type) { + case CACHE_INVALIDATION: + GF_ASSERT (notify_event->extra); + up_req.flags = notify_event->invalidate_flags; + up_req.expire_time_attr = up_client_entry->expire_time_attr; + break; + default: + goto out; + } + + up_req.event_type = notify_event->event_type; + + ret = default_notify (this, event, &up_req); + + /* + * notify may fail as the client could have been + * dis(re)connected. Cleanup the client entry. + */ + if (ret < 0) + __upcall_cleanup_client_entry (up_client_entry); + } + break; + default: + default_notify (this, event, data); + break; + } + ret = 0; + +out: + return ret; +} + +struct xlator_fops fops = { + .open = up_open, + .readv = up_readv, + .writev = up_writev, + .truncate = up_truncate, + .lk = up_lk, + .setattr = up_setattr, + .rename = up_rename, + .unlink = up_unlink, /* invalidate both file and parent dir */ + .rmdir = up_rmdir, /* same as above */ + .link = up_link, /* invalidate both file and parent dir */ + .create = up_create, /* update only direntry */ + .mkdir = up_mkdir, /* update only dirent */ +#ifdef WIP + .ftruncate = up_ftruncate, /* reqd? */ + .getattr = up_getattr, /* ?? */ + .getxattr = up_getxattr, /* ?? */ + .access = up_access, + .lookup = up_lookup, + .symlink = up_symlink, /* invalidate both file and parent dir maybe */ + .readlink = up_readlink, /* Needed? readlink same as read? */ + .readdirp = up_readdirp, + .readdir = up_readdir, +/* other fops to be considered - Bug1200271 + * lookup, stat, opendir, readdir, readdirp, readlink, mknod, statfs, flush, + * fsync, mknod, fsyncdir, setxattr, removexattr, rchecksum, fallocate, discard, + * zerofill, (also variants of above similar to fsetattr) + */ +#endif +}; + +struct xlator_cbks cbks = { + .forget = upcall_forget, + .release = upcall_release, +}; + +struct volume_options options[] = { + { .key = {NULL} }, +}; diff --git a/xlators/features/upcall/src/upcall.h b/xlators/features/upcall/src/upcall.h new file mode 100644 index 00000000000..a5aff9d091e --- /dev/null +++ b/xlators/features/upcall/src/upcall.h @@ -0,0 +1,134 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ +#ifndef __UPCALL_H__ +#define __UPCALL_H__ + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include "compat-errno.h" +#include "stack.h" +#include "call-stub.h" +#include "upcall-mem-types.h" +#include "client_t.h" +#include "rpcsvc.h" +#include "lkowner.h" +#include "locking.h" +#include "upcall-messages.h" +#include "upcall-cache-invalidation.h" + +#define EXIT_IF_UPCALL_OFF(label) do { \ + if (!(ON_CACHE_INVALIDATION)) \ + goto label; \ +} while (0) + +#define UPCALL_STACK_UNWIND(fop, frame, params ...) do { \ + upcall_local_t *__local = NULL; \ + xlator_t *__xl = NULL; \ + if (frame) { \ + __xl = frame->this; \ + __local = frame->local; \ + frame->local = NULL; \ + } \ + STACK_UNWIND_STRICT (fop, frame, params); \ + upcall_local_wipe (__xl, __local); \ + } while (0) + +#define UPCALL_STACK_DESTROY(frame) do { \ + upcall_local_t *__local = NULL; \ + xlator_t *__xl = NULL; \ + __xl = frame->this; \ + __local = frame->local; \ + frame->local = NULL; \ + STACK_DESTROY (frame->root); \ + upcall_local_wipe (__xl, __local); \ + } while (0) + +struct _upcalls_private_t { + int client_id; /* Not sure if reqd */ +}; +typedef struct _upcalls_private_t upcalls_private_t; + +enum _upcall_event_type_t { + EVENT_NULL, + CACHE_INVALIDATION, +}; +typedef enum _upcall_event_type_t upcall_event_type_t; + +struct _upcall_client_t { + struct list_head client_list; + /* strdup to store client_uid, strdup. Free it explicitly */ + char *client_uid; + time_t access_time; /* time last accessed */ + /* the amount of time which client can cache this entry */ + uint32_t expire_time_attr; +}; +typedef struct _upcall_client_t upcall_client_t; + +/* Upcall entries are maintained in inode_ctx */ +struct _upcall_inode_ctx_t { + struct list_head client_list; + pthread_mutex_t client_list_lock; /* mutex for clients list + of this upcall entry */ +}; +typedef struct _upcall_inode_ctx_t upcall_inode_ctx_t; + +struct _notify_event_data { + uuid_t gfid; + upcall_client_t *client_entry; + upcall_event_type_t event_type; + uint32_t invalidate_flags; + /* any extra data needed, like inode flags + * to be invalidated incase of cache invalidation, + * may be fd for lease recalls */ + void *extra; +}; +typedef struct _notify_event_data notify_event_data_t; + +struct upcall_local { + /* XXX: need to check if we can store + * pointers in 'local' which may get freed + * in future by other thread + */ + upcall_inode_ctx_t *upcall_inode_ctx; + inode_t *inode; +}; +typedef struct upcall_local upcall_local_t; + +void upcall_local_wipe (xlator_t *this, upcall_local_t *local); +upcall_local_t *upcall_local_init (call_frame_t *frame, xlator_t *this, inode_t *inode); + +upcall_client_t *add_upcall_client (call_frame_t *frame, uuid_t gfid, + client_t *client, + upcall_inode_ctx_t *up_inode_ctx); +upcall_client_t *__add_upcall_client (call_frame_t *frame, uuid_t gfid, + client_t *client, + upcall_inode_ctx_t *up_inode_ctx); +upcall_client_t *__get_upcall_client (call_frame_t *frame, uuid_t gfid, + client_t *client, + upcall_inode_ctx_t *up_inode_ctx); +int __upcall_cleanup_client_entry (upcall_client_t *up_client); + +int __upcall_inode_ctx_set (inode_t *inode, xlator_t *this); +upcall_inode_ctx_t *__upcall_inode_ctx_get (inode_t *inode, xlator_t *this); +upcall_inode_ctx_t *upcall_inode_ctx_get (inode_t *inode, xlator_t *this); +int upcall_cleanup_inode_ctx (xlator_t *this, inode_t *inode); + +void upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client, + inode_t *inode, uint32_t flags); +void upcall_client_cache_invalidate (xlator_t *xl, uuid_t gfid, + upcall_client_t *up_client_entry, + uint32_t flags); +void upcall_cache_forget (xlator_t *this, inode_t *inode, + upcall_inode_ctx_t *up_inode_ctx); + +#endif /* __UPCALL_H__ */ diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c index 779d6be34a4..f20490423f6 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.c +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c @@ -569,7 +569,6 @@ optget_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme, return 0; } - /* This getter considers defaults also. */ static int volgen_dict_get (dict_t *dict, char *key, char **value) @@ -1848,6 +1847,29 @@ out: } static int +brick_graph_add_upcall (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, + dict_t *set_dict, glusterd_brickinfo_t *brickinfo) +{ + + xlator_t *xl = NULL; + int ret = -1; + + if (!graph || !volinfo || !set_dict) + goto out; + + xl = volgen_graph_add (graph, "features/upcall", volinfo->volname); + if (!xl) { + gf_log ("glusterd", GF_LOG_WARNING, + "failed to add features/upcall to graph"); + goto out; + } + + ret = 0; +out: + return ret; +} + +static int brick_graph_add_server (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, dict_t *set_dict, glusterd_brickinfo_t *brickinfo) { @@ -2044,6 +2066,7 @@ static volgen_brick_xlator_t server_graph_table[] = { {brick_graph_add_index, "index"}, {brick_graph_add_barrier, NULL}, {brick_graph_add_iot, "io-threads"}, + {brick_graph_add_upcall, "upcall"}, {brick_graph_add_pump, NULL}, {brick_graph_add_locks, "locks"}, {brick_graph_add_acl, "acl"}, diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h index 947de76c926..0d742aae056 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.h +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h @@ -87,6 +87,7 @@ typedef enum { GF_XLATOR_POSIX = 0, GF_XLATOR_ACL, GF_XLATOR_LOCKS, + GF_XLATOR_UPCALL, GF_XLATOR_IOT, GF_XLATOR_INDEX, GF_XLATOR_MARKER, diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c index d7eab16bf13..023f2a6234f 100644 --- a/xlators/protocol/server/src/server.c +++ b/xlators/protocol/server/src/server.c @@ -27,6 +27,12 @@ #include "authenticate.h" #include "event.h" +rpcsvc_cbk_program_t server_cbk_prog = { + .progname = "Gluster Callback", + .prognum = GLUSTER_CBK_PROGRAM, + .progver = GLUSTER_CBK_VERSION, +}; + void grace_time_handler (void *data) { @@ -1093,22 +1099,72 @@ fini (xlator_t *this) int notify (xlator_t *this, int32_t event, void *data, ...) { - int ret = 0; - int32_t val = 0; - dict_t *dict = NULL; - dict_t *output = NULL; - va_list ap; + int ret = 0; + int32_t val = 0; + dict_t *dict = NULL; + dict_t *output = NULL; + va_list ap; + client_t *client = NULL; + char *client_uid = NULL; + struct gf_upcall *upcall_data = NULL; + gfs3_upcall_req up_req; + server_conf_t *conf = NULL; + rpc_transport_t *xprt = NULL; dict = data; va_start (ap, data); output = va_arg (ap, dict_t*); va_end (ap); + conf = this->private; + if (!conf) + return 0; + switch (event) { + case GF_EVENT_UPCALL: + { + if (!data) { + ret = -1; + goto out; + } + + upcall_data = (struct gf_upcall *)data; + + client_uid = upcall_data->client_uid; + + if (!client_uid) { + ret = -1; + goto out; + } + + gf_proto_upcall_from_upcall (&up_req, upcall_data); + + pthread_mutex_lock (&conf->mutex); + { + list_for_each_entry (xprt, &conf->xprt_list, list) { + client = xprt->xl_private; + + if (strcmp(client->client_uid, client_uid)) + continue; + + rpcsvc_request_submit( + conf->rpc, xprt, + &server_cbk_prog, + GF_CBK_UPCALL, + &up_req, + this->ctx, + (xdrproc_t)xdr_gfs3_upcall_req); + break; + } + } + pthread_mutex_unlock (&conf->mutex); + break; + } default: default_notify (this, event, data); break; } +out: return ret; } |