summaryrefslogtreecommitdiffstats
path: root/api/src
diff options
context:
space:
mode:
Diffstat (limited to 'api/src')
-rw-r--r--api/src/gfapi.aliases3
-rw-r--r--api/src/gfapi.map3
-rw-r--r--api/src/glfs-fops.c85
-rw-r--r--api/src/glfs-handleops.c129
-rw-r--r--api/src/glfs-handles.h83
-rw-r--r--api/src/glfs-internal.h27
-rw-r--r--api/src/glfs-master.c3
-rw-r--r--api/src/glfs-mem-types.h1
-rw-r--r--api/src/glfs.c40
9 files changed, 370 insertions, 4 deletions
diff --git a/api/src/gfapi.aliases b/api/src/gfapi.aliases
index 2ab7d443eb5..6dfc1089d86 100644
--- a/api/src/gfapi.aliases
+++ b/api/src/gfapi.aliases
@@ -126,8 +126,9 @@ _pub_glfs_get_volfile _glfs_get_volfile$GFAPI_3.6.0
_pub_glfs_h_access _glfs_h_access$GFAPI_3.6.0
_pub_glfs_ipc _glfs_ipc$GFAPI_3.7.0
+_pub_glfs_h_poll_upcall _glfs_h_poll_upcall$GFAPI_3.7.0
_priv_glfs_free_from_ctx _glfs_free_from_ctx$GFAPI_PRIVATE_3.7.0
_priv_glfs_new_from_ctx _glfs_new_from_ctx$GFAPI_PRIVATE_3.7.0
_priv_glfs_resolve _glfs_resolve$GFAPI_PRIVATE_3.7.0
-
+_priv_glfs_process_upcall_event _glfs_process_upcall_event$GFAPI_PRIVATE_3.7.0
diff --git a/api/src/gfapi.map b/api/src/gfapi.map
index 39202e1883f..4721efdff80 100644
--- a/api/src/gfapi.map
+++ b/api/src/gfapi.map
@@ -148,6 +148,7 @@ GFAPI_3.6.0 {
GFAPI_3.7.0 {
global:
glfs_ipc;
+ glfs_h_poll_upcall;
} GFAPI_3.6.0;
GFAPI_PRIVATE_3.7.0 {
@@ -155,5 +156,5 @@ GFAPI_PRIVATE_3.7.0 {
glfs_free_from_ctx;
glfs_new_from_ctx;
glfs_resolve;
+ glfs_process_upcall_event;
} GFAPI_3.7.0;
-
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index 182317fa41a..f0c769def29 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -16,6 +16,7 @@
#include "glfs.h"
#include "compat-errno.h"
#include <limits.h>
+#include "glusterfs3.h"
#ifdef NAME_MAX
#define GF_NAME_MAX NAME_MAX
@@ -3495,3 +3496,87 @@ out:
GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0);
+/*
+ * This routine is called in case of any notification received
+ * from the server. All the upcall events are queued up in a list
+ * to be read by the applications.
+ *
+ * XXX: Applications may register a cbk function for each 'fs'
+ * which then needs to be called by this routine incase of any
+ * event received. The cbk fn is responsible for notifying the
+ * applications the way it desires for each event queued (for eg.,
+ * can raise a signal or broadcast a cond variable etc.)
+ */
+void
+priv_glfs_process_upcall_event (struct glfs *fs, void *data)
+{
+ int ret = -1;
+ inode_t *inode = NULL;
+ uuid_t gfid;
+ upcall_entry *u_list = NULL;
+ glusterfs_ctx_t *ctx = NULL;
+ struct gf_upcall *upcall_data = NULL;
+ struct glfs_object *object = NULL;
+
+ gf_log (THIS->name, GF_LOG_DEBUG,
+ "Upcall gfapi callback is called");
+
+ if (!fs || !data)
+ goto out;
+
+ /* Unlike in I/O path, "glfs_fini" would not have freed
+ * 'fs' by the time we take lock as it waits for all epoll
+ * threads to exit including this
+ */
+ pthread_mutex_lock (&fs->mutex);
+ {
+ ctx = fs->ctx;
+
+ if (ctx->cleanup_started) {
+ pthread_mutex_unlock (&fs->mutex);
+ goto out;
+ }
+
+ fs->pin_refcnt++;
+ }
+ pthread_mutex_unlock (&fs->mutex);
+
+ upcall_data = (struct gf_upcall *)data;
+
+ gf_log (THIS->name, GF_LOG_DEBUG, "Upcall gfapi gfid = %s"
+ "ret = %d", (char *)(upcall_data->gfid), ret);
+
+ memcpy(gfid, (char *)(upcall_data->gfid), 16);
+ u_list = GF_CALLOC (1, sizeof(*u_list),
+ glfs_mt_upcall_entry_t);
+
+ if (!u_list) {
+ gf_log (THIS->name, GF_LOG_ERROR, "Upcall entry allocation"
+ "failed.");
+ goto out;
+ }
+
+ INIT_LIST_HEAD (&u_list->upcall_list);
+
+ uuid_copy (u_list->gfid, gfid);
+ u_list->event_type = upcall_data->event_type;
+ u_list->flags = (uint32_t)(upcall_data->flags);
+ u_list->expire_time_attr = upcall_data->expire_time_attr;
+
+ pthread_mutex_lock (&fs->upcall_list_mutex);
+ {
+ list_add_tail (&u_list->upcall_list,
+ &fs->upcall_list);
+ }
+ pthread_mutex_unlock (&fs->upcall_list_mutex);
+
+ pthread_mutex_lock (&fs->mutex);
+ {
+ fs->pin_refcnt--;
+ }
+ pthread_mutex_unlock (&fs->mutex);
+out:
+ return;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0);
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
index 631af01d97b..037315a518d 100644
--- a/api/src/glfs-handleops.c
+++ b/api/src/glfs-handleops.c
@@ -1594,3 +1594,132 @@ out:
GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2);
+/*
+ * This API is used to poll for upcall events stored in the
+ * upcall list. Current users of this API is NFS-Ganesha.
+ * Incase of any event received, it will be mapped appropriately
+ * into 'callback_arg' along with the handle to be passed to
+ * NFS-Ganesha.
+ *
+ * Application is responsible for allocating and passing the
+ * references of all the pointers except for "glhandle".
+ * After processing the event, it needs to free "glhandle"
+ *
+ * TODO: there should be a glfs api to destroy these handles,
+ * maybe "glfs_destroy_object" to free the object.
+ *
+ * Also similar to I/Os, the application should ideally stop polling
+ * before calling glfs_fini(..). Hence making an assumption that
+ * 'fs' & ctx structures cannot be freed while in this routine.
+ */
+int
+pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
+{
+ struct glfs_object *handle = NULL;
+ uuid_t gfid;
+ upcall_entry *u_list = NULL;
+ upcall_entry *tmp = NULL;
+ xlator_t *subvol = NULL;
+ int found = 0;
+ int reason = 0;
+ glusterfs_ctx_t *ctx = NULL;
+ int ret = -1;
+
+ if (!fs || !up_arg) {
+ errno = EINVAL;
+ goto err;
+ }
+
+ __glfs_entry_fs (fs);
+
+ /* get the active volume */
+ subvol = glfs_active_subvol (fs);
+
+ if (!subvol) {
+ errno = EIO;
+ goto err;
+ }
+
+ up_arg->handle = NULL;
+
+ /* Ideally applications should stop polling before calling
+ * 'glfs_fini'. Yet cross check if cleanup has started
+ */
+ pthread_mutex_lock (&fs->mutex);
+ {
+ ctx = fs->ctx;
+
+ if (ctx->cleanup_started) {
+ pthread_mutex_unlock (&fs->mutex);
+ goto out;
+ }
+
+ fs->pin_refcnt++;
+ }
+ pthread_mutex_unlock (&fs->mutex);
+
+ pthread_mutex_lock (&fs->upcall_list_mutex);
+ {
+ list_for_each_entry_safe (u_list, tmp,
+ &fs->upcall_list,
+ upcall_list) {
+ uuid_copy (gfid, u_list->gfid);
+ found = 1;
+ break;
+ }
+ }
+ /* No other thread can delete this entry. So unlock it */
+ pthread_mutex_unlock (&fs->upcall_list_mutex);
+
+ if (found) {
+ handle = glfs_h_create_from_handle (fs, gfid,
+ GFAPI_HANDLE_LENGTH,
+ &up_arg->buf);
+
+ if (!handle) {
+ errno = ENOMEM;
+ goto out;
+ }
+
+ switch (u_list->event_type) {
+ case CACHE_INVALIDATION:
+ if (u_list->flags & (~(INODE_UPDATE_FLAGS))) {
+ /* Invalidate CACHE */
+ reason = INODE_INVALIDATE;
+ gf_log (subvol->name, GF_LOG_DEBUG,
+ "Reason - INODE_INVALIDATION");
+ } else {
+ reason = INODE_UPDATE;
+ gf_log (subvol->name, GF_LOG_DEBUG,
+ "Reason - INODE_UPDATE");
+ }
+ break;
+ default:
+ break;
+ }
+
+ up_arg->handle = handle;
+ up_arg->reason = reason;
+ up_arg->flags = u_list->flags;
+ up_arg->expire_time_attr = u_list->expire_time_attr;
+
+ list_del_init (&u_list->upcall_list);
+ GF_FREE (u_list);
+ }
+
+ ret = 0;
+
+out:
+ pthread_mutex_lock (&fs->mutex);
+ {
+ fs->pin_refcnt--;
+ }
+ pthread_mutex_unlock (&fs->mutex);
+
+ glfs_subvol_done (fs, subvol);
+
+err:
+ return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_poll_upcall, 3.7.0);
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
index 12bae6d6e86..c88f134b001 100644
--- a/api/src/glfs-handles.h
+++ b/api/src/glfs-handles.h
@@ -60,6 +60,22 @@
* glfs_h_create_from_handle */
#define GFAPI_HANDLE_LENGTH 16
+/* These flags should be in sync to the ones defined in upcall.h */
+#define UP_NLINK 0x00000001 /* update nlink */
+#define UP_MODE 0x00000002 /* update mode and ctime */
+#define UP_OWN 0x00000004 /* update mode,uid,gid and ctime */
+#define UP_SIZE 0x00000008 /* update fsize */
+#define UP_TIMES 0x00000010 /* update all times */
+#define UP_ATIME 0x00000020 /* update atime only */
+#define UP_PERM 0x00000040 /* update fields needed for
+ permission checking */
+#define UP_RENAME 0x00000080 /* this is a rename op -
+ delete the cache entry */
+
+#define INODE_UPDATE_FLAGS (UP_NLINK | UP_MODE | \
+ UP_OWN | UP_SIZE | \
+ UP_TIMES | UP_ATIME)
+
/* Portability non glibc c++ build systems */
#ifndef __THROW
# if defined __cplusplus
@@ -82,6 +98,36 @@ __BEGIN_DECLS
struct glfs_object;
typedef struct glfs_object glfs_object_t;
+/*
+ * Applications (currently NFS-Ganesha) can make use of this
+ * structure to read upcall notifications sent by server.
+ *
+ * They are responsible for allocating and passing the references
+ * of all the pointers except for "handle".
+ *
+ * After processing the event, they need to free "handle"
+ * TODO: there should be a glfs api to destroy these handles,
+ * maybe "glfs_destroy_object" to free the object.
+ */
+struct callback_arg {
+ struct glfs *fs; /* glfs object */
+ int reason; /* Upcall event type */
+ struct glfs_object *handle; /* Handle which need to be acted upon */
+ int flags; /* Cache UPDATE/INVALIDATE flags */
+ struct stat buf; /* Latest stat of this entry */
+ unsigned int expire_time_attr; /* the amount of time for which
+ * the application need to cache
+ * this entry
+ */
+};
+
+/* reason list in callback_arg */
+enum callback_type {
+ CBK_EVENT_NULL,
+ INODE_INVALIDATE,
+ INODE_UPDATE,
+};
+
/* Handle based operations */
/* Operations that generate handles */
struct glfs_object *glfs_h_lookupat (struct glfs *fs,
@@ -188,6 +234,43 @@ int
glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask) __THROW
GFAPI_PUBLIC(glfs_h_access, 3.6.0);
+/*
+ SYNOPSIS
+
+ glfs_h_poll_upcall: Poll for upcall events given a 'glfs' object.
+
+ DESCRIPTION
+
+ This API is used to poll for upcall events stored in the
+ upcall list. Current users of this API is NFS-Ganesha.
+ Incase of any event received, it will be mapped appropriately
+ into 'callback_arg' along with the handle('glfs_object') to be
+ passed to NFS-Ganesha.
+
+ In case of success, applications need to check the value of
+ cbk->handle to be NON NULL before processing the upcall
+ events.
+
+ PARAMETERS
+
+ @fs: glfs object to poll the upcall events for
+ @cbk: Structure to store upcall events as desired by the application.
+ Application is responsible for allocating and passing the
+ references of all the pointers of this structure except for
+ "handle". In case of any events received, it needs to free
+ "handle"
+
+ RETURN VALUES
+
+ 0 : Success.
+ -1 : Error condition, mostly due to out of memory.
+
+*/
+
+int
+glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *cbk) __THROW
+ GFAPI_PUBLIC(glfs_h_poll_upcall, 3.7.0);
+
__END_DECLS
#endif /* !_GLFS_HANDLES_H */
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index b704c558722..2c0dfe8074e 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -108,6 +108,25 @@
struct glfs;
+/* This enum should be in sync with
+ * 'upcall_event_type' declared in
+ * 'xlators/features/upcall/src/upcall.h'
+ */
+enum upcall_event_type_t {
+ EVENT_NULL,
+ CACHE_INVALIDATION,
+};
+typedef enum upcall_event_type_t upcall_event_type;
+
+struct _upcall_entry_t {
+ struct list_head upcall_list;
+ uuid_t gfid;
+ upcall_event_type event_type;
+ uint32_t flags;
+ uint32_t expire_time_attr;
+};
+typedef struct _upcall_entry_t upcall_entry;
+
typedef int (*glfs_init_cbk) (struct glfs *fs, int ret);
struct glfs {
@@ -140,6 +159,11 @@ struct glfs {
struct list_head openfds;
gf_boolean_t migration_in_progress;
+
+ struct list_head upcall_list;
+ pthread_mutex_t upcall_list_mutex; /* mutex for upcall entry list */
+
+ uint32_t pin_refcnt;
};
struct glfs_fd {
@@ -182,6 +206,9 @@ fd_t *__glfs_migrate_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd
int glfs_first_lookup (xlator_t *subvol);
+void glfs_process_upcall_event (struct glfs *fs, void *data);
+ GFAPI_PRIVATE(glfs_process_upcall_event, 3.7.0);
+
static inline void
__glfs_entry_fs (struct glfs *fs)
{
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
index 0e54719d72c..dfce2f9b78c 100644
--- a/api/src/glfs-master.c
+++ b/api/src/glfs-master.c
@@ -113,6 +113,9 @@ notify (xlator_t *this, int event, void *data, ...)
break;
case GF_EVENT_CHILD_CONNECTING:
break;
+ case GF_EVENT_UPCALL:
+ glfs_process_upcall_event (fs, data);
+ break;
default:
gf_log (this->name, GF_LOG_DEBUG,
"got notify event %d", event);
diff --git a/api/src/glfs-mem-types.h b/api/src/glfs-mem-types.h
index 0a2d4a7df22..c1883f089fd 100644
--- a/api/src/glfs-mem-types.h
+++ b/api/src/glfs-mem-types.h
@@ -25,6 +25,7 @@ enum glfs_mem_types_ {
glfs_mt_server_cmdline_t,
glfs_mt_glfs_object_t,
glfs_mt_readdirbuf_t,
+ glfs_mt_upcall_entry_t,
glfs_mt_end
};
diff --git a/api/src/glfs.c b/api/src/glfs.c
index f23481bbb4c..02a8984f450 100644
--- a/api/src/glfs.c
+++ b/api/src/glfs.c
@@ -600,6 +600,11 @@ pub_glfs_new (const char *volname)
INIT_LIST_HEAD (&fs->openfds);
+ INIT_LIST_HEAD (&fs->upcall_list);
+ pthread_mutex_init (&fs->upcall_list_mutex, NULL);
+
+ fs->pin_refcnt = 0;
+
return fs;
}
@@ -626,6 +631,11 @@ priv_glfs_new_from_ctx (glusterfs_ctx_t *ctx)
INIT_LIST_HEAD (&fs->openfds);
+ INIT_LIST_HEAD (&fs->upcall_list);
+ pthread_mutex_init (&fs->upcall_list_mutex, NULL);
+
+ fs->pin_refcnt = 0;
+
return fs;
}
@@ -635,9 +645,20 @@ GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_new_from_ctx, 3.7.0);
void
priv_glfs_free_from_ctx (struct glfs *fs)
{
+ upcall_entry *u_list = NULL;
+ upcall_entry *tmp = NULL;
+
if (!fs)
return;
+ /* cleanup upcall structures */
+ list_for_each_entry_safe (u_list, tmp,
+ &fs->upcall_list,
+ upcall_list) {
+ list_del_init (&u_list->upcall_list);
+ }
+ (void) pthread_mutex_destroy (&fs->upcall_list_mutex);
+
(void) pthread_cond_destroy (&fs->cond);
(void) pthread_cond_destroy (&fs->child_down_cond);
@@ -906,6 +927,7 @@ pub_glfs_fini (struct glfs *fs)
int fs_init = 0;
int err = -1;
+
if (!fs) {
errno = EINVAL;
return 0;
@@ -923,10 +945,24 @@ pub_glfs_fini (struct glfs *fs)
while (countdown--) {
/* give some time for background frames to finish */
- if (!call_pool->cnt)
- break;
+ pthread_mutex_lock (&fs->mutex);
+ {
+ /* Do we need to increase countdown? */
+ if ((!call_pool->cnt) && (!fs->pin_refcnt)) {
+ gf_log ("glfs", GF_LOG_ERROR,
+ "call_pool_cnt - %ld,"
+ "pin_refcnt - %d",
+ call_pool->cnt, fs->pin_refcnt);
+
+ ctx->cleanup_started = 1;
+ pthread_mutex_unlock (&fs->mutex);
+ break;
+ }
+ }
+ pthread_mutex_unlock (&fs->mutex);
usleep (100000);
}
+
/* leaked frames may exist, we ignore */
/*We deem glfs_fini as successful if there are no pending frames in the call
generated by cgit (git 2.34.1) at 2024-11-15 12:14:30 +0000