summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSoumya Koduri <skoduri@redhat.com>2018-11-18 23:38:08 +0530
committerShyamsundar Ranganathan <srangana@redhat.com>2018-12-12 12:53:23 +0000
commitb710f812101233c0b25edcb0f59d197bcd9d0026 (patch)
tree66d6ab86a459ef8eac97ffda9eedd6c3512fe4f4
parentf17188e4aa5e02d266cf147cf418e6cc27f5db21 (diff)
gfapi: Offload callback notifications to synctask
Upcall notifications are received from server via epoll and same thread is used to forward these notifications to the application. This may lead to deadlock and hang in the following scenario. Consider if as part of handling these callbacks, application has to do some operations which involve sending I/Os to gfapi stack which inturn have to wait for epoll threads to receive repsonse. Thus this may lead to deadlock if all the epoll threads are waiting to complete these callback notifications. To address it, instead of using epoll thread itself, make use of synctask to send those notificaitons to the application. Change-Id: If614e0d09246e4279b9d1f40d883a32a39c8fd90 updates: bz#1651323 Signed-off-by: Soumya Koduri <skoduri@redhat.com> (cherry picked from commit ad35193718a99494ab1b852ca4cbdf054f73de88)
-rw-r--r--api/src/gfapi-messages.h2
-rw-r--r--api/src/glfs-fops.c53
-rw-r--r--tests/features/glfs-lease-recall.c372
-rwxr-xr-xtests/features/glfs-lease.t2
4 files changed, 419 insertions, 10 deletions
diff --git a/api/src/gfapi-messages.h b/api/src/gfapi-messages.h
index 20418627690..03269a1c9d8 100644
--- a/api/src/gfapi-messages.h
+++ b/api/src/gfapi-messages.h
@@ -49,6 +49,6 @@ GLFS_MSGID(API, API_MSG_MEM_ACCT_INIT_FAILED, API_MSG_MASTER_XLATOR_INIT_FAILED,
API_MSG_INODE_LINK_FAILED, API_MSG_STATEDUMP_FAILED,
API_MSG_XREADDIRP_R_FAILED, API_MSG_LOCK_INSERT_MERGE_FAILED,
API_MSG_SETTING_LOCK_TYPE_FAILED, API_MSG_INODE_FIND_FAILED,
- API_MSG_FDCTX_SET_FAILED);
+ API_MSG_FDCTX_SET_FAILED, API_MSG_UPCALL_SYNCOP_FAILED);
#endif /* !_GFAPI_MESSAGES_H__ */
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index 104d3ffd4b1..0bb779a7be1 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -31,6 +31,11 @@
#define GF_NAME_MAX 255
#endif
+struct upcall_syncop_args {
+ struct glfs *fs;
+ struct gf_upcall *upcall_data;
+};
+
#define READDIRBUF_SIZE (sizeof(struct dirent) + GF_NAME_MAX + 1)
typedef void (*glfs_io_cbk34)(glfs_fd_t *fd, ssize_t ret, void *data);
@@ -4863,19 +4868,17 @@ out:
return ret;
}
-static void
-glfs_cbk_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
+static int
+glfs_cbk_upcall_syncop(void *opaque)
{
+ struct upcall_syncop_args *args = opaque;
int ret = -1;
struct glfs_upcall *up_arg = NULL;
+ struct glfs *fs;
+ struct gf_upcall *upcall_data;
- if (!fs || !upcall_data)
- goto out;
-
- if (!(fs->upcall_events & upcall_data->event_type)) {
- /* ignore events which application hasn't registered*/
- goto out;
- }
+ fs = args->fs;
+ upcall_data = args->upcall_data;
up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall), glfs_release_upcall,
glfs_mt_upcall_entry_t);
@@ -4922,6 +4925,38 @@ out:
GLFS_FREE(up_arg);
}
+ return ret;
+}
+
+static void
+glfs_cbk_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+ struct upcall_syncop_args args = {
+ 0,
+ };
+ int ret = -1;
+
+ if (!fs || !upcall_data)
+ goto out;
+
+ if (!(fs->upcall_events & upcall_data->event_type)) {
+ /* ignore events which application hasn't registered*/
+ goto out;
+ }
+
+ args.fs = fs;
+ args.upcall_data = upcall_data;
+
+ ret = synctask_new(THIS->ctx->env, glfs_cbk_upcall_syncop, NULL, NULL,
+ &args);
+ /* should we retry incase of failure? */
+ if (ret) {
+ gf_msg(THIS->name, GF_LOG_ERROR, errno, API_MSG_UPCALL_SYNCOP_FAILED,
+ "Synctak for Upcall event_type(%d) and gfid(%s) failed",
+ upcall_data->event_type, (char *)(upcall_data->gfid));
+ }
+
+out:
return;
}
diff --git a/tests/features/glfs-lease-recall.c b/tests/features/glfs-lease-recall.c
new file mode 100644
index 00000000000..9a60f9beec1
--- /dev/null
+++ b/tests/features/glfs-lease-recall.c
@@ -0,0 +1,372 @@
+#include <glusterfs/api/glfs.h>
+#include <glusterfs/api/glfs-handles.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+
+/* Few rules:
+ * 1. A client may have multiple lease keys, but a lease key cannot be shared by
+ * multiple clients.
+ * 2. Lease key can be set before open, or in glfs_lease request. A lease key
+ * set like this is valid for the lifetime of the fd, i.e. a fd cannot have
+ * multiple lease key. But a lease key can be shared across multiple fds.
+ */
+glfs_t *client1 = NULL, *client2 = NULL;
+glfs_fd_t *fd1 = NULL;
+FILE *log_file = NULL;
+char lid1[GLFS_LEASE_ID_SIZE] = "lid1-clnt1",
+ lid2[GLFS_LEASE_ID_SIZE] = "lid2-clnt2";
+char lid3[GLFS_LEASE_ID_SIZE] = "lid3-clnt2", lid4[GLFS_LEASE_ID_SIZE] = {
+ 0,
+};
+char *volname = NULL, *glfs_log_file = NULL;
+int upcall_recv = 0;
+
+#define MAX_CLIENTS 4
+#define MAX_FDS 4
+#define TEST_FILE "/test/lease"
+#define SHUD_PASS 0
+#define SHUD_FAIL -1
+#define NONE 0
+
+static void
+recall_cbk(struct glfs_lease lease, void *data);
+
+static int
+set_read_lease(glfs_fd_t *fd, char ld[])
+{
+ struct glfs_lease lease = {
+ 0,
+ };
+ int ret = 0;
+
+ memset(&lease, 0, sizeof(lease));
+ lease.cmd = GLFS_SET_LEASE;
+ lease.lease_type = GLFS_RD_LEASE;
+ memcpy(&lease.lease_id, ld, GLFS_LEASE_ID_SIZE);
+ ret = glfs_lease(fd, &lease, &recall_cbk, fd);
+ if (ret < 0) {
+ fprintf(log_file, "\n RD_LEASE failed with ret: %d (%s)", ret,
+ strerror(errno));
+ return -1;
+ }
+ fprintf(log_file, "\n Took RD_LEASE");
+ return ret;
+}
+
+static int
+set_write_lease(glfs_fd_t *fd, char ld[])
+{
+ struct glfs_lease lease = {
+ 0,
+ };
+ int ret = 0;
+
+ memset(&lease, 0, sizeof(lease));
+ lease.cmd = GLFS_SET_LEASE;
+ lease.lease_type = GLFS_RW_LEASE;
+ memcpy(&lease.lease_id, ld, GLFS_LEASE_ID_SIZE);
+ ret = glfs_lease(fd, &lease, &recall_cbk, NULL);
+ if (ret < 0) {
+ fprintf(log_file, "\n RW_LEASE failed with ret: %d (%s)", ret,
+ strerror(errno));
+ return -1;
+ }
+ fprintf(log_file, "\n Took RW_LEASE");
+ return ret;
+}
+
+static int
+get_lease(glfs_fd_t *fd, char ld[])
+{
+ struct glfs_lease lease = {
+ 0,
+ };
+ int ret = 0;
+
+ memset(&lease, 0, sizeof(lease));
+ lease.cmd = GLFS_GET_LEASE;
+ lease.lease_type = -1;
+ memcpy(&lease.lease_id, ld, GLFS_LEASE_ID_SIZE);
+ ret = glfs_lease(fd, &lease, &recall_cbk, NULL);
+ if (ret < 0) {
+ fprintf(log_file, "\n GET_LEASE failed with ret: %d (%s)", ret,
+ strerror(errno));
+ return -1;
+ }
+ if (lease.lease_type == GLFS_RD_LEASE)
+ fprintf(log_file, "\n Esisting Lease: RD_LEASE");
+ else if (lease.lease_type == GLFS_RW_LEASE)
+ fprintf(log_file, "\n Esisting Lease: RW_LEASE");
+ else if (lease.lease_type == 3)
+ fprintf(log_file, "\n Esisting Lease: RD_LEASE|RW_LEASE");
+ else if (lease.lease_type == 0)
+ fprintf(log_file, "\n Esisting Lease: NONE");
+ else
+ fprintf(log_file, "\n Existing lease type:%d", lease.lease_type);
+ return lease.lease_type;
+}
+
+static int
+unlk_write_lease(glfs_fd_t *fd, char ld[])
+{
+ struct glfs_lease lease = {
+ 0,
+ };
+ int ret = 0;
+
+ memset(&lease, 0, sizeof(lease));
+ lease.cmd = GLFS_UNLK_LEASE;
+ lease.lease_type = GLFS_RW_LEASE;
+ memcpy(&lease.lease_id, ld, GLFS_LEASE_ID_SIZE);
+ ret = glfs_lease(fd, &lease, &recall_cbk, NULL);
+ if (ret < 0) {
+ fprintf(log_file, "\n Unlock RW_LESAE failed with ret: %d (%s)", ret,
+ strerror(errno));
+ return -1;
+ }
+ fprintf(log_file, "\n Unlocked RW_LEASE");
+ return ret;
+}
+
+static int
+unlk_read_lease(glfs_fd_t *fd, char ld[])
+{
+ struct glfs_lease lease = {
+ 0,
+ };
+ int ret = 0;
+
+ memset(&lease, 0, sizeof(lease));
+ lease.cmd = GLFS_UNLK_LEASE;
+ lease.lease_type = GLFS_RD_LEASE;
+ memcpy(&lease.lease_id, ld, GLFS_LEASE_ID_SIZE);
+
+ ret = glfs_lease(fd, &lease, &recall_cbk, NULL);
+ if (ret < 0) {
+ fprintf(log_file, "\n Unlock RD_LEASE failed with ret: %d (%s)", ret,
+ strerror(errno));
+ return -1;
+ }
+ fprintf(log_file, "\n Unlocked RD_LEASE");
+ return ret;
+}
+
+void
+up_async_lease_recall(struct glfs_upcall *up_arg, void *data)
+{
+ struct glfs_upcall_lease *in_arg = NULL;
+ enum glfs_upcall_reason reason = 0;
+ struct glfs_object *object = NULL;
+ uint64_t flags = 0;
+ uint64_t expire = 0;
+
+ if (!up_arg)
+ return;
+
+ reason = glfs_upcall_get_reason(up_arg);
+
+ /* Expect 'GLFS_UPCALL_RECALL_LEASE' upcall event. */
+
+ if (reason == GLFS_UPCALL_RECALL_LEASE) {
+ in_arg = glfs_upcall_get_event(up_arg);
+
+ object = glfs_upcall_lease_get_object(in_arg);
+
+ fprintf(log_file,
+ " upcall event type - %d,"
+ " object(%p)\n",
+ reason, object);
+ upcall_recv = 1;
+ }
+
+ glfs_free(up_arg);
+ return;
+}
+
+glfs_t *
+setup_new_client(char *volname, char *log_fileile)
+{
+ int ret = 0;
+ glfs_t *fs = NULL;
+ int up_events = GLFS_EVENT_ANY;
+
+ fs = glfs_new(volname);
+ if (!fs) {
+ fprintf(log_file, "\nglfs_new: returned NULL (%s)\n", strerror(errno));
+ goto error;
+ }
+
+ ret = glfs_set_volfile_server(fs, "tcp", "localhost", 24007);
+ if (ret < 0) {
+ fprintf(log_file, "\nglfs_set_volfile_server failed ret:%d (%s)\n", ret,
+ strerror(errno));
+ goto error;
+ }
+
+ ret = glfs_set_logging(fs, log_fileile, 7);
+ if (ret < 0) {
+ fprintf(log_file, "\nglfs_set_logging failed with ret: %d (%s)\n", ret,
+ strerror(errno));
+ goto error;
+ }
+
+ ret = glfs_init(fs);
+ if (ret < 0) {
+ fprintf(log_file, "\nglfs_init failed with ret: %d (%s)\n", ret,
+ strerror(errno));
+ goto error;
+ }
+
+ /* Register Upcalls */
+ ret = glfs_upcall_register(fs, up_events, up_async_lease_recall, NULL);
+
+ /* Check if the return mask contains the event */
+ if ((ret < 0) || !(ret & GLFS_EVENT_RECALL_LEASE)) {
+ fprintf(stderr,
+ "glfs_upcall_register return doesn't contain"
+ " upcall event - GLFS_EVENT_RECALL_LEASE\n");
+ goto error;
+ }
+
+ return fs;
+error:
+ if (fs)
+ glfs_fini(fs);
+ return NULL;
+}
+
+#define OPEN(client, flags, fd, lease_id) \
+ do { \
+ int ret_val = 0; \
+ ret_val = glfs_setfsleaseid(lease_id); \
+ if (ret_val) { \
+ fprintf(log_file, \
+ "\nglfs_setfsleaseid failed with ret: %d (%s)\n", ret, \
+ strerror(errno)); \
+ return -1; \
+ } \
+ fd = glfs_open(client, TEST_FILE, flags); \
+ if (fd == NULL) { \
+ fprintf(log_file, "\nglfs_open failed with ret: %d (%s)\n", ret, \
+ strerror(errno)); \
+ return -1; \
+ } \
+ } while (0)
+
+#define VERIFY_RESULT(test_case, ret, value) \
+ do { \
+ if (ret != value) { \
+ fprintf(log_file, \
+ "\n Testcase %d failed, ret = %d, value=%d\n", \
+ test_case, ret, value); \
+ goto error; /*test unsuccessful*/ \
+ } \
+ fprintf(log_file, "\n Testcase %d Succeeded\n", test_case); \
+ } while (0)
+
+static void
+recall_cbk(struct glfs_lease lease, void *data)
+{
+ int ret = -1;
+ char ld[GLFS_LEASE_ID_SIZE] = "";
+
+ fprintf(log_file, "\nRECALL received on lease_id:(%s)", lease.lease_id);
+ memcpy(ld, lease.lease_id, GLFS_LEASE_ID_SIZE);
+ ret = unlk_write_lease((glfs_fd_t *)data, ld);
+ VERIFY_RESULT(500, ret, SHUD_PASS);
+error:
+ return;
+}
+
+static int
+testcase_recall_conflict_lease()
+{
+ struct glfs_object *obj = NULL;
+ glfs_fd_t *fd1 = NULL;
+ int ret = 0;
+ struct glfs_lease lease = {
+ 0,
+ };
+
+ fprintf(log_file,
+ "\n Basic test case for conflicting lease causing recall");
+
+ memset(&lease, 0, sizeof(lease));
+ lease.cmd = GLFS_SET_LEASE;
+ lease.lease_type = GLFS_RD_LEASE;
+ memcpy(&lease.lease_id, lid2, GLFS_LEASE_ID_SIZE);
+ /* Open fd on client 1 in RD mode */
+ OPEN(client1, O_RDWR, fd1, lid1);
+ ret = set_write_lease(fd1, lid1);
+ VERIFY_RESULT(1, ret, SHUD_PASS);
+
+ /* reset counter */
+ upcall_recv = 0;
+
+ obj = glfs_h_lookupat(client2, NULL, TEST_FILE, NULL, 0);
+ ret = glfs_h_lease(client2, obj, &lease);
+ VERIFY_RESULT(2, ret, SHUD_FAIL);
+
+ sleep(3);
+ /* should recv upcall */
+ VERIFY_RESULT(6, !upcall_recv, SHUD_PASS);
+
+ ret = unlk_write_lease(fd1, lid1);
+ VERIFY_RESULT(5, ret, SHUD_PASS);
+
+ ret = glfs_h_close(obj);
+ VERIFY_RESULT(3, ret, SHUD_PASS);
+ ret = glfs_close(fd1);
+ VERIFY_RESULT(4, ret, SHUD_PASS);
+
+ return 0;
+error:
+ return -1;
+}
+
+int
+main(int argc, char *argv[])
+{
+ int ret = 0;
+ int i = 0;
+ glfs_fd_t *fd = NULL;
+ glfs_fd_t *fd1 = NULL;
+ char *topdir = "topdir", *filename = "file1";
+ char *buf = NULL;
+ int x = 0;
+ ssize_t xattr_size = -1;
+
+ if (argc != 4) {
+ fprintf(stderr,
+ "Expect following args %s <Vol> <glfs client log file> "
+ "<testcase log file>\n",
+ argv[0]);
+ return -1;
+ }
+
+ log_file = fopen(argv[3], "w");
+ if (!log_file)
+ goto error;
+
+ volname = argv[1];
+ glfs_log_file = argv[2];
+
+ /* Setup 2 clients */
+ client1 = setup_new_client(volname, glfs_log_file);
+ client2 = setup_new_client(volname, glfs_log_file);
+
+ ret = testcase_recall_conflict_lease();
+ VERIFY_RESULT(101, ret, SHUD_PASS);
+
+ glfs_fini(client1);
+ glfs_fini(client2);
+
+ fclose(log_file);
+ return 0;
+error:
+ return -1;
+}
diff --git a/tests/features/glfs-lease.t b/tests/features/glfs-lease.t
index 27c4661aef9..6ef6da05043 100755
--- a/tests/features/glfs-lease.t
+++ b/tests/features/glfs-lease.t
@@ -21,7 +21,9 @@ TEST mkdir $M0/test
TEST touch $M0/test/lease
build_tester $(dirname $0)/glfs-lease.c -lgfapi
+build_tester $(dirname $0)/glfs-lease-recall.c -lgfapi
TEST $(dirname $0)/glfs-lease $V0 $logdir/glfs-lease.log $logdir/lease-test.log
+TEST $(dirname $0)/glfs-lease-recall $V0 $logdir/glfs-lease-recall.log $logdir/lease-test-recall.log
TEST $CLI volume set $V0 leases off