summaryrefslogtreecommitdiffstats
path: root/libglusterfs
diff options
context:
space:
mode:
authorRaghavendra G <rgowdapp@redhat.com>2017-05-05 15:21:30 +0530
committerRaghavendra G <rgowdapp@redhat.com>2017-05-12 05:26:42 +0000
commitcea8b702506ff914deadd056f4b7dd20a3ca7670 (patch)
tree954ca7e37696d57725d06343168bf7c6ed8bf22d /libglusterfs
parent333474e0d6efe1a2b3a9ecffc9bdff3e49325910 (diff)
event/epoll: Add back socket for polling of events immediately after
reading the entire rpc message from the wire Currently socket is added back for future events after higher layers (rpc, xlators etc) have processed the message. If message processing involves signficant delay (as in writev replies processed by Erasure Coding), performance takes hit. Hence this patch modifies transport/socket to add back the socket for polling of events immediately after reading the entire rpc message, but before notification to higher layers. credits: Thanks to "Kotresh Hiremath Ravishankar" <khiremat@redhat.com> for assitance in fixing a regression in bitrot caused by this patch. Change-Id: I04b6b9d0b51a1cfb86ecac3c3d87a5f388cf5800 BUG: 1448364 Signed-off-by: Raghavendra G <rgowdapp@redhat.com> Reviewed-on: https://review.gluster.org/15036 CentOS-regression: Gluster Build System <jenkins@build.gluster.org> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org> Smoke: Gluster Build System <jenkins@build.gluster.org> Reviewed-by: Amar Tumballi <amarts@redhat.com>
Diffstat (limited to 'libglusterfs')
-rw-r--r--libglusterfs/src/event-epoll.c81
-rw-r--r--libglusterfs/src/event-poll.c4
-rw-r--r--libglusterfs/src/event.c59
-rw-r--r--libglusterfs/src/event.h11
-rw-r--r--libglusterfs/src/glusterfs.h2
5 files changed, 109 insertions, 48 deletions
diff --git a/libglusterfs/src/event-epoll.c b/libglusterfs/src/event-epoll.c
index e2b40602e7a..4b76cc96fd3 100644
--- a/libglusterfs/src/event-epoll.c
+++ b/libglusterfs/src/event-epoll.c
@@ -569,38 +569,11 @@ pre_unlock:
if (!handler)
goto out;
- ret = handler (fd, idx, data,
+ ret = handler (fd, idx, gen, data,
(event->events & (EPOLLIN|EPOLLPRI)),
(event->events & (EPOLLOUT)),
(event->events & (EPOLLERR|EPOLLHUP)));
- LOCK (&slot->lock);
- {
- slot->in_handler--;
-
- if (gen != slot->gen) {
- /* event_unregister() happened while we were
- in handler()
- */
- gf_msg_debug ("epoll", 0, "generation bumped on idx=%d"
- " from gen=%d to slot->gen=%d, fd=%d, "
- "slot->fd=%d", idx, gen, slot->gen, fd,
- slot->fd);
- goto post_unlock;
- }
-
- /* This call also picks up the changes made by another
- thread calling event_select_on_epoll() while this
- thread was busy in handler()
- */
- if (slot->in_handler == 0) {
- event->events = slot->events;
- ret = epoll_ctl (event_pool->fd, EPOLL_CTL_MOD,
- fd, event);
- }
- }
-post_unlock:
- UNLOCK (&slot->lock);
out:
event_slot_unref (event_pool, slot, idx);
@@ -891,6 +864,55 @@ event_pool_destroy_epoll (struct event_pool *event_pool)
return ret;
}
+static int
+event_handled_epoll (struct event_pool *event_pool, int fd, int idx, int gen)
+{
+ struct event_slot_epoll *slot = NULL;
+ struct epoll_event epoll_event = {0, };
+ struct event_data *ev_data = (void *)&epoll_event.data;
+ int ret = 0;
+
+ slot = event_slot_get (event_pool, idx);
+
+ assert (slot->fd == fd);
+
+ LOCK (&slot->lock);
+ {
+ slot->in_handler--;
+
+ if (gen != slot->gen) {
+ /* event_unregister() happened while we were
+ in handler()
+ */
+ gf_msg_debug ("epoll", 0, "generation bumped on idx=%d"
+ " from gen=%d to slot->gen=%d, fd=%d, "
+ "slot->fd=%d", idx, gen, slot->gen, fd,
+ slot->fd);
+ goto post_unlock;
+ }
+
+ /* This call also picks up the changes made by another
+ thread calling event_select_on_epoll() while this
+ thread was busy in handler()
+ */
+ if (slot->in_handler == 0) {
+ epoll_event.events = slot->events;
+ ev_data->idx = idx;
+ ev_data->gen = gen;
+
+ ret = epoll_ctl (event_pool->fd, EPOLL_CTL_MOD,
+ fd, &epoll_event);
+ }
+ }
+post_unlock:
+ UNLOCK (&slot->lock);
+
+ event_slot_unref (event_pool, slot, idx);
+
+ return ret;
+}
+
+
struct event_ops event_ops_epoll = {
.new = event_pool_new_epoll,
.event_register = event_register_epoll,
@@ -899,7 +921,8 @@ struct event_ops event_ops_epoll = {
.event_unregister_close = event_unregister_close_epoll,
.event_dispatch = event_dispatch_epoll,
.event_reconfigure_threads = event_reconfigure_threads_epoll,
- .event_pool_destroy = event_pool_destroy_epoll
+ .event_pool_destroy = event_pool_destroy_epoll,
+ .event_handled = event_handled_epoll,
};
#endif
diff --git a/libglusterfs/src/event-poll.c b/libglusterfs/src/event-poll.c
index 2006e33d33b..3bffc4784d7 100644
--- a/libglusterfs/src/event-poll.c
+++ b/libglusterfs/src/event-poll.c
@@ -40,7 +40,7 @@ event_register_poll (struct event_pool *event_pool, int fd,
static int
-__flush_fd (int fd, int idx, void *data,
+__flush_fd (int fd, int idx, int gen, void *data,
int poll_in, int poll_out, int poll_err)
{
char buf[64];
@@ -386,7 +386,7 @@ unlock:
pthread_mutex_unlock (&event_pool->mutex);
if (handler)
- ret = handler (ufds[i].fd, idx, data,
+ ret = handler (ufds[i].fd, idx, 0, data,
(ufds[i].revents & (POLLIN|POLLPRI)),
(ufds[i].revents & (POLLOUT)),
(ufds[i].revents & (POLLERR|POLLHUP|POLLNVAL)));
diff --git a/libglusterfs/src/event.c b/libglusterfs/src/event.c
index 6aaa53499df..bba6f8429a1 100644
--- a/libglusterfs/src/event.c
+++ b/libglusterfs/src/event.c
@@ -159,8 +159,9 @@ event_pool_destroy (struct event_pool *event_pool)
}
pthread_mutex_unlock (&event_pool->mutex);
- if (!destroy || (activethreadcount > 0))
+ if (!destroy || (activethreadcount > 0)) {
goto out;
+ }
ret = event_pool->ops->event_pool_destroy (event_pool);
out:
@@ -168,19 +169,27 @@ out:
}
int
-poller_destroy_handler (int fd, int idx, void *data,
+poller_destroy_handler (int fd, int idx, int gen, void *data,
int poll_out, int poll_in, int poll_err)
{
- int readfd = -1;
- char buf = '\0';
+ struct event_destroy_data *destroy = NULL;
+ int readfd = -1, ret = -1;
+ char buf = '\0';
- readfd = *(int *)data;
- if (readfd < 0)
- return -1;
+ destroy = data;
+ readfd = destroy->readfd;
+ if (readfd < 0) {
+ goto out;
+ }
while (sys_read (readfd, &buf, 1) > 0) {
}
- return 0;
+
+ ret = 0;
+out:
+ event_handled (destroy->pool, fd, idx, gen);
+
+ return ret;
}
/* This function destroys all the poller threads.
@@ -197,11 +206,12 @@ poller_destroy_handler (int fd, int idx, void *data,
int
event_dispatch_destroy (struct event_pool *event_pool)
{
- int ret = -1;
- int fd[2] = {-1};
- int idx = -1;
- int flags = 0;
- struct timespec sleep_till = {0, };
+ int ret = -1, threadcount = 0;
+ int fd[2] = {-1};
+ int idx = -1;
+ int flags = 0;
+ struct timespec sleep_till = {0, };
+ struct event_destroy_data data = {0, };
GF_VALIDATE_OR_GOTO ("event", event_pool, out);
@@ -223,10 +233,13 @@ event_dispatch_destroy (struct event_pool *event_pool)
if (ret < 0)
goto out;
+ data.pool = event_pool;
+ data.readfd = fd[1];
+
/* From the main thread register an event on the pipe fd[0],
*/
idx = event_register (event_pool, fd[0], poller_destroy_handler,
- &fd[1], 1, 0);
+ &data, 1, 0);
if (idx < 0)
goto out;
@@ -235,6 +248,7 @@ event_dispatch_destroy (struct event_pool *event_pool)
*/
pthread_mutex_lock (&event_pool->mutex);
{
+ threadcount = event_pool->eventthreadcount;
event_pool->destroy = 1;
}
pthread_mutex_unlock (&event_pool->mutex);
@@ -254,9 +268,11 @@ event_dispatch_destroy (struct event_pool *event_pool)
*/
int retry = 0;
- while (event_pool->activethreadcount > 0 && retry++ < 10) {
- if (sys_write (fd[1], "dummy", 6) == -1)
+ while (event_pool->activethreadcount > 0
+ && (retry++ < (threadcount + 10))) {
+ if (sys_write (fd[1], "dummy", 6) == -1) {
break;
+ }
sleep_till.tv_sec = time (NULL) + 1;
ret = pthread_cond_timedwait (&event_pool->cond,
&event_pool->mutex,
@@ -275,3 +291,14 @@ event_dispatch_destroy (struct event_pool *event_pool)
return ret;
}
+
+int
+event_handled (struct event_pool *event_pool, int fd, int idx, int gen)
+{
+ int ret = 0;
+
+ if (event_pool->ops->event_handled)
+ ret = event_pool->ops->event_handled (event_pool, fd, idx, gen);
+
+ return ret;
+}
diff --git a/libglusterfs/src/event.h b/libglusterfs/src/event.h
index 1348f5d05c0..c60b14ad04b 100644
--- a/libglusterfs/src/event.h
+++ b/libglusterfs/src/event.h
@@ -23,7 +23,7 @@ struct event_data {
} __attribute__ ((__packed__, __may_alias__));
-typedef int (*event_handler_t) (int fd, int idx, void *data,
+typedef int (*event_handler_t) (int fd, int idx, int gen, void *data,
int poll_in, int poll_out, int poll_err);
#define EVENT_EPOLL_TABLES 1024
@@ -73,6 +73,11 @@ struct event_pool {
};
+struct event_destroy_data {
+ int readfd;
+ struct event_pool *pool;
+};
+
struct event_ops {
struct event_pool * (*new) (int count, int eventthreadcount);
@@ -93,6 +98,8 @@ struct event_ops {
int (*event_reconfigure_threads) (struct event_pool *event_pool,
int newcount);
int (*event_pool_destroy) (struct event_pool *event_pool);
+ int (*event_handled) (struct event_pool *event_pool, int fd, int idx,
+ int gen);
};
struct event_pool *event_pool_new (int count, int eventthreadcount);
@@ -107,4 +114,6 @@ int event_dispatch (struct event_pool *event_pool);
int event_reconfigure_threads (struct event_pool *event_pool, int value);
int event_pool_destroy (struct event_pool *event_pool);
int event_dispatch_destroy (struct event_pool *event_pool);
+int event_handled (struct event_pool *event_pool, int fd, int idx, int gen);
+
#endif /* _EVENT_H_ */
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 839a1d47d2a..2e709b9d703 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -520,6 +520,8 @@ struct _glusterfs_ctx {
int notifying;
struct gf_ctx_tw *tw; /* refcounted timer_wheel */
+
+ gf_lock_t volfile_lock;
};
typedef struct _glusterfs_ctx glusterfs_ctx_t;