epoll: edge triggered and multi-threaded epoll

- edge triggered (oneshot) polling with epoll
- pick one event to avoid multiple events getting picked up by same
  thread
  and so get better distribution of events against multiple threads
- wire support for multiple poll threads to epoll_wait in parallel
- evdata to store absolute index and not hint for epoll
- store index and gen of slot instead of fd and index hint
- perform fd close asynchronously inside event.c for multithread safety
- poll is still single threaded

Change-Id: I536851dda0ab224c5d5a1b130a571397c9cace8f
BUG: 1104462
Signed-off-by: Anand Avati <avati@redhat.com>
Signed-off-by: Vijaikumar M <vmallika@redhat.com>
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Signed-off-by: Shyam <srangana@redhat.com>
Reviewed-on: http://review.gluster.org/3842
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 13 insertions, 7 deletions

diff --git a/libglusterfs/src/event.h b/libglusterfs/src/event.h
index 7ed182492e2..3b3ab0e4b2f 100644
--- a/libglusterfs/src/event.h
+++ b/libglusterfs/src/event.h
@@ -20,15 +20,20 @@
 
 struct event_pool;
 struct event_ops;
+struct event_slot_poll;
+struct event_slot_epoll;
 struct event_data {
-	int fd;
 	int idx;
+	int gen;
 } __attribute__ ((__packed__, __may_alias__));
 
 
 typedef int (*event_handler_t) (int fd, int idx, void *data,
 				int poll_in, int poll_out, int poll_err);
 
+#define EVENT_EPOLL_TABLES 1024
+#define EVENT_EPOLL_SLOTS 1024
+
 struct event_pool {
 	struct event_ops *ops;
 
@@ -36,12 +41,9 @@ struct event_pool {
 	int breaker[2];
 
 	int count;
-	struct {
-		int fd;
-		int events;
-		void *data;
-		event_handler_t handler;
-	} *reg;
+	struct event_slot_poll  *reg;
+	struct event_slot_epoll *ereg[EVENT_EPOLL_TABLES];
+	int slots_used[EVENT_EPOLL_TABLES];
 
 	int used;
 	int changed;
@@ -65,6 +67,9 @@ struct event_ops {
 
         int (*event_unregister) (struct event_pool *event_pool, int fd, int idx);
 
+        int (*event_unregister_close) (struct event_pool *event_pool, int fd,
+				       int idx);
+
         int (*event_dispatch) (struct event_pool *event_pool);
 };
 
@@ -75,6 +80,7 @@ int event_register (struct event_pool *event_pool, int fd,
 		    event_handler_t handler,
 		    void *data, int poll_in, int poll_out);
 int event_unregister (struct event_pool *event_pool, int fd, int idx);
+int event_unregister_close (struct event_pool *event_pool, int fd, int idx);
 int event_dispatch (struct event_pool *event_pool);
 
 #endif /* _EVENT_H_ */