epoll: edge triggered and multi-threaded epoll

- edge triggered (oneshot) polling with epoll
- pick one event to avoid multiple events getting picked up by same
  thread
  and so get better distribution of events against multiple threads
- wire support for multiple poll threads to epoll_wait in parallel
- evdata to store absolute index and not hint for epoll
- store index and gen of slot instead of fd and index hint
- perform fd close asynchronously inside event.c for multithread safety
- poll is still single threaded

Change-Id: I536851dda0ab224c5d5a1b130a571397c9cace8f
BUG: 1104462
Signed-off-by: Anand Avati <avati@redhat.com>
Signed-off-by: Vijaikumar M <vmallika@redhat.com>
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Signed-off-by: Shyam <srangana@redhat.com>
Reviewed-on: http://review.gluster.org/3842
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 36 insertions, 7 deletions

diff --git a/libglusterfs/src/event-poll.c b/libglusterfs/src/event-poll.c
index 7f7f560d086..a7e2e663103 100644
--- a/libglusterfs/src/event-poll.c
+++ b/libglusterfs/src/event-poll.c
@@ -26,6 +26,16 @@
 #include "config.h"
 #endif
 
+
+
+struct event_slot_poll {
+	int fd;
+	int events;
+	void *data;
+	event_handler_t handler;
+};
+
+
 static int
 event_register_poll (struct event_pool *event_pool, int fd,
                      event_handler_t handler,
@@ -63,12 +73,16 @@ __event_getindex (struct event_pool *event_pool, int fd, int idx)
 
         GF_VALIDATE_OR_GOTO ("event", event_pool, out);
 
+        /* lookup in used space based on index provided */
         if (idx > -1 && idx < event_pool->used) {
-                if (event_pool->reg[idx].fd == fd)
+                if (event_pool->reg[idx].fd == fd) {
                         ret = idx;
+                        goto out;
+                }
         }
 
-        for (i=0; ret == -1 && i<event_pool->used; i++) {
+        /* search in used space, if lookup fails */
+        for (i = 0; i < event_pool->used; i++) {
                 if (event_pool->reg[i].fd == fd) {
                         ret = i;
                         break;
@@ -264,6 +278,20 @@ out:
 
 
 static int
+event_unregister_close_poll (struct event_pool *event_pool, int fd,
+			     int idx_hint)
+{
+	int ret = -1;
+
+	ret = event_unregister_poll (event_pool, fd, idx_hint);
+
+	close (fd);
+
+        return ret;
+}
+
+
+static int
 event_select_on_poll (struct event_pool *event_pool, int fd, int idx_hint,
                       int poll_in, int poll_out)
 {
@@ -443,9 +471,10 @@ out:
 
 
 struct event_ops event_ops_poll = {
-        .new              = event_pool_new_poll,
-        .event_register   = event_register_poll,
-        .event_select_on  = event_select_on_poll,
-        .event_unregister = event_unregister_poll,
-        .event_dispatch   = event_dispatch_poll
+        .new                    = event_pool_new_poll,
+        .event_register         = event_register_poll,
+        .event_select_on        = event_select_on_poll,
+        .event_unregister       = event_unregister_poll,
+        .event_unregister_close = event_unregister_close_poll,
+        .event_dispatch         = event_dispatch_poll
 };