server-protocol - reimplement connection cleanup to happen in 2 phases

- first phase, which happens when POLLERR is received on transport,
     releases all locks, flushes all open fds.
   - second phase, which happens when both the transports of connection destroyed,
     destroys the containers like lock table, fd table along with the connection.
   - the first phase, clears up any references to transport held by translators
     like posix-locks(in the form of blocked locks) paving way for the second phase.

Signed-off-by: Anand V. Avati <avati@amp.gluster.com>

6 files changed, 261 insertions, 32 deletions

diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index 918b6c14dcc..deb3c800145 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -129,26 +129,68 @@ gf_fd_fdtable_alloc (void)
 	return fdtable;
 }
 
+fd_t **
+__gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count)
+{
+        fd_t **fds = NULL;
+
+        if (count == NULL) {
+                goto out;
+        }
+
+        fds = fdtable->fds;
+        fdtable->fds = calloc (fdtable->max_fds, sizeof (fd_t *));
+        *count = fdtable->max_fds;
+
+out:
+        return fds;
+}
+
+fd_t **
+gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count)
+{
+        fd_t **fds = NULL;
+        if (fdtable) {
+                pthread_mutex_lock (&fdtable->lock);
+                {
+                        fds = __gf_fd_fdtable_get_all_fds (fdtable, count);
+                }
+                pthread_mutex_unlock (&fdtable->lock);
+        }
+
+        return fds;
+}
+
 void 
 gf_fd_fdtable_destroy (fdtable_t *fdtable)
 {
-				
-	int32_t i = 0;
+        struct list_head  list = {0, };
+        fd_t             *fd = NULL;
+        fd_t            **fds = NULL;
+        uint32_t          fd_count = 0;
+        int32_t           i = 0; 
+
+        INIT_LIST_HEAD (&list);
 
 	if (fdtable) {
 		pthread_mutex_lock (&fdtable->lock);
 		{
-			for (i=0; i < fdtable->max_fds; i++) {
-				if (fdtable->fds[i]) {
-					fd_t *fd = fdtable->fds[i];
-						  
-					fd_unref (fd);
-				}
-			}
-
+                        fds = __gf_fd_fdtable_get_all_fds (fdtable, &fd_count);
 			FREE (fdtable->fds);
 		}
 		pthread_mutex_unlock (&fdtable->lock);
+
+                if (fds != NULL) {
+                        for (i = 0; i < fd_count; i++) {
+                                fd = fds[i];
+                                if (fd != NULL) {
+                                        fd_unref (fd);
+                                }
+                        }
+
+                        FREE (fds);
+                }
+
 		pthread_mutex_destroy (&fdtable->lock);
 		FREE (fdtable);
 	}
diff --git a/libglusterfs/src/fd.h b/libglusterfs/src/fd.h
index 0382ccdc218..2916a44812e 100644
--- a/libglusterfs/src/fd.h
+++ b/libglusterfs/src/fd.h
@@ -77,6 +77,9 @@ gf_fd_unused_get (fdtable_t *fdtable, fd_t *fdptr);
 int32_t 
 gf_fd_unused_get2 (fdtable_t *fdtable, fd_t *fdptr, int32_t fd);
 
+fd_t **
+gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count);
+
 void 
 gf_fd_fdtable_destroy (fdtable_t *fdtable);
 
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 1fa400d5fab..9a6932f67f8 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -398,19 +398,192 @@ out:
 }
 
 
+static int32_t
+server_connection_cleanup_flush_cbk (call_frame_t *frame,
+                                     void *cookie,
+                                     xlator_t *this,
+                                     int32_t op_ret,
+                                     int32_t op_errno)
+{
+        fd_t *fd = NULL;
+        fd = frame->local;
+
+        fd_unref (fd);
+        frame->local = NULL;
+
+        STACK_DESTROY (frame->root);
+        return 0;
+}
+
+
 int
-server_connection_destroy (xlator_t *this, server_connection_t *conn)
+server_connection_cleanup (xlator_t *this, server_connection_t *conn)
 {
+	call_frame_t       *frame = NULL, *tmp_frame = NULL;
+	xlator_t           *bound_xl = NULL;
+	server_state_t     *state = NULL;
+	struct list_head    file_lockers;
+	struct list_head    dir_lockers;
+	struct _lock_table *ltable = NULL;
+	struct _locker     *locker = NULL, *tmp = NULL;
+	struct flock        flock = {0,};
+        fd_t               *fd = NULL;
+        uint32_t            fd_count = 0;
+        fd_t              **fds = NULL;
+        int32_t             i = 0;
+ 
+	bound_xl = (xlator_t *) (conn->bound_xl);
+
+	if (bound_xl) {
+		/* trans will have ref_count = 1 after this call, but its 
+		   ok since this function is called in 
+		   GF_EVENT_TRANSPORT_CLEANUP */
+		frame = create_frame (this, this->ctx->pool);
 
-	call_frame_t      *frame = NULL, *tmp_frame = NULL;
-	xlator_t          *bound_xl = NULL;
-	int32_t            ret = -1;
-	server_state_t    *state = NULL;
-	struct list_head   file_lockers;
-	struct list_head   dir_lockers;
+		pthread_mutex_lock (&(conn->lock));
+		{
+			if (conn->ltable) {
+				ltable = conn->ltable;
+				conn->ltable = gf_lock_table_new ();
+			}
+		}
+		pthread_mutex_unlock (&conn->lock);
+
+		INIT_LIST_HEAD (&file_lockers);
+		INIT_LIST_HEAD (&dir_lockers);
+
+		LOCK (&ltable->lock);
+		{
+			list_splice_init (&ltable->file_lockers, 
+					  &file_lockers);
+
+			list_splice_init (&ltable->dir_lockers, &dir_lockers);
+		}
+		UNLOCK (&ltable->lock);
+		free (ltable);
+
+		flock.l_type  = F_UNLCK;
+		flock.l_start = 0;
+		flock.l_len   = 0;
+		list_for_each_entry_safe (locker, 
+					  tmp, &file_lockers, lockers) {
+			tmp_frame = copy_frame (frame);
+			/* 
+			   pid = 0 is a special case that tells posix-locks
+			   to release all locks from this transport
+			*/
+			tmp_frame->root->pid = 0;
+			tmp_frame->root->trans = conn;
+
+			if (locker->fd) {
+				STACK_WIND (tmp_frame, server_nop_cbk,
+					    bound_xl,
+					    bound_xl->fops->finodelk,
+                                            locker->volume,
+					    locker->fd, F_SETLK, &flock);
+				fd_unref (locker->fd);
+			} else {
+				STACK_WIND (tmp_frame, server_nop_cbk,
+					    bound_xl,
+					    bound_xl->fops->inodelk,
+                                            locker->volume,
+					    &(locker->loc), F_SETLK, &flock);
+				loc_wipe (&locker->loc);
+			}
+
+                        free (locker->volume);
+
+			list_del_init (&locker->lockers);
+			free (locker);
+		}
+
+		tmp = NULL;
+		locker = NULL;
+		list_for_each_entry_safe (locker, tmp, &dir_lockers, lockers) {
+			tmp_frame = copy_frame (frame);
+
+			tmp_frame->root->pid = 0;
+			tmp_frame->root->trans = conn;
+
+			if (locker->fd) {
+				STACK_WIND (tmp_frame, server_nop_cbk,
+					    bound_xl,
+					    bound_xl->fops->fentrylk,
+                                            locker->volume,
+					    locker->fd, NULL, 
+					    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
+				fd_unref (locker->fd);
+			} else {
+				STACK_WIND (tmp_frame, server_nop_cbk,
+					    bound_xl,
+					    bound_xl->fops->entrylk,
+                                            locker->volume,
+					    &(locker->loc), NULL, 
+					    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
+				loc_wipe (&locker->loc);
+			}
+
+                        free (locker->volume);
+
+			list_del_init (&locker->lockers);
+			free (locker);
+		}
+
+                pthread_mutex_lock (&conn->lock);
+                {
+                        if (conn->fdtable) {
+                                fds = gf_fd_fdtable_get_all_fds (conn->fdtable,
+                                                                 &fd_count);
+                        }
+                }
+                pthread_mutex_unlock (&conn->lock);
+
+                if (fds != NULL) {
+                        for (i = 0;i < fd_count; i++) {
+                                fd = fds[i];
+
+                                if (fd != NULL) {
+                                        tmp_frame = copy_frame (frame);
+                                        tmp_frame->local = fd;
+                                        
+                                        tmp_frame->root->pid = 0;
+                                        tmp_frame->root->trans = conn;
+                                        STACK_WIND (tmp_frame,
+                                                    server_connection_cleanup_flush_cbk,
+                                                    bound_xl,
+                                                    bound_xl->fops->flush,
+                                                    fd);
+                                }
+                        }
+                        FREE (fds);
+                }
+
+		state = CALL_STATE (frame);
+		if (state)
+			free (state);
+		STACK_DESTROY (frame->root);
+        }
+
+        return 0;
+}
+
+
+int
+server_connection_destroy (xlator_t *this, server_connection_t *conn)
+{
+	call_frame_t       *frame = NULL, *tmp_frame = NULL;
+	xlator_t           *bound_xl = NULL;
+	int32_t             ret = -1;
+	server_state_t     *state = NULL;
+	struct list_head    file_lockers;
+	struct list_head    dir_lockers;
 	struct _lock_table *ltable = NULL;
 	struct _locker     *locker = NULL, *tmp = NULL;
 	struct flock        flock = {0,};
+        fd_t               *fd = NULL; 
+        int32_t             i = 0;
+        fd_t              **fds = NULL;
+        uint32_t             fd_count = 0;
 
 
 	bound_xl = (xlator_t *) (conn->bound_xl);
@@ -518,12 +691,30 @@ server_connection_destroy (xlator_t *this, server_connection_t *conn)
 		pthread_mutex_lock (&(conn->lock));
 		{
 			if (conn->fdtable) {
+                                fds = gf_fd_fdtable_get_all_fds (conn->fdtable,
+                                                                 &fd_count);
 				gf_fd_fdtable_destroy (conn->fdtable);
 				conn->fdtable = NULL;
 			}
 		}
 		pthread_mutex_unlock (&conn->lock);
 
+                if (fds != NULL) {
+                        for (i = 0; i < fd_count; i++) {
+                                fd = fds[i];
+                                if (fd != NULL) {
+                                        tmp_frame = copy_frame (frame);
+                                        tmp_frame->local = fd;
+
+                                        STACK_WIND (tmp_frame,
+                                                    server_connection_cleanup_flush_cbk,
+                                                    bound_xl,
+                                                    bound_xl->fops->flush,
+                                                    fd);
+                                }
+                        }
+                        FREE (fds);
+                }
 	}
 
 	gf_log (this->name, GF_LOG_INFO, "destroyed connection of %s",
diff --git a/xlators/protocol/server/src/server-helpers.h b/xlators/protocol/server/src/server-helpers.h
index eb26de84f86..f26091ea2ce 100644
--- a/xlators/protocol/server/src/server-helpers.h
+++ b/xlators/protocol/server/src/server-helpers.h
@@ -74,4 +74,5 @@ gf_del_locker (struct _lock_table *table, const char *volume,
 int32_t
 gf_direntry_to_bin (dir_entry_t *head,
 		    char **bufferp);
+
 #endif /* __SERVER_HELPERS_H__ */
diff --git a/xlators/protocol/server/src/server-protocol.c b/xlators/protocol/server/src/server-protocol.c
index 1153a545100..5eee27b9239 100644
--- a/xlators/protocol/server/src/server-protocol.c
+++ b/xlators/protocol/server/src/server-protocol.c
@@ -4286,27 +4286,15 @@ server_release (call_frame_t *frame, xlator_t *bound_xl,
 	state = CALL_STATE(frame);
 	
 	state->fd_no = ntoh64 (req->fd);
-	state->fd = gf_fd_fdptr_get (conn->fdtable, 
-				     state->fd_no);
-	
-	GF_VALIDATE_OR_GOTO(bound_xl->name, state->fd, fail);
 
 	gf_fd_put (conn->fdtable, 
 		   state->fd_no);
 
 	gf_log (bound_xl->name, GF_LOG_DEBUG,
-		"%"PRId64": RELEASE \'fd=%"PRId64" (%"PRId64")\'", 
-		frame->root->unique, state->fd_no, state->fd->inode->ino);
+		"%"PRId64": RELEASE \'fd=%"PRId64"\'", 
+		frame->root->unique, state->fd_no);
 
-	STACK_WIND (frame,
-		    server_release_cbk,
-		    BOUND_XL(frame),
-		    BOUND_XL(frame)->fops->flush,
-		    state->fd);
-	return 0;
-fail:
-	server_release_cbk (frame, NULL, frame->this,
-			    -1, EINVAL);
+	server_release_cbk (frame, NULL, frame->this, 0, 0);
 	return 0;
 }
 
@@ -8269,6 +8257,7 @@ notify (xlator_t *this, int32_t event, void *data, ...)
 
 		ret = -1;
 		transport_disconnect (trans);
+                server_connection_cleanup (this, trans->xl_private);
 	}
 	break;
 
diff --git a/xlators/protocol/server/src/server-protocol.h b/xlators/protocol/server/src/server-protocol.h
index f00b584fb25..2e6923fb4e2 100644
--- a/xlators/protocol/server/src/server-protocol.h
+++ b/xlators/protocol/server/src/server-protocol.h
@@ -83,6 +83,9 @@ int
 server_connection_destroy (xlator_t *this, server_connection_t *conn);
 
 int
+server_connection_cleanup (xlator_t *this, server_connection_t *conn);
+
+int
 server_nop_cbk (call_frame_t *frame, void *cookie,
 		xlator_t *this, int32_t op_ret, int32_t op_errno);