per frame time out - i.e individual frames are unwound instead of the transport point itself being disconnected. timeout is configured using "transport-timeout".

Signed-off-by: Anand V. Avati <avati@amp.gluster.com>

3 files changed, 115 insertions, 41 deletions

diff --git a/xlators/protocol/client/src/client-protocol.c b/xlators/protocol/client/src/client-protocol.c
index 5c93bd6f135..569295400f6 100644
--- a/xlators/protocol/client/src/client-protocol.c
+++ b/xlators/protocol/client/src/client-protocol.c
@@ -43,7 +43,7 @@
 /* for default_*_cbk functions */
 #include "defaults.c"
 #include "saved-frames.h"
-
+#include "common-utils.h"
 
 int protocol_client_cleanup (transport_t *trans);
 int protocol_client_interpret (xlator_t *this, transport_t *trans,
@@ -255,8 +255,18 @@ call_bail (void *data)
 {
 	client_connection_t *conn = NULL;
 	struct timeval       current;
-	int32_t              bail_out = 0;
 	transport_t         *trans = NULL;
+	struct list_head     list;
+	struct saved_frame  *saved_frame = NULL;
+	struct saved_frame   *trav = NULL;
+	struct saved_frame   *tmp = NULL;
+	call_frame_t         *frame = NULL;
+	gf_hdr_common_t       hdr = {0, };
+	dict_t               *reply = NULL;
+	char                **gf_op_list = NULL;
+	gf_op_t              *gf_ops = NULL;
+	struct tm             frame_sent_tm;
+	char                  frame_sent[32] = {0,};
 
 	GF_VALIDATE_OR_GOTO("client", data, out);
 	trans = data;
@@ -264,6 +274,8 @@ call_bail (void *data)
 	conn = trans->xl_private;
 
 	gettimeofday (&current, NULL);
+	INIT_LIST_HEAD (&list);
+
 	pthread_mutex_lock (&conn->lock);
 	{
 		/* Chaining to get call-always functionality from 
@@ -277,55 +289,93 @@ call_bail (void *data)
 
 			gf_timer_call_cancel (trans->xl->ctx, conn->timer);
 			conn->timer = gf_timer_call_after (trans->xl->ctx,
-							       timeout,
-							       timer_cbk,
-							       trans);
+							   timeout,
+							   timer_cbk,
+							   trans);
 			if (conn->timer == NULL) {
 				gf_log (trans->xl->name, GF_LOG_DEBUG,
 					"Cannot create bailout timer");
 			}
 		}
 
-		if (((conn->saved_frames->count > 0) &&
-		     (RECEIVE_TIMEOUT(conn, current)) && 
-		     (SEND_TIMEOUT(conn, current)))) {
-
-			struct tm last_sent_tm, last_received_tm;
-			char last_sent[32] = {0,}, last_received[32] = {0,};
+		/* TODO while(1) is not nice - use splice */
 
-			bail_out = 1;
+		do {
+			saved_frame = 
+			saved_frames_get_timedout (conn->saved_frames,
+						   GF_OP_TYPE_MOP_REQUEST,
+						   conn->transport_timeout,
+						   &current);
+			if (saved_frame)
+				list_add (&saved_frame->list, &list);
 			
-			localtime_r (&conn->last_sent.tv_sec, 
-				     &last_sent_tm);
-			localtime_r (&conn->last_received.tv_sec, 
-				     &last_received_tm);
-			
-			strftime (last_sent, 32, 
-				  "%Y-%m-%d %H:%M:%S", &last_sent_tm);
-			strftime (last_received, 32, 
-				  "%Y-%m-%d %H:%M:%S", &last_received_tm);
-			
-			gf_log (trans->xl->name, GF_LOG_ERROR,
-				"activating bail-out. pending frames = %d. "
-				"last sent = %s. last received = %s. "
-				"transport-timeout = %d",
-				(int32_t) conn->saved_frames->count,
-				last_sent, last_received,
-				conn->transport_timeout);
-		}
+		} while (saved_frame);
+
+		do {
+			saved_frame = 
+			saved_frames_get_timedout (conn->saved_frames,
+						   GF_OP_TYPE_FOP_REQUEST,
+						   conn->transport_timeout,
+						   &current);
+			if (saved_frame)
+				list_add (&saved_frame->list, &list);
+		} while (saved_frame);
+
+		do {
+			saved_frame = 
+			saved_frames_get_timedout (conn->saved_frames,
+						   GF_OP_TYPE_CBK_REQUEST,
+						   conn->transport_timeout,
+						   &current);
+			if (saved_frame)
+				list_add (&saved_frame->list, &list);
+		} while (saved_frame);
 	}
+	pthread_mutex_unlock (&conn->lock);
 
-	if (bail_out) {
-		conn->ping_started = 0;
-	}
+	reply = get_new_dict();
+	dict_ref (reply);
 
-	pthread_mutex_unlock (&conn->lock);
+	hdr.rsp.op_ret   = hton32 (-1);
+	hdr.rsp.op_errno = hton32 (ENOTCONN);
+
+	list_for_each_entry_safe (trav, tmp, &list, list) {
+		switch (trav->type)
+		{
+		case GF_OP_TYPE_FOP_REQUEST:
+			gf_ops = gf_fops;
+			gf_op_list = gf_fop_list;
+			break;
+		case GF_OP_TYPE_MOP_REQUEST:
+			gf_ops = gf_mops;
+			gf_op_list = gf_mop_list;
+			break;
+		case GF_OP_TYPE_CBK_REQUEST:
+			gf_ops = gf_cbks;
+			gf_op_list = gf_cbk_list;
+			break;
+		}
+
+		localtime_r (&trav->saved_at.tv_sec, &frame_sent_tm);
+		strftime (frame_sent, 32, "%Y-%m-%d %H:%M:%S", &frame_sent_tm);
+
+		gf_log (trans->xl->name, GF_LOG_ERROR,
+			"activating bail-out :"
+			"frame sent = %s. transport-timeout = %d",
+			frame_sent, conn->transport_timeout);
+
+		hdr.type = hton32 (trav->type);
+		hdr.op   = hton32 (trav->op);
+
+		frame = trav->frame;
+		frame->root->rsp_refs = reply;
 
-	if (bail_out) {
-		gf_log (trans->xl->name, GF_LOG_CRITICAL,
-			"bailing transport");
-		transport_disconnect (trans);
+		gf_ops[trav->op] (frame, &hdr, sizeof (hdr), NULL, 0);
+
+		list_del_init (&trav->list);
+		FREE (trav);
 	}
+	dict_unref (reply);
 out:
 	return;
 }
@@ -398,7 +448,6 @@ client_get_forgets (xlator_t *this, client_forget_t *forget)
 	return ret;
 }
 
-
 void 
 client_ping_timer_expired (void *data)
 {
diff --git a/xlators/protocol/client/src/saved-frames.c b/xlators/protocol/client/src/saved-frames.c
index 0d1366d8222..6f5b32aedb1 100644
--- a/xlators/protocol/client/src/saved-frames.c
+++ b/xlators/protocol/client/src/saved-frames.c
@@ -87,9 +87,9 @@ saved_frames_put (struct saved_frames *frames, call_frame_t *frame,
 	saved_frame->type   = type;
 	saved_frame->callid = callid;
 
-//	gettimeofday (&saved_frame->saved_at, NULL);
+	gettimeofday (&saved_frame->saved_at, NULL);
 
-	list_add (&saved_frame->list, &head_frame->list);
+	list_add_tail (&saved_frame->list, &head_frame->list);
 	frames->count++;
 
 	return 0;
@@ -124,6 +124,26 @@ saved_frames_get (struct saved_frames *frames, int32_t op,
 	return frame;
 }
 
+struct saved_frame *
+saved_frames_get_timedout (struct saved_frames *frames, int8_t type, 
+			   uint32_t timeout, struct timeval *current)
+{
+	struct saved_frame *bailout_frame = NULL, *tmp = NULL;
+	struct saved_frame *head_frame = NULL;
+
+	head_frame = get_head_frame_for_type (frames, type);
+
+	if (!list_empty(&head_frame->list)) {
+		tmp = list_entry (head_frame->list.next, typeof (*tmp), list);
+		if ((tmp->saved_at.tv_sec + timeout) < current->tv_sec) {
+			bailout_frame = tmp;
+			list_del_init (&bailout_frame->list);
+			frames->count--;
+		}
+	}
+
+	return bailout_frame;
+}
 
 void
 saved_frames_unwind (xlator_t *this, struct saved_frames *saved_frames,
diff --git a/xlators/protocol/client/src/saved-frames.h b/xlators/protocol/client/src/saved-frames.h
index e402feba33b..96d956884b2 100644
--- a/xlators/protocol/client/src/saved-frames.h
+++ b/xlators/protocol/client/src/saved-frames.h
@@ -67,6 +67,11 @@ int saved_frames_put (struct saved_frames *frames, call_frame_t *frame,
 		      int32_t op, int8_t type, int64_t callid);
 call_frame_t *saved_frames_get (struct saved_frames *frames, int32_t op,
 				int8_t type, int64_t callid);
+
+struct saved_frame *
+saved_frames_get_timedout (struct saved_frames *frames, int8_t type, 
+			   uint32_t timeout, struct timeval *current);
+
 void saved_frames_destroy (xlator_t *this, struct saved_frames *frames,
 			   gf_op_t gf_fops[], gf_op_t gf_mops[],
 			   gf_op_t gf_cbks[]);