VPP-659 TCP improvements

- builtin test echo server
- fix SYN-ACK retransmit canceling
- avoid sending spurious ACK if in LAST_ACK
- improved client dummy test app
- renamed tx fifo dequeuing and sending functions to avoid confusion
- improved RST handling

Change-Id: Ia14aad3df319540dcf6e6a4e18a9f8d423a4b83b
Signed-off-by: Florin Coras <fcoras@cisco.com>
Signed-off-by: Dave Barach <dave@barachs.net>

6 files changed, 238 insertions, 54 deletions

diff --git a/src/vnet/tcp/builtin_server.c b/src/vnet/tcp/builtin_server.c
index be65642ae3b..9b697a01d5f 100644
--- a/src/vnet/tcp/builtin_server.c
+++ b/src/vnet/tcp/builtin_server.c
@@ -18,10 +18,24 @@
 #include <vnet/session/application.h>
 #include <vnet/session/application_interface.h>
 
+typedef struct
+{
+  u8 *rx_buf;
+  unix_shared_memory_queue_t **vpp_queue;
+  vlib_main_t *vlib_main;
+} builtin_server_main_t;
+
+builtin_server_main_t builtin_server_main;
+
+
 int
 builtin_session_accept_callback (stream_session_t * s)
 {
+  builtin_server_main_t *bsm = &builtin_server_main;
   clib_warning ("called...");
+
+  bsm->vpp_queue[s->thread_index] =
+    session_manager_get_vpp_event_queue (s->thread_index);
   s->session_state = SESSION_STATE_READY;
   return 0;
 }
@@ -30,8 +44,19 @@ void
 builtin_session_disconnect_callback (stream_session_t * s)
 {
   clib_warning ("called...");
+
+  vnet_disconnect_session (s->session_index, s->thread_index);
 }
 
+void
+builtin_session_reset_callback (stream_session_t * s)
+{
+  clib_warning ("called.. ");
+
+  stream_session_cleanup (s);
+}
+
+
 int
 builtin_session_connected_callback (u32 client_index,
 				    stream_session_t * s, u8 is_fail)
@@ -56,9 +81,57 @@ builtin_redirect_connect_callback (u32 client_index, void *mp)
 }
 
 int
-builtin_server_rx_callback (stream_session_t * s)
+builtin_server_rx_callback (stream_session_t * s, session_fifo_event_t * e)
 {
-  clib_warning ("called...");
+  int n_written, bytes, total_copy_bytes;
+  int n_read;
+  svm_fifo_t *tx_fifo;
+  builtin_server_main_t *bsm = &builtin_server_main;
+  session_fifo_event_t evt;
+  static int serial_number = 0;
+
+  bytes = e->enqueue_length;
+  if (PREDICT_FALSE (bytes <= 0))
+    {
+      clib_warning ("bizarre rx callback: bytes %d", bytes);
+      return 0;
+    }
+
+  tx_fifo = s->server_tx_fifo;
+
+  /* Number of bytes we're going to copy */
+  total_copy_bytes = (bytes < (tx_fifo->nitems - tx_fifo->cursize)) ? bytes :
+    tx_fifo->nitems - tx_fifo->cursize;
+
+  if (PREDICT_FALSE (total_copy_bytes <= 0))
+    {
+      clib_warning ("no space in tx fifo, event had %d bytes", bytes);
+      return 0;
+    }
+
+  vec_validate (bsm->rx_buf, total_copy_bytes - 1);
+  _vec_len (bsm->rx_buf) = total_copy_bytes;
+
+  n_read = svm_fifo_dequeue_nowait (s->server_rx_fifo, 0, total_copy_bytes,
+				    bsm->rx_buf);
+  ASSERT (n_read == total_copy_bytes);
+
+  /*
+   * Echo back
+   */
+
+  n_written = svm_fifo_enqueue_nowait (tx_fifo, 0, n_read, bsm->rx_buf);
+  ASSERT (n_written == total_copy_bytes);
+
+  /* Fabricate TX event, send to vpp */
+  evt.fifo = tx_fifo;
+  evt.event_type = FIFO_EVENT_SERVER_TX;
+  evt.enqueue_length = total_copy_bytes;
+  evt.event_id = serial_number++;
+
+  unix_shared_memory_queue_add (bsm->vpp_queue[s->thread_index], (u8 *) & evt,
+				0 /* do wait for mutex */ );
+
   return 0;
 }
 
@@ -68,7 +141,8 @@ static session_cb_vft_t builtin_session_cb_vft = {
   .session_connected_callback = builtin_session_connected_callback,
   .add_segment_callback = builtin_add_segment_callback,
   .redirect_connect_callback = builtin_redirect_connect_callback,
-  .builtin_server_rx_callback = builtin_server_rx_callback
+  .builtin_server_rx_callback = builtin_server_rx_callback,
+  .session_reset_callback = builtin_session_reset_callback
 };
 
 static int
@@ -77,6 +151,11 @@ server_create (vlib_main_t * vm)
   vnet_bind_args_t _a, *a = &_a;
   u64 options[SESSION_OPTIONS_N_OPTIONS];
   char segment_name[128];
+  u32 num_threads;
+  vlib_thread_main_t *vtm = vlib_get_thread_main ();
+
+  num_threads = 1 /* main thread */  + vtm->n_threads;
+  vec_validate (builtin_server_main.vpp_queue, num_threads - 1);
 
   memset (a, 0, sizeof (*a));
   memset (options, 0, sizeof (options));
@@ -110,6 +189,7 @@ server_create_command_fn (vlib_main_t * vm,
     }
 #endif
 
+  vnet_session_enable_disable (vm, 1 /* turn on TCP, etc. */ );
   rv = server_create (vm);
   switch (rv)
     {
@@ -121,10 +201,14 @@ server_create_command_fn (vlib_main_t * vm,
   return 0;
 }
 
+/* *INDENT-OFF* */
 VLIB_CLI_COMMAND (server_create_command, static) =
 {
-.path = "test server",.short_help = "test server",.function =
-    server_create_command_fn,};
+  .path = "test server",
+  .short_help = "test server",
+  .function = server_create_command_fn,
+};
+/* *INDENT-ON* */
 
 /*
 * fd.io coding-style-patch-verification: ON
diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c
index 69433e26a8c..d2df5c3eae0 100644
--- a/src/vnet/tcp/tcp.c
+++ b/src/vnet/tcp/tcp.c
@@ -139,6 +139,20 @@ tcp_connection_del (tcp_connection_t * tc)
   tcp_connection_cleanup (tc);
 }
 
+/** Notify session that connection has been reset.
+ *
+ * Switch state to closed and wait for session to call cleanup.
+ */
+void
+tcp_connection_reset (tcp_connection_t * tc)
+{
+  if (tc->state == TCP_STATE_CLOSED)
+    return;
+
+  tc->state = TCP_STATE_CLOSED;
+  stream_session_reset_notify (&tc->connection);
+}
+
 /**
  * Begin connection closing procedure.
  *
@@ -149,6 +163,8 @@ tcp_connection_del (tcp_connection_t * tc)
  * calls cleanup.
  * 2) TIME_WAIT (active close) whereby after 2MSL the 2MSL timer triggers
  * and cleanup is called.
+ *
+ * N.B. Half-close connections are not supported
  */
 void
 tcp_connection_close (tcp_connection_t * tc)
@@ -166,9 +182,9 @@ tcp_connection_close (tcp_connection_t * tc)
   else if (tc->state == TCP_STATE_CLOSE_WAIT)
     tc->state = TCP_STATE_LAST_ACK;
 
-  /* Half-close connections are not supported XXX */
-
-  if (tc->state == TCP_STATE_CLOSED)
+  /* If in CLOSED and WAITCLOSE timer is not set, delete connection now */
+  if (tc->timers[TCP_TIMER_WAITCLOSE] == TCP_TIMER_HANDLE_INVALID
+      && tc->state == TCP_STATE_CLOSED)
     tcp_connection_del (tc);
 }
 
@@ -185,7 +201,10 @@ tcp_session_cleanup (u32 conn_index, u32 thread_index)
 {
   tcp_connection_t *tc;
   tc = tcp_connection_get (conn_index, thread_index);
-  tcp_connection_cleanup (tc);
+
+  /* Wait for the session tx events to clear */
+  tc->state = TCP_STATE_CLOSED;
+  tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, TCP_CLEANUP_TIME);
 }
 
 void *
@@ -227,7 +246,8 @@ tcp_allocate_local_port (tcp_main_t * tm, ip46_address_t * ip)
 {
   transport_endpoint_t *tep;
   u32 time_now, tei;
-  u16 min = 1024, max = 65535, tries;	/* XXX configurable ? */
+  u16 min = 1024, max = 65535;	/* XXX configurable ? */
+  int tries;
 
   tries = max - min;
   time_now = tcp_time_now ();
@@ -505,10 +525,10 @@ tcp_session_send_space (transport_connection_t * trans_conn)
 }
 
 u32
-tcp_session_rx_fifo_offset (transport_connection_t * trans_conn)
+tcp_session_tx_fifo_offset (transport_connection_t * trans_conn)
 {
   tcp_connection_t *tc = (tcp_connection_t *) trans_conn;
-  return (tc->snd_una_max - tc->snd_una);
+  return (tc->snd_nxt - tc->snd_una);
 }
 
 /* *INDENT-OFF* */
@@ -524,7 +544,7 @@ const static transport_proto_vft_t tcp4_proto = {
   .cleanup = tcp_session_cleanup,
   .send_mss = tcp_session_send_mss,
   .send_space = tcp_session_send_space,
-  .rx_fifo_offset = tcp_session_rx_fifo_offset,
+  .tx_fifo_offset = tcp_session_tx_fifo_offset,
   .format_connection = format_tcp_session_ip4,
   .format_listener = format_tcp_listener_session_ip4,
   .format_half_open = format_tcp_half_open_session_ip4
@@ -542,7 +562,7 @@ const static transport_proto_vft_t tcp6_proto = {
   .cleanup = tcp_session_cleanup,
   .send_mss = tcp_session_send_mss,
   .send_space = tcp_session_send_space,
-  .rx_fifo_offset = tcp_session_rx_fifo_offset,
+  .tx_fifo_offset = tcp_session_tx_fifo_offset,
   .format_connection = format_tcp_session_ip6,
   .format_listener = format_tcp_listener_session_ip6,
   .format_half_open = format_tcp_half_open_session_ip6
@@ -579,13 +599,32 @@ tcp_timer_establish_handler (u32 conn_index)
 }
 
 void
-tcp_timer_2msl_handler (u32 conn_index)
+tcp_timer_waitclose_handler (u32 conn_index)
 {
   u32 cpu_index = os_get_cpu_number ();
   tcp_connection_t *tc;
 
   tc = tcp_connection_get (conn_index, cpu_index);
-  tc->timers[TCP_TIMER_2MSL] = TCP_TIMER_HANDLE_INVALID;
+  tc->timers[TCP_TIMER_WAITCLOSE] = TCP_TIMER_HANDLE_INVALID;
+
+  /* Session didn't come back with a close(). Send FIN either way
+   * and switch to LAST_ACK. */
+  if (tc->state == TCP_STATE_CLOSE_WAIT)
+    {
+      if (tc->flags & TCP_CONN_FINSNT)
+	{
+	  clib_warning ("FIN was sent and still in CLOSE WAIT. Weird!");
+	}
+
+      tcp_send_fin (tc);
+      tc->state = TCP_STATE_LAST_ACK;
+
+      /* Make sure we don't wait in LAST ACK forever */
+      tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME);
+
+      /* Don't delete the connection yet */
+      return;
+    }
 
   tcp_connection_del (tc);
 }
@@ -597,7 +636,7 @@ static timer_expiration_handler *timer_expiration_handlers[TCP_N_TIMERS] =
     tcp_timer_delack_handler,
     0,
     tcp_timer_keep_handler,
-    tcp_timer_2msl_handler,
+    tcp_timer_waitclose_handler,
     tcp_timer_retransmit_syn_handler,
     tcp_timer_establish_handler
 };
diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h
index 7d443433fa2..3b3d8fc7cff 100644
--- a/src/vnet/tcp/tcp.h
+++ b/src/vnet/tcp/tcp.h
@@ -63,8 +63,8 @@ format_function_t format_tcp_state;
   _(DELACK, "DELAYED ACK")              \
   _(PERSIST, "PERSIST")                 \
   _(KEEP, "KEEP")                       \
-  _(2MSL, "2MSL")                       \
-  _(RETRANSMIT_SYN, "RETRANSMIT_SYN")   \
+  _(WAITCLOSE, "WAIT CLOSE")            \
+  _(RETRANSMIT_SYN, "RETRANSMIT SYN")   \
   _(ESTABLISH, "ESTABLISH")
 
 typedef enum _tcp_timers
@@ -89,6 +89,8 @@ extern timer_expiration_handler tcp_timer_retransmit_syn_handler;
 #define TCP_DELACK_TIME         1	/* 0.1s */
 #define TCP_ESTABLISH_TIME      750	/* 75s */
 #define TCP_2MSL_TIME           300	/* 30s */
+#define TCP_CLOSEWAIT_TIME	1	/* 0.1s */
+#define TCP_CLEANUP_TIME	5	/* 0.5s Time to wait before cleanup */
 
 #define TCP_RTO_MAX 60 * THZ	/* Min max RTO (60s) as per RFC6298 */
 #define TCP_RTT_MAX 30 * THZ	/* 30s (probably too much) */
@@ -102,6 +104,7 @@ void tcp_update_time (f64 now, u32 thread_index);
   _(DELACK, "Delay ACK")                        \
   _(SNDACK, "Send ACK")                         \
   _(BURSTACK, "Burst ACK set")                  \
+  _(FINSNT, "FIN sent")				\
   _(SENT_RCV_WND0, "Sent 0 receive window")     \
   _(RECOVERY, "Recovery on")                    \
   _(FAST_RECOVERY, "Fast Recovery on")
@@ -331,6 +334,8 @@ clib_error_t *vnet_tcp_enable_disable (vlib_main_t * vm, u8 is_en);
 always_inline tcp_connection_t *
 tcp_connection_get (u32 conn_index, u32 thread_index)
 {
+  if (pool_is_free_index (tcp_main.connections[thread_index], conn_index))
+    return 0;
   return pool_elt_at_index (tcp_main.connections[thread_index], conn_index);
 }
 
@@ -347,6 +352,7 @@ tcp_connection_get_if_valid (u32 conn_index, u32 thread_index)
 void tcp_connection_close (tcp_connection_t * tc);
 void tcp_connection_cleanup (tcp_connection_t * tc);
 void tcp_connection_del (tcp_connection_t * tc);
+void tcp_connection_reset (tcp_connection_t * tc);
 
 always_inline tcp_connection_t *
 tcp_listener_get (u32 tli)
@@ -361,7 +367,7 @@ tcp_half_open_connection_get (u32 conn_index)
 }
 
 void tcp_make_ack (tcp_connection_t * ts, vlib_buffer_t * b);
-void tcp_make_finack (tcp_connection_t * tc, vlib_buffer_t * b);
+void tcp_make_fin (tcp_connection_t * tc, vlib_buffer_t * b);
 void tcp_make_synack (tcp_connection_t * ts, vlib_buffer_t * b);
 void tcp_send_reset (vlib_buffer_t * pkt, u8 is_ip4);
 void tcp_send_syn (tcp_connection_t * tc);
@@ -467,7 +473,7 @@ tcp_timer_set (tcp_connection_t * tc, u8 timer_id, u32 interval)
 }
 
 always_inline void
-tcp_retransmit_timer_set (tcp_main_t * tm, tcp_connection_t * tc)
+tcp_retransmit_timer_set (tcp_connection_t * tc)
 {
   /* XXX Switch to faster TW */
   tcp_timer_set (tc, TCP_TIMER_RETRANSMIT,
diff --git a/src/vnet/tcp/tcp_error.def b/src/vnet/tcp/tcp_error.def
index cff5ec13a70..2dbdd9b38d1 100644
--- a/src/vnet/tcp/tcp_error.def
+++ b/src/vnet/tcp/tcp_error.def
@@ -17,13 +17,13 @@ tcp_error (NONE, "no error")
 tcp_error (NO_LISTENER, "no listener for dst port")
 tcp_error (LOOKUP_DROPS, "lookup drops")
 tcp_error (DISPATCH, "Dispatch error")
-tcp_error (ENQUEUED, "Packets pushed into rx fifo")                              
+tcp_error (ENQUEUED, "Packets pushed into rx fifo") 
 tcp_error (PURE_ACK, "Pure acks")
 tcp_error (SYNS_RCVD, "SYNs received")
 tcp_error (SYN_ACKS_RCVD, "SYN-ACKs received")
-tcp_error (NOT_READY, "Session not ready for packets")                               
-tcp_error (FIFO_FULL, "Packets dropped for lack of rx fifo space")               
-tcp_error (EVENT_FIFO_FULL, "Events not sent for lack of event fifo space")      
+tcp_error (NOT_READY, "Session not ready for packets") 
+tcp_error (FIFO_FULL, "Packets dropped for lack of rx fifo space") 
+tcp_error (EVENT_FIFO_FULL, "Events not sent for lack of event fifo space") 
 tcp_error (API_QUEUE_FULL, "Sessions not created for lack of API queue space")
 tcp_error (CREATE_SESSION_FAIL, "Sessions couldn't be allocated")
 tcp_error (SEGMENT_INVALID, "Invalid segment")
@@ -32,4 +32,5 @@ tcp_error (ACK_DUP, "Duplicate ACK")
 tcp_error (ACK_OLD, "Old ACK")
 tcp_error (PKTS_SENT, "Packets sent")
 tcp_error (FILTERED_DUPACKS, "Filtered duplicate ACKs")
-tcp_error (RST_SENT, "Resets sent")
\ No newline at end of file
+tcp_error (RST_SENT, "Resets sent")
+tcp_error (INVALID_CONNECTION, "Invalid connection")
diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c
index 0a907d0a3d7..f19fbf874dc 100644
--- a/src/vnet/tcp/tcp_input.c
+++ b/src/vnet/tcp/tcp_input.c
@@ -274,10 +274,7 @@ tcp_segment_validate (vlib_main_t * vm, tcp_connection_t * tc0,
   /* 2nd: check the RST bit */
   if (tcp_rst (th0))
     {
-      /* Notify session that connection has been reset. Switch
-       * state to closed and await for session to do the cleanup. */
-      stream_session_reset_notify (&tc0->connection);
-      tc0->state = TCP_STATE_CLOSED;
+      tcp_connection_reset (tc0);
       return -1;
     }
 
@@ -1023,6 +1020,12 @@ tcp46_established_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  tc0 = tcp_connection_get (vnet_buffer (b0)->tcp.connection_index,
 				    my_thread_index);
 
+	  if (PREDICT_FALSE (tc0 == 0))
+	    {
+	      error0 = TCP_ERROR_INVALID_CONNECTION;
+	      goto drop;
+	    }
+
 	  /* Checksum computed by ipx_local no need to compute again */
 
 	  if (is_ip4)
@@ -1072,12 +1075,12 @@ tcp46_established_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  /* 8: check the FIN bit */
 	  if (tcp_fin (th0))
 	    {
-	      /* Send ACK and enter CLOSE-WAIT */
-	      tcp_make_ack (tc0, b0);
-	      tcp_connection_force_ack (tc0, b0);
-	      next0 = tcp_next_output (tc0->c_is_ip4);
+	      /* Enter CLOSE-WAIT and notify session. Don't send ACK, instead
+	       * wait for session to call close. To avoid lingering
+	       * in CLOSE-WAIT, set timer (reuse WAITCLOSE). */
 	      tc0->state = TCP_STATE_CLOSE_WAIT;
 	      stream_session_disconnect_notify (&tc0->connection);
+	      tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, TCP_CLOSEWAIT_TIME);
 	    }
 
 	drop:
@@ -1468,7 +1471,7 @@ VLIB_REGISTER_NODE (tcp6_syn_sent_node) =
 
 VLIB_NODE_FUNCTION_MULTIARCH (tcp6_syn_sent_node, tcp6_syn_sent_rcv);
 /**
- * Handles reception for all states except LISTEN, SYN-SEND and ESTABLISHED
+ * Handles reception for all states except LISTEN, SYN-SENT and ESTABLISHED
  * as per RFC793 p. 64
  */
 always_inline uword
@@ -1511,6 +1514,11 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  b0 = vlib_get_buffer (vm, bi0);
 	  tc0 = tcp_connection_get (vnet_buffer (b0)->tcp.connection_index,
 				    my_thread_index);
+	  if (PREDICT_FALSE (tc0 == 0))
+	    {
+	      error0 = TCP_ERROR_INVALID_CONNECTION;
+	      goto drop;
+	    }
 
 	  /* Checksum computed by ipx_local no need to compute again */
 
@@ -1587,7 +1595,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      /* Shoulder tap the server */
 	      stream_session_accept_notify (&tc0->connection);
 
-	      tcp_timer_reset (tc0, TCP_TIMER_RETRANSMIT_SYN);
+	      /* Reset SYN-ACK retransmit timer */
+	      tcp_timer_reset (tc0, TCP_TIMER_RETRANSMIT);
 	      break;
 	    case TCP_STATE_ESTABLISHED:
 	      /* We can get packets in established state here because they
@@ -1602,9 +1611,14 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	       * continue processing in that state. */
 	      if (tcp_rcv_ack (tc0, b0, tcp0, &next0, &error0))
 		goto drop;
-	      tc0->state = TCP_STATE_FIN_WAIT_2;
-	      /* Stop all timers, 2MSL will be set lower */
-	      tcp_connection_timers_reset (tc0);
+
+	      /* If FIN is ACKed */
+	      if (tc0->snd_una == tc0->snd_una_max)
+		{
+		  tc0->state = TCP_STATE_FIN_WAIT_2;
+		  /* Stop all timers, 2MSL will be set lower */
+		  tcp_connection_timers_reset (tc0);
+		}
 	      break;
 	    case TCP_STATE_FIN_WAIT_2:
 	      /* In addition to the processing for the ESTABLISHED state, if
@@ -1639,7 +1653,17 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      if (!tcp_rcv_ack_is_acceptable (tc0, b0))
 		goto drop;
 
-	      tcp_connection_del (tc0);
+	      tc0->state = TCP_STATE_CLOSED;
+
+	      /* Don't delete the connection/session yet. Instead, wait a
+	       * reasonable amount of time until the pipes are cleared. In
+	       * particular, this makes sure that we won't have dead sessions
+	       * when processing events on the tx path */
+	      tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_CLEANUP_TIME);
+
+	      /* Stop retransmit */
+	      tcp_timer_reset (tc0, TCP_TIMER_RETRANSMIT);
+
 	      goto drop;
 
 	      break;
@@ -1684,7 +1708,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	    case TCP_STATE_SYN_RCVD:
 	      /* Send FIN-ACK notify app and enter CLOSE-WAIT */
 	      tcp_connection_timers_reset (tc0);
-	      tcp_make_finack (tc0, b0);
+	      tcp_make_fin (tc0, b0);
 	      next0 = tcp_next_output (tc0->c_is_ip4);
 	      stream_session_disconnect_notify (&tc0->connection);
 	      tc0->state = TCP_STATE_CLOSE_WAIT;
@@ -1697,12 +1721,12 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	    case TCP_STATE_FIN_WAIT_1:
 	      tc0->state = TCP_STATE_TIME_WAIT;
 	      tcp_connection_timers_reset (tc0);
-	      tcp_timer_set (tc0, TCP_TIMER_2MSL, TCP_2MSL_TIME);
+	      tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME);
 	      break;
 	    case TCP_STATE_FIN_WAIT_2:
 	      /* Got FIN, send ACK! */
 	      tc0->state = TCP_STATE_TIME_WAIT;
-	      tcp_timer_set (tc0, TCP_TIMER_2MSL, TCP_2MSL_TIME);
+	      tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME);
 	      tcp_make_ack (tc0, b0);
 	      next0 = tcp_next_output (is_ip4);
 	      break;
@@ -1710,7 +1734,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      /* Remain in the TIME-WAIT state. Restart the 2 MSL time-wait
 	       * timeout.
 	       */
-	      tcp_timer_update (tc0, TCP_TIMER_2MSL, TCP_2MSL_TIME);
+	      tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME);
 	      break;
 	    }
 
@@ -2113,6 +2137,7 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  n_left_to_next -= 1;
 
 	  b0 = vlib_get_buffer (vm, bi0);
+	  vnet_buffer (b0)->tcp.flags = 0;
 
 	  if (is_ip4)
 	    {
@@ -2168,7 +2193,6 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      /* Send reset */
 	      next0 = TCP_INPUT_NEXT_RESET;
 	      error0 = TCP_ERROR_NO_LISTENER;
-	      vnet_buffer (b0)->tcp.flags = 0;
 	    }
 
 	  b0->error = error0 ? node->errors[error0] : 0;
@@ -2288,6 +2312,7 @@ do {                                                       	\
   _(ESTABLISHED, TCP_FLAG_FIN, TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
   _(ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_ESTABLISHED,
     TCP_ERROR_NONE);
+  _(ESTABLISHED, TCP_FLAG_RST, TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
   /* ACK or FIN-ACK to our FIN */
   _(FIN_WAIT_1, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
   _(FIN_WAIT_1, TCP_FLAG_ACK | TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS,
diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index 7e431cd0454..aa43e9f37f0 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -396,6 +396,7 @@ tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b)
 
   /* Leave enough space for headers */
   vlib_buffer_make_headroom (b, MAX_HDRS_LEN);
+  vnet_buffer (b)->tcp.flags = 0;
 }
 
 /**
@@ -443,16 +444,22 @@ tcp_make_ack (tcp_connection_t * tc, vlib_buffer_t * b)
  * Convert buffer to FIN-ACK
  */
 void
-tcp_make_finack (tcp_connection_t * tc, vlib_buffer_t * b)
+tcp_make_fin (tcp_connection_t * tc, vlib_buffer_t * b)
 {
   tcp_main_t *tm = vnet_get_tcp_main ();
   vlib_main_t *vm = tm->vlib_main;
+  u8 flags = 0;
 
   tcp_reuse_buffer (vm, b);
-  tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_ACK | TCP_FLAG_FIN);
+
+  if (tc->rcv_las == tc->rcv_nxt)
+    flags = TCP_FLAG_FIN;
+  else
+    flags = TCP_FLAG_FIN | TCP_FLAG_ACK;
+
+  tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, flags);
 
   /* Reset flags, make sure ack is sent */
-  tc->flags = TCP_CONN_SNDACK;
   vnet_buffer (b)->tcp.flags &= ~TCP_BUF_FLAG_DUPACK;
 
   tc->snd_nxt += 1;
@@ -500,7 +507,7 @@ tcp_make_synack (tcp_connection_t * tc, vlib_buffer_t * b)
   vnet_buffer (b)->tcp.flags = TCP_BUF_FLAG_ACK;
 
   /* Init retransmit timer */
-  tcp_retransmit_timer_set (tm, tc);
+  tcp_retransmit_timer_set (tc);
 }
 
 always_inline void
@@ -818,9 +825,9 @@ tcp_send_fin (tcp_connection_t * tc)
   /* Leave enough space for headers */
   vlib_buffer_make_headroom (b, MAX_HDRS_LEN);
 
-  tcp_make_finack (tc, b);
-
+  tcp_make_fin (tc, b);
   tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
+  tc->flags |= TCP_CONN_FINSNT;
 }
 
 always_inline u8
@@ -1038,7 +1045,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn)
       tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
 
       /* Re-enable retransmit timer */
-      tcp_retransmit_timer_set (tm, tc);
+      tcp_retransmit_timer_set (tc);
     }
   else
     {
@@ -1139,7 +1146,6 @@ tcp46_output_inline (vlib_main_t * vm,
 		     vlib_node_runtime_t * node,
 		     vlib_frame_t * from_frame, int is_ip4)
 {
-  tcp_main_t *tm = vnet_get_tcp_main ();
   u32 n_left_from, next_index, *from, *to_next;
   u32 my_thread_index = vm->cpu_index;
 
@@ -1172,6 +1178,13 @@ tcp46_output_inline (vlib_main_t * vm,
 	  b0 = vlib_get_buffer (vm, bi0);
 	  tc0 = tcp_connection_get (vnet_buffer (b0)->tcp.connection_index,
 				    my_thread_index);
+	  if (PREDICT_FALSE (tc0 == 0 || tc0->state == TCP_STATE_CLOSED))
+	    {
+	      error0 = TCP_ERROR_INVALID_CONNECTION;
+	      next0 = TCP_OUTPUT_NEXT_DROP;
+	      goto done;
+	    }
+
 	  th0 = vlib_buffer_get_current (b0);
 
 	  if (is_ip4)
@@ -1229,6 +1242,22 @@ tcp46_output_inline (vlib_main_t * vm,
 		  tc0->rtt_ts = tcp_time_now ();
 		  tc0->rtt_seq = tc0->snd_nxt;
 		}
+
+	      if (1)
+		{
+		  ELOG_TYPE_DECLARE (e) =
+		  {
+		  .format =
+		      "output: snd_una %u snd_una_max %u",.format_args =
+		      "i4i4",};
+		  struct
+		  {
+		    u32 data[2];
+		  } *ed;
+		  ed = ELOG_DATA (&vm->elog_main, e);
+		  ed->data[0] = tc0->snd_una - tc0->iss;
+		  ed->data[1] = tc0->snd_una_max - tc0->iss;
+		}
 	    }
 
 	  /* Set the retransmit timer if not set already and not
@@ -1236,7 +1265,7 @@ tcp46_output_inline (vlib_main_t * vm,
 	  if (!tcp_timer_is_active (tc0, TCP_TIMER_RETRANSMIT)
 	      && tc0->snd_nxt != tc0->snd_una)
 	    {
-	      tcp_retransmit_timer_set (tm, tc0);
+	      tcp_retransmit_timer_set (tc0);
 	      tc0->rto_boff = 0;
 	    }