Fixes and improved tcp/session debugging

- Fix rx sack option parsing
- Add session sack scoreboard tracing and replaying
- Add svm fifo tracing and replaying
- Scoreboard/svm fifo ooo segment reception fixes
- Improved overall debugging

Change-Id: Ieae07eba355e66f5935253232bb00f2dfb7ece00
Signed-off-by: Florin Coras <fcoras@cisco.com>

6 files changed, 631 insertions, 42 deletions

diff --git a/src/vnet/tcp/builtin_client.c b/src/vnet/tcp/builtin_client.c
index a6c8a23582b..a92bacaa37d 100644
--- a/src/vnet/tcp/builtin_client.c
+++ b/src/vnet/tcp/builtin_client.c
@@ -464,6 +464,8 @@ builtin_session_connected_callback (u32 app_index, u32 api_context,
 static void
 builtin_session_reset_callback (stream_session_t * s)
 {
+  if (s->session_state == SESSION_STATE_READY)
+    clib_warning ("Reset active connection %U", format_stream_session, s, 2);
   return;
 }
 
diff --git a/src/vnet/tcp/builtin_server.c b/src/vnet/tcp/builtin_server.c
index 8e958ac0b45..4ecaf56a70c 100644
--- a/src/vnet/tcp/builtin_server.c
+++ b/src/vnet/tcp/builtin_server.c
@@ -99,8 +99,7 @@ builtin_session_disconnect_callback (stream_session_t * s)
 void
 builtin_session_reset_callback (stream_session_t * s)
 {
-  clib_warning ("called.. ");
-
+  clib_warning ("Reset session %U", format_stream_session, s, 2);
   stream_session_cleanup (s);
 }
 
@@ -224,10 +223,6 @@ builtin_server_rx_callback (stream_session_t * s)
 	      clib_warning ("session stuck: %U", format_stream_session, s, 2);
 	    }
 	}
-      else
-	{
-	  bsm->rx_retries[thread_index][s->session_index] = 0;
-	}
 
       return 0;
     }
diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c
index f379e699839..8ed325d2809 100644
--- a/src/vnet/tcp/tcp.c
+++ b/src/vnet/tcp/tcp.c
@@ -732,6 +732,7 @@ format_tcp_connection (u8 * s, va_list * args)
       if (verbose > 1)
 	s = format (s, " %U\n%U", format_tcp_timers, tc, format_tcp_vars, tc);
     }
+
   return s;
 }
 
@@ -792,6 +793,30 @@ format_tcp_sacks (u8 * s, va_list * args)
 }
 
 u8 *
+format_tcp_rcv_sacks (u8 * s, va_list * args)
+{
+  tcp_connection_t *tc = va_arg (*args, tcp_connection_t *);
+  sack_block_t *sacks = tc->rcv_opts.sacks;
+  sack_block_t *block;
+  int i, len = 0;
+
+  len = vec_len (sacks);
+  for (i = 0; i < len - 1; i++)
+    {
+      block = &sacks[i];
+      s = format (s, " start %u end %u\n", block->start - tc->iss,
+		  block->end - tc->iss);
+    }
+  if (len)
+    {
+      block = &sacks[len - 1];
+      s = format (s, " start %u end %u", block->start - tc->iss,
+		  block->end - tc->iss);
+    }
+  return s;
+}
+
+u8 *
 format_tcp_sack_hole (u8 * s, va_list * args)
 {
   sack_scoreboard_hole_t *hole = va_arg (*args, sack_scoreboard_hole_t *);
@@ -820,6 +845,7 @@ format_tcp_scoreboard (u8 * s, va_list * args)
       s = format (s, "%U", format_tcp_sack_hole, hole);
       hole = scoreboard_next_hole (sb, hole);
     }
+
   return s;
 }
 
@@ -1304,7 +1330,189 @@ VLIB_CLI_COMMAND (tcp_src_address_command, static) =
 };
 /* *INDENT-ON* */
 
+static u8 *
+tcp_scoreboard_dump_trace (u8 * s, sack_scoreboard_t * sb)
+{
+#if TCP_SCOREBOARD_TRACE
+
+  scoreboard_trace_elt_t *block;
+  int i = 0;
+
+  if (!sb->trace)
+    return s;
+
+  s = format (s, "scoreboard trace:");
+  vec_foreach (block, sb->trace)
+  {
+    s = format (s, "{%u, %u, %u, %u, %u}, ", block->start, block->end,
+		block->ack, block->snd_una_max, block->group);
+    if ((++i % 3) == 0)
+      s = format (s, "\n");
+  }
+  return s;
+#else
+  return 0;
+#endif
+}
+
+static clib_error_t *
+tcp_show_scoreboard_trace_fn (vlib_main_t * vm, unformat_input_t * input,
+			      vlib_cli_command_t * cmd_arg)
+{
+  transport_connection_t *tconn = 0;
+  tcp_connection_t *tc;
+  u8 *s = 0;
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "%U", unformat_transport_connection, &tconn,
+		    TRANSPORT_PROTO_TCP))
+	;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+
+  if (!TCP_SCOREBOARD_TRACE)
+    {
+      vlib_cli_output (vm, "scoreboard tracing not enabled");
+      return 0;
+    }
+
+  tc = tcp_get_connection_from_transport (tconn);
+  s = tcp_scoreboard_dump_trace (s, &tc->sack_sb);
+  vlib_cli_output (vm, "%v", s);
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (tcp_show_scoreboard_trace_command, static) =
+{
+  .path = "show tcp scoreboard trace",
+  .short_help = "show tcp scoreboard trace <connection>",
+  .function = tcp_show_scoreboard_trace_fn,
+};
+/* *INDENT-ON* */
 
+u8 *
+tcp_scoreboard_replay (u8 * s, tcp_connection_t * tc, u8 verbose)
+{
+  int i, trace_len;
+  scoreboard_trace_elt_t *trace;
+  u32 next_ack, left, group, has_new_ack = 0;
+  tcp_connection_t _dummy_tc, *dummy_tc = &_dummy_tc;
+  sack_block_t *block;
+
+  if (!tc)
+    return s;
+
+  memset (dummy_tc, 0, sizeof (*dummy_tc));
+  tcp_connection_timers_init (dummy_tc);
+  scoreboard_init (&dummy_tc->sack_sb);
+  dummy_tc->rcv_opts.flags |= TCP_OPTS_FLAG_SACK;
+
+#if TCP_SCOREBOARD_TRACE
+  trace = tc->sack_sb.trace;
+  trace_len = vec_len (tc->sack_sb.trace);
+#else
+  trace = 0;
+  trace_len = 0;
+#endif
+
+  for (i = 0; i < trace_len; i++)
+    {
+      if (trace[i].ack != 0)
+	{
+	  dummy_tc->snd_una = trace[i].ack - 1448;
+	  dummy_tc->snd_una_max = trace[i].ack;
+	}
+    }
+
+  left = 0;
+  while (left < trace_len)
+    {
+      group = trace[left].group;
+      vec_reset_length (dummy_tc->rcv_opts.sacks);
+      has_new_ack = 0;
+      while (trace[left].group == group)
+	{
+	  if (trace[left].ack != 0)
+	    {
+	      if (verbose)
+		s = format (s, "Adding ack %u, snd_una_max %u, segs: ",
+			    trace[left].ack, trace[left].snd_una_max);
+	      dummy_tc->snd_una_max = trace[left].snd_una_max;
+	      next_ack = trace[left].ack;
+	      has_new_ack = 1;
+	    }
+	  else
+	    {
+	      if (verbose)
+		s = format (s, "[%u, %u], ", trace[left].start,
+			    trace[left].end);
+	      vec_add2 (dummy_tc->rcv_opts.sacks, block, 1);
+	      block->start = trace[left].start;
+	      block->end = trace[left].end;
+	    }
+	  left++;
+	}
+
+      /* Push segments */
+      tcp_rcv_sacks (dummy_tc, next_ack);
+      if (has_new_ack)
+	dummy_tc->snd_una = next_ack + dummy_tc->sack_sb.snd_una_adv;
+
+      if (verbose)
+	s = format (s, "result: %U", format_tcp_scoreboard,
+		    &dummy_tc->sack_sb);
+
+    }
+  s = format (s, "result: %U", format_tcp_scoreboard, &dummy_tc->sack_sb);
+
+  return s;
+}
+
+static clib_error_t *
+tcp_scoreboard_trace_fn (vlib_main_t * vm, unformat_input_t * input,
+			 vlib_cli_command_t * cmd_arg)
+{
+  transport_connection_t *tconn = 0;
+  tcp_connection_t *tc = 0;
+  u8 *str = 0;
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "%U", unformat_transport_connection, &tconn,
+		    TRANSPORT_PROTO_TCP))
+	;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+
+  if (!TCP_SCOREBOARD_TRACE)
+    {
+      vlib_cli_output (vm, "scoreboard tracing not enabled");
+      return 0;
+    }
+
+  tc = tcp_get_connection_from_transport (tconn);
+  if (!tc)
+    {
+      vlib_cli_output (vm, "connection not found");
+      return 0;
+    }
+  str = tcp_scoreboard_replay (str, tc, 1);
+  vlib_cli_output (vm, "%v", str);
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (tcp_replay_scoreboard_command, static) =
+{
+  .path = "tcp replay scoreboard",
+  .short_help = "tcp replay scoreboard <connection>",
+  .function = tcp_scoreboard_trace_fn,
+};
+/* *INDENT-ON* */
 
 /*
  * fd.io coding-style-patch-verification: ON
diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h
index 37b10fd4753..fd0d02b99d6 100644
--- a/src/vnet/tcp/tcp.h
+++ b/src/vnet/tcp/tcp.h
@@ -62,6 +62,7 @@ typedef enum _tcp_state
 format_function_t format_tcp_state;
 format_function_t format_tcp_flags;
 format_function_t format_tcp_sacks;
+format_function_t format_tcp_rcv_sacks;
 
 /** TCP timers */
 #define foreach_tcp_timer               \
@@ -151,9 +152,19 @@ enum
 #undef _
 };
 
+#define TCP_SCOREBOARD_TRACE (0)
 #define TCP_MAX_SACK_BLOCKS 15	/**< Max number of SACK blocks stored */
 #define TCP_INVALID_SACK_HOLE_INDEX ((u32)~0)
 
+typedef struct _scoreboard_trace_elt
+{
+  u32 start;
+  u32 end;
+  u32 ack;
+  u32 snd_una_max;
+  u32 group;
+} scoreboard_trace_elt_t;
+
 typedef struct _sack_scoreboard_hole
 {
   u32 next;		/**< Index for next entry in linked list */
@@ -177,8 +188,38 @@ typedef struct _sack_scoreboard
   u32 rescue_rxt;			/**< Rescue sequence number */
   u32 lost_bytes;			/**< Bytes lost as per RFC6675 */
   u32 cur_rxt_hole;			/**< Retransmitting from this hole */
+
+#if TCP_SCOREBOARD_TRACE
+  scoreboard_trace_elt_t *trace;
+#endif
+
 } sack_scoreboard_t;
 
+#if TCP_SCOREBOARD_TRACE
+#define tcp_scoreboard_trace_add(_tc, _ack) 				\
+{									\
+    static u64 _group = 0;						\
+    sack_scoreboard_t *_sb = &_tc->sack_sb;				\
+    sack_block_t *_sack, *_sacks;					\
+    scoreboard_trace_elt_t *_elt;					\
+    int i;								\
+    _group++;								\
+    _sacks = _tc->rcv_opts.sacks;					\
+    for (i = 0; i < vec_len (_sacks); i++) 				\
+      {									\
+	_sack = &_sacks[i];						\
+	vec_add2 (_sb->trace, _elt, 1);					\
+	_elt->start = _sack->start;					\
+	_elt->end = _sack->end;						\
+	_elt->ack = _elt->end == _ack ? _ack : 0;			\
+	_elt->snd_una_max = _elt->end == _ack ? _tc->snd_una_max : 0;	\
+	_elt->group = _group;						\
+      }									\
+}
+#else
+#define tcp_scoreboard_trace_add(_tc, _ack)
+#endif
+
 typedef enum _tcp_cc_algorithm_type
 {
   TCP_CC_NEWRENO,
@@ -405,6 +446,12 @@ tcp_connection_get_if_valid (u32 conn_index, u32 thread_index)
   return pool_elt_at_index (tcp_main.connections[thread_index], conn_index);
 }
 
+always_inline tcp_connection_t *
+tcp_get_connection_from_transport (transport_connection_t * tconn)
+{
+  return (tcp_connection_t *) tconn;
+}
+
 void tcp_connection_close (tcp_connection_t * tc);
 void tcp_connection_cleanup (tcp_connection_t * tc);
 void tcp_connection_del (tcp_connection_t * tc);
@@ -414,6 +461,8 @@ u8 *format_tcp_connection_id (u8 * s, va_list * args);
 u8 *format_tcp_connection (u8 * s, va_list * args);
 u8 *format_tcp_scoreboard (u8 * s, va_list * args);
 
+u8 *tcp_scoreboard_replay (u8 * s, tcp_connection_t * tc, u8 verbose);
+
 always_inline tcp_connection_t *
 tcp_listener_get (u32 tli)
 {
@@ -689,7 +738,7 @@ sack_scoreboard_hole_t *scoreboard_next_rxt_hole (sack_scoreboard_t * sb,
 						  start, u8 have_sent_1_smss,
 						  u8 * can_rescue,
 						  u8 * snd_limited);
-void scoreboard_init_high_rxt (sack_scoreboard_t * sb);
+void scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 seq);
 
 always_inline sack_scoreboard_hole_t *
 scoreboard_get_hole (sack_scoreboard_t * sb, u32 index)
@@ -740,6 +789,7 @@ scoreboard_clear (sack_scoreboard_t * sb)
       scoreboard_remove_hole (sb, hole);
     }
   ASSERT (sb->head == sb->tail && sb->head == TCP_INVALID_SACK_HOLE_INDEX);
+  ASSERT (pool_elts (sb->holes) == 0);
   sb->sacked_bytes = 0;
   sb->last_sacked_bytes = 0;
   sb->last_bytes_delivered = 0;
@@ -759,6 +809,7 @@ scoreboard_hole_bytes (sack_scoreboard_hole_t * hole)
 always_inline u32
 scoreboard_hole_index (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
 {
+  ASSERT (!pool_is_free_index (sb->holes, hole - sb->holes));
   return hole - sb->holes;
 }
 
diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c
index 45db0da69c6..bc7d9015789 100644
--- a/src/vnet/tcp/tcp_input.c
+++ b/src/vnet/tcp/tcp_input.c
@@ -206,8 +206,8 @@ tcp_options_parse (tcp_header_t * th, tcp_options_t * to)
 	  vec_reset_length (to->sacks);
 	  for (j = 0; j < to->n_sack_blocks; j++)
 	    {
-	      b.start = clib_net_to_host_u32 (*(u32 *) (data + 2 + 4 * j));
-	      b.end = clib_net_to_host_u32 (*(u32 *) (data + 6 + 4 * j));
+	      b.start = clib_net_to_host_u32 (*(u32 *) (data + 2 + 8 * j));
+	      b.end = clib_net_to_host_u32 (*(u32 *) (data + 6 + 8 * j));
 	      vec_add1 (to->sacks, b);
 	    }
 	  break;
@@ -540,6 +540,10 @@ scoreboard_remove_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
   if (scoreboard_hole_index (sb, hole) == sb->cur_rxt_hole)
     sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
 
+  /* Poison the entry */
+  if (CLIB_DEBUG > 0)
+    memset (hole, 0xfe, sizeof (*hole));
+
   pool_put (sb->holes, hole);
 }
 
@@ -555,7 +559,7 @@ scoreboard_insert_hole (sack_scoreboard_t * sb, u32 prev_index,
 
   hole->start = start;
   hole->end = end;
-  hole_index = hole - sb->holes;
+  hole_index = scoreboard_hole_index (sb, hole);
 
   prev = scoreboard_get_hole (sb, prev_index);
   if (prev)
@@ -680,12 +684,30 @@ scoreboard_next_rxt_hole (sack_scoreboard_t * sb,
 }
 
 void
-scoreboard_init_high_rxt (sack_scoreboard_t * sb)
+scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 seq)
 {
   sack_scoreboard_hole_t *hole;
   hole = scoreboard_first_hole (sb);
-  sb->high_rxt = hole->start;
-  sb->cur_rxt_hole = sb->head;
+  if (hole)
+    {
+      seq = seq_gt (seq, hole->start) ? seq : hole->start;
+      sb->cur_rxt_hole = sb->head;
+    }
+  sb->high_rxt = seq;
+}
+
+/**
+ * Test that scoreboard is sane after recovery
+ *
+ * Returns 1 if scoreboard is empty or if first hole beyond
+ * snd_una.
+ */
+u8
+tcp_scoreboard_is_sane_post_recovery (tcp_connection_t * tc)
+{
+  sack_scoreboard_hole_t *hole;
+  hole = scoreboard_first_hole (&tc->sack_sb);
+  return (!hole || seq_geq (hole->start, tc->snd_una));
 }
 
 void
@@ -712,7 +734,7 @@ tcp_rcv_sacks (tcp_connection_t * tc, u32 ack)
     {
       if (seq_lt (blk->start, blk->end)
 	  && seq_gt (blk->start, tc->snd_una)
-	  && seq_gt (blk->start, ack) && seq_leq (blk->end, tc->snd_nxt))
+	  && seq_gt (blk->start, ack) && seq_leq (blk->end, tc->snd_una_max))
 	{
 	  blk++;
 	  continue;
@@ -731,6 +753,8 @@ tcp_rcv_sacks (tcp_connection_t * tc, u32 ack)
   if (vec_len (tc->rcv_opts.sacks) == 0)
     return;
 
+  tcp_scoreboard_trace_add (tc, ack);
+
   /* Make sure blocks are ordered */
   for (i = 0; i < vec_len (tc->rcv_opts.sacks); i++)
     for (j = i + 1; j < vec_len (tc->rcv_opts.sacks); j++)
@@ -797,7 +821,7 @@ tcp_rcv_sacks (tcp_connection_t * tc, u32 ack)
 		      sb->last_bytes_delivered +=
 			next_hole->start - hole->end;
 		    }
-		  else if (!next_hole)
+		  else
 		    {
 		      ASSERT (seq_geq (sb->high_sacked, ack));
 		      sb->snd_una_adv = sb->high_sacked - ack;
@@ -824,12 +848,14 @@ tcp_rcv_sacks (tcp_connection_t * tc, u32 ack)
 	  if (seq_lt (blk->end, hole->end))
 	    {
 	      hole_index = scoreboard_hole_index (sb, hole);
-	      scoreboard_insert_hole (sb, hole_index, blk->end, hole->end);
+	      next_hole = scoreboard_insert_hole (sb, hole_index, blk->end,
+						  hole->end);
 
 	      /* Pool might've moved */
 	      hole = scoreboard_get_hole (sb, hole_index);
 	      hole->end = blk->start;
 	      blk_index++;
+	      ASSERT (hole->next == scoreboard_hole_index (sb, next_hole));
 	    }
 	  else if (seq_lt (blk->start, hole->end))
 	    {
@@ -957,7 +983,7 @@ tcp_cc_recover (tcp_connection_t * tc)
 
   ASSERT (tc->rto_boff == 0);
   ASSERT (!tcp_in_cong_recovery (tc));
-
+  ASSERT (tcp_scoreboard_is_sane_post_recovery (tc));
   TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 3);
   return 0;
 }
@@ -965,7 +991,7 @@ tcp_cc_recover (tcp_connection_t * tc)
 static void
 tcp_cc_update (tcp_connection_t * tc, vlib_buffer_t * b)
 {
-  ASSERT (!tcp_in_cong_recovery (tc));
+  ASSERT (!tcp_in_cong_recovery (tc) || tcp_is_lost_fin (tc));
 
   /* Congestion avoidance */
   tc->cc_algo->rcv_ack (tc);
@@ -1064,10 +1090,10 @@ tcp_cc_handle_event (tcp_connection_t * tc, u32 is_dack)
 	  ASSERT (tc->cwnd >= tc->snd_mss);
 
 	  /* If cwnd allows, send more data */
-	  if (tcp_opts_sack_permitted (&tc->rcv_opts)
-	      && scoreboard_first_hole (&tc->sack_sb))
+	  if (tcp_opts_sack_permitted (&tc->rcv_opts))
 	    {
-	      scoreboard_init_high_rxt (&tc->sack_sb);
+	      scoreboard_init_high_rxt (&tc->sack_sb,
+					tc->snd_una + tc->snd_mss);
 	      tcp_fast_retransmit_sack (tc);
 	    }
 	  else
@@ -1134,12 +1160,13 @@ partial_ack:
   /* Remove retransmitted bytes that have been delivered */
   ASSERT (tc->bytes_acked + tc->sack_sb.snd_una_adv
 	  >= tc->sack_sb.last_bytes_delivered);
-  rxt_delivered = tc->bytes_acked + tc->sack_sb.snd_una_adv
-    - tc->sack_sb.last_bytes_delivered;
-  if (0 && rxt_delivered && seq_gt (tc->sack_sb.high_rxt, tc->snd_una))
+
+  if (seq_lt (tc->snd_una, tc->sack_sb.high_rxt))
     {
       /* If we have sacks and we haven't gotten an ack beyond high_rxt,
        * remove sacked bytes delivered */
+      rxt_delivered = tc->bytes_acked + tc->sack_sb.snd_una_adv
+	- tc->sack_sb.last_bytes_delivered;
       ASSERT (tc->snd_rxt_bytes >= rxt_delivered);
       tc->snd_rxt_bytes -= rxt_delivered;
     }
@@ -1256,6 +1283,18 @@ tcp_rcv_ack (tcp_connection_t * tc, vlib_buffer_t * b,
   return 0;
 }
 
+static u8
+tcp_sack_vector_is_sane (sack_block_t * sacks)
+{
+  int i;
+  for (i = 1; i < vec_len (sacks); i++)
+    {
+      if (sacks[i - 1].end == sacks[i].start)
+	return 0;
+    }
+  return 1;
+}
+
 /**
  * Build SACK list as per RFC2018.
  *
@@ -1316,6 +1355,9 @@ tcp_update_sack_list (tcp_connection_t * tc, u32 start, u32 end)
   /* Replace old vector with new one */
   vec_free (tc->snd_sacks);
   tc->snd_sacks = new_list;
+
+  /* Segments should not 'touch' */
+  ASSERT (tcp_sack_vector_is_sane (tc->snd_sacks));
 }
 
 /** Enqueue data for delivery to application */
@@ -1330,7 +1372,6 @@ tcp_session_enqueue_data (tcp_connection_t * tc, vlib_buffer_t * b,
   /* Pure ACK. Update rcv_nxt and be done. */
   if (PREDICT_FALSE (data_len == 0))
     {
-      tc->rcv_nxt = vnet_buffer (b)->tcp.seq_end;
       return TCP_ERROR_PURE_ACK;
     }
 
@@ -1385,7 +1426,7 @@ tcp_session_enqueue_ooo (tcp_connection_t * tc, vlib_buffer_t * b,
 			 u16 data_len)
 {
   stream_session_t *s0;
-  int rv;
+  int rv, offset;
 
   ASSERT (seq_gt (vnet_buffer (b)->tcp.seq_number, tc->rcv_nxt));
 
@@ -1421,12 +1462,12 @@ tcp_session_enqueue_ooo (tcp_connection_t * tc, vlib_buffer_t * b,
       newest = svm_fifo_newest_ooo_segment (s0->server_rx_fifo);
       if (newest)
 	{
-	  start =
-	    tc->rcv_nxt + ooo_segment_offset (s0->server_rx_fifo, newest);
+	  offset = ooo_segment_offset (s0->server_rx_fifo, newest);
+	  ASSERT (offset <= vnet_buffer (b)->tcp.seq_number - tc->rcv_nxt);
+	  start = tc->rcv_nxt + offset;
 	  end = start + ooo_segment_length (s0->server_rx_fifo, newest);
 	  tcp_update_sack_list (tc, start, end);
-
-	  ASSERT (seq_gt (start, tc->rcv_nxt));
+	  svm_fifo_newest_ooo_segment_reset (s0->server_rx_fifo);
 	}
     }
 
@@ -2736,12 +2777,12 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      /* lookup session */
 	      tc0 =
 		(tcp_connection_t *)
-		stream_session_lookup_transport4 (&ip40->dst_address,
-						  &ip40->src_address,
-						  tcp0->dst_port,
-						  tcp0->src_port,
-						  SESSION_TYPE_IP4_TCP,
-						  my_thread_index);
+		stream_session_lookup_transport_wt4 (&ip40->dst_address,
+						     &ip40->src_address,
+						     tcp0->dst_port,
+						     tcp0->src_port,
+						     SESSION_TYPE_IP4_TCP,
+						     my_thread_index);
 	    }
 	  else
 	    {
@@ -2754,12 +2795,12 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
 	      tc0 =
 		(tcp_connection_t *)
-		stream_session_lookup_transport6 (&ip60->src_address,
-						  &ip60->dst_address,
-						  tcp0->src_port,
-						  tcp0->dst_port,
-						  SESSION_TYPE_IP6_TCP,
-						  my_thread_index);
+		stream_session_lookup_transport_wt6 (&ip60->src_address,
+						     &ip60->dst_address,
+						     tcp0->src_port,
+						     tcp0->dst_port,
+						     SESSION_TYPE_IP6_TCP,
+						     my_thread_index);
 	    }
 
 	  /* Length check */
@@ -2931,6 +2972,8 @@ do {                                                       	\
   _(ESTABLISHED, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_ESTABLISHED,
     TCP_ERROR_NONE);
   _(ESTABLISHED, TCP_FLAG_SYN, TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
+  _(ESTABLISHED, TCP_FLAG_SYN | TCP_FLAG_ACK, TCP_INPUT_NEXT_ESTABLISHED,
+    TCP_ERROR_NONE);
   /* ACK or FIN-ACK to our FIN */
   _(FIN_WAIT_1, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
   _(FIN_WAIT_1, TCP_FLAG_ACK | TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS,
@@ -2954,6 +2997,7 @@ do {                                                       	\
   _(TIME_WAIT, TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
   _(TIME_WAIT, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
     TCP_ERROR_NONE);
+  _(TIME_WAIT, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
   _(CLOSED, TCP_FLAG_ACK, TCP_INPUT_NEXT_RESET, TCP_ERROR_CONNECTION_CLOSED);
   _(CLOSED, TCP_FLAG_RST, TCP_INPUT_NEXT_DROP, TCP_ERROR_CONNECTION_CLOSED);
   _(CLOSED, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RESET,
diff --git a/src/vnet/tcp/tcp_test.c b/src/vnet/tcp/tcp_test.c
index 510deb4fec0..f37ba96dab8 100644
--- a/src/vnet/tcp/tcp_test.c
+++ b/src/vnet/tcp/tcp_test.c
@@ -34,6 +34,38 @@
     }								\
 }
 
+/* *INDENT-OFF* */
+scoreboard_trace_elt_t sb_trace[] = {};
+/* *INDENT-ON* */
+
+static int
+tcp_test_scoreboard_replay (vlib_main_t * vm, unformat_input_t * input)
+{
+  int verbose = 0;
+  tcp_connection_t _tc, *tc = &_tc;
+  u8 *s = 0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "detail"))
+	verbose = 1;
+      else
+	{
+	  clib_error_t *e = clib_error_return
+	    (0, "unknown input `%U'", format_unformat_error, input);
+	  clib_error_report (e);
+	  return -1;
+	}
+    }
+
+#if TCP_SCOREBOARD_TRACE
+  tc->sack_sb.trace = sb_trace;
+#endif
+  s = tcp_scoreboard_replay (s, tc, verbose);
+  vlib_cli_output (vm, "%v", s);
+  return 0;
+}
+
 static int
 tcp_test_sack_rx (vlib_main_t * vm, unformat_input_t * input)
 {
@@ -47,6 +79,8 @@ tcp_test_sack_rx (vlib_main_t * vm, unformat_input_t * input)
     {
       if (unformat (input, "verbose"))
 	verbose = 1;
+      else if (unformat (input, "replay"))
+	return tcp_test_scoreboard_replay (vm, input);
     }
 
   memset (tc, 0, sizeof (*tc));
@@ -282,6 +316,44 @@ tcp_test_sack_rx (vlib_main_t * vm, unformat_input_t * input)
   TCP_TEST ((sb->last_bytes_delivered == 400),
 	    "last bytes delivered %d", sb->last_bytes_delivered);
 
+  /*
+   * One hole close to head, patch head, split in two and start acking
+   * the lowest part
+   */
+  scoreboard_clear (sb);
+  tc->snd_una = 0;
+  tc->snd_una_max = 1000;
+  tc->snd_nxt = 1000;
+
+  block.start = 500;
+  block.end = 1000;
+  vec_add1 (tc->rcv_opts.sacks, block);
+  tc->rcv_opts.n_sack_blocks = vec_len (tc->rcv_opts.sacks);
+
+  tcp_rcv_sacks (tc, 0);
+  if (verbose)
+    vlib_cli_output (vm, "sb added [500, 1000]:\n%U",
+		     format_tcp_scoreboard, sb);
+
+  vec_reset_length (tc->rcv_opts.sacks);
+  block.start = 300;
+  block.end = 400;
+  vec_add1 (tc->rcv_opts.sacks, block);
+  tc->rcv_opts.n_sack_blocks = vec_len (tc->rcv_opts.sacks);
+  tcp_rcv_sacks (tc, 100);
+  if (verbose)
+    vlib_cli_output (vm, "sb added [0, 100] [300, 400]:\n%U",
+		     format_tcp_scoreboard, sb);
+  TCP_TEST ((pool_elts (sb->holes) == 2),
+	    "scoreboard has %d elements", pool_elts (sb->holes));
+
+  tc->snd_una = 100;
+  tcp_rcv_sacks (tc, 200);
+  tcp_rcv_sacks (tc, 300);
+  if (verbose)
+    vlib_cli_output (vm, "sb added [0, 300]:\n%U", format_tcp_scoreboard, sb);
+  TCP_TEST ((sb->sacked_bytes == 500), "sacked bytes %d", sb->sacked_bytes);
+
   return 0;
 }
 
@@ -390,6 +462,37 @@ tcp_test_sack_tx (vlib_main_t * vm, unformat_input_t * input)
     vlib_cli_output (vm, "advance rcv_nxt to 1200\n%U", format_tcp_sacks, tc);
   TCP_TEST ((vec_len (tc->snd_sacks) == 0), "sack blocks %d expected %d",
 	    vec_len (tc->snd_sacks), 0);
+
+
+  /*
+   * Add 2 blocks, overwrite first and update rcv_nxt to also remove it
+   */
+
+  vec_reset_length (tc->snd_sacks);
+  tc->rcv_nxt = 0;
+
+  tcp_update_sack_list (tc, 100, 200);
+  tcp_update_sack_list (tc, 300, 400);
+
+  if (verbose)
+    vlib_cli_output (vm, "add [100, 200] [300, 400]\n%U",
+		     format_tcp_sacks, tc);
+  TCP_TEST ((vec_len (tc->snd_sacks) == 2),
+	    "sack blocks %d expected %d", vec_len (tc->snd_sacks), 2);
+  TCP_TEST ((tc->snd_sacks[0].start == 300),
+	    "first sack block start %u expected %u", tc->snd_sacks[0].start,
+	    300);
+
+  tc->rcv_nxt = 100;
+  tcp_update_sack_list (tc, 100, 100);
+  if (verbose)
+    vlib_cli_output (vm, "add [100, 200] rcv_nxt = 100\n%U",
+		     format_tcp_sacks, tc);
+  TCP_TEST ((vec_len (tc->snd_sacks) == 1),
+	    "sack blocks %d expected %d", vec_len (tc->snd_sacks), 1);
+  TCP_TEST ((tc->snd_sacks[0].start == 300),
+	    "first sack block start %u expected %u", tc->snd_sacks[0].start,
+	    300);
   return 0;
 }
 
@@ -1188,6 +1291,176 @@ tcp_test_fifo4 (vlib_main_t * vm, unformat_input_t * input)
   return 0;
 }
 
+static u32
+fifo_pos (svm_fifo_t * f, u32 pos)
+{
+  return pos % f->nitems;
+}
+
+static int
+tcp_test_fifo5 (vlib_main_t * vm, unformat_input_t * input)
+{
+  svm_fifo_t *f;
+  u32 fifo_size = 400, j = 0, offset = 200;
+  int i, rv, verbose = 0;
+  u8 *test_data = 0, *data_buf = 0;
+  ooo_segment_t *ooo_seg;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "verbose"))
+	verbose = 1;
+      else
+	{
+	  clib_error_t *e = clib_error_return
+	    (0, "unknown input `%U'", format_unformat_error, input);
+	  clib_error_report (e);
+	  return -1;
+	}
+    }
+
+  f = fifo_prepare (fifo_size);
+  svm_fifo_init_pointers (f, offset);
+
+  vec_validate (test_data, 399);
+  for (i = 0; i < vec_len (test_data); i++)
+    test_data[i] = i % 0xff;
+
+  /*
+   * Start with [100, 200] and [300, 400]
+   */
+  svm_fifo_enqueue_with_offset (f, 100, 100, &test_data[100]);
+  svm_fifo_enqueue_with_offset (f, 300, 100, &test_data[300]);
+
+  TCP_TEST ((svm_fifo_number_ooo_segments (f) == 2),
+	    "number of ooo segments %u", svm_fifo_number_ooo_segments (f));
+  TCP_TEST ((f->ooos_newest == 1), "newest %u", f->ooos_newest);
+  if (verbose)
+    vlib_cli_output (vm, "fifo after [100, 200] and [300, 400] : %U",
+		     format_svm_fifo, f, 2 /* verbose */ );
+
+  /*
+   * Add [225, 275]
+   */
+
+  rv = svm_fifo_enqueue_with_offset (f, 225, 50, &test_data[200]);
+  if (verbose)
+    vlib_cli_output (vm, "fifo after [225, 275] : %U",
+		     format_svm_fifo, f, 2 /* verbose */ );
+  TCP_TEST ((svm_fifo_number_ooo_segments (f) == 3),
+	    "number of ooo segments %u", svm_fifo_number_ooo_segments (f));
+  ooo_seg = svm_fifo_first_ooo_segment (f);
+  TCP_TEST ((ooo_seg->start == fifo_pos (f, 100 + offset)),
+	    "first seg start %u expected %u", ooo_seg->start,
+	    fifo_pos (f, 100 + offset));
+  TCP_TEST ((ooo_seg->length == 100), "first seg length %u expected %u",
+	    ooo_seg->length, 100);
+  ooo_seg = ooo_segment_next (f, ooo_seg);
+  TCP_TEST ((ooo_seg->start == fifo_pos (f, 225 + offset)),
+	    "second seg start %u expected %u",
+	    ooo_seg->start, fifo_pos (f, 225 + offset));
+  TCP_TEST ((ooo_seg->length == 50), "second seg length %u expected %u",
+	    ooo_seg->length, 50);
+  ooo_seg = ooo_segment_next (f, ooo_seg);
+  TCP_TEST ((ooo_seg->start == fifo_pos (f, 300 + offset)),
+	    "third seg start %u expected %u",
+	    ooo_seg->start, fifo_pos (f, 300 + offset));
+  TCP_TEST ((ooo_seg->length == 100), "third seg length %u expected %u",
+	    ooo_seg->length, 100);
+  TCP_TEST ((f->ooos_newest == 2), "newest %u", f->ooos_newest);
+  /*
+   * Add [190, 310]
+   */
+  rv = svm_fifo_enqueue_with_offset (f, 190, 120, &test_data[190]);
+  if (verbose)
+    vlib_cli_output (vm, "fifo after [190, 310] : %U",
+		     format_svm_fifo, f, 1 /* verbose */ );
+  TCP_TEST ((svm_fifo_number_ooo_segments (f) == 1),
+	    "number of ooo segments %u", svm_fifo_number_ooo_segments (f));
+  ooo_seg = svm_fifo_first_ooo_segment (f);
+  TCP_TEST ((ooo_seg->start == fifo_pos (f, offset + 100)),
+	    "first seg start %u expected %u",
+	    ooo_seg->start, fifo_pos (f, offset + 100));
+  TCP_TEST ((ooo_seg->length == 300), "first seg length %u expected %u",
+	    ooo_seg->length, 300);
+
+  /*
+   * Add [0, 150]
+   */
+  rv = svm_fifo_enqueue_nowait (f, 150, test_data);
+
+  if (verbose)
+    vlib_cli_output (vm, "fifo after [0 150] : %U", format_svm_fifo, f,
+		     2 /* verbose */ );
+
+  TCP_TEST ((rv == 400), "managed to enqueue %u expected %u", rv, 400);
+  TCP_TEST ((svm_fifo_number_ooo_segments (f) == 0),
+	    "number of ooo segments %u", svm_fifo_number_ooo_segments (f));
+
+  vec_validate (data_buf, 399);
+  svm_fifo_peek (f, 0, 400, data_buf);
+  if (compare_data (data_buf, test_data, 0, 400, &j))
+    {
+      TCP_TEST (0, "[%d] peeked %u expected %u", j, data_buf[j],
+		test_data[j]);
+    }
+
+  /*
+   * Add [100 200] and overlap it with [50 250]
+   */
+  svm_fifo_free (f);
+  f = fifo_prepare (fifo_size);
+
+  svm_fifo_enqueue_with_offset (f, 100, 100, &test_data[100]);
+  svm_fifo_enqueue_with_offset (f, 50, 200, &test_data[50]);
+  TCP_TEST ((svm_fifo_number_ooo_segments (f) == 1),
+	    "number of ooo segments %u", svm_fifo_number_ooo_segments (f));
+  ooo_seg = svm_fifo_first_ooo_segment (f);
+  TCP_TEST ((ooo_seg->start == 50), "first seg start %u expected %u",
+	    ooo_seg->start, 50);
+  TCP_TEST ((ooo_seg->length == 200), "first seg length %u expected %u",
+	    ooo_seg->length, 200);
+
+  svm_fifo_free (f);
+  vec_free (test_data);
+  return 0;
+}
+
+/* *INDENT-OFF* */
+svm_fifo_trace_elem_t fifo_trace[] = {};
+/* *INDENT-ON* */
+
+static int
+tcp_test_fifo_replay (vlib_main_t * vm, unformat_input_t * input)
+{
+  svm_fifo_t f;
+  int verbose = 0;
+  u8 no_read = 0, *str = 0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "verbose"))
+	verbose = 1;
+      else if (unformat (input, "no-read"))
+	no_read = 1;
+      else
+	{
+	  clib_error_t *e = clib_error_return
+	    (0, "unknown input `%U'", format_unformat_error, input);
+	  clib_error_report (e);
+	  return -1;
+	}
+    }
+
+#if SVMF_FIFO_TRACE
+  f.trace = fifo_trace;
+#endif
+
+  str = svm_fifo_replay (str, &f, no_read, verbose);
+  vlib_cli_output (vm, "%v", str);
+  return 0;
+}
+
 static int
 tcp_test_fifo (vlib_main_t * vm, unformat_input_t * input)
 {
@@ -1237,6 +1510,14 @@ tcp_test_fifo (vlib_main_t * vm, unformat_input_t * input)
       if (tcp_test_fifo3 (vm, input))
 	return -1;
       unformat_free (input);
+
+      res = tcp_test_fifo4 (vm, input);
+      if (res)
+	return res;
+
+      res = tcp_test_fifo5 (vm, input);
+      if (res)
+	return res;
     }
   else
     {
@@ -1256,6 +1537,14 @@ tcp_test_fifo (vlib_main_t * vm, unformat_input_t * input)
 	{
 	  res = tcp_test_fifo4 (vm, input);
 	}
+      else if (unformat (input, "fifo5"))
+	{
+	  res = tcp_test_fifo5 (vm, input);
+	}
+      else if (unformat (input, "replay"))
+	{
+	  res = tcp_test_fifo_replay (vm, input);
+	}
     }
 
   return res;