Fixes and improved tcp/session debugging

- Fix rx sack option parsing
- Add session sack scoreboard tracing and replaying
- Add svm fifo tracing and replaying
- Scoreboard/svm fifo ooo segment reception fixes
- Improved overall debugging

Change-Id: Ieae07eba355e66f5935253232bb00f2dfb7ece00
Signed-off-by: Florin Coras <fcoras@cisco.com>

11 files changed, 1111 insertions, 131 deletions

diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index da60fee51ec..fc2189c5176 100644
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -61,6 +61,90 @@ format_ooo_segment (u8 * s, va_list * args)
 }
 
 u8 *
+svm_fifo_dump_trace (u8 * s, svm_fifo_t * f)
+{
+#if SVM_FIFO_TRACE
+  svm_fifo_trace_elem_t *seg = 0;
+  int i = 0;
+
+  if (f->trace)
+    {
+      vec_foreach (seg, f->trace)
+      {
+	s = format (s, "{%u, %u, %u}, ", seg->offset, seg->len, seg->action);
+	i++;
+	if (i % 5 == 0)
+	  s = format (s, "\n");
+      }
+      s = format (s, "\n");
+    }
+  return s;
+#else
+  return 0;
+#endif
+}
+
+u8 *
+svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose)
+{
+  int i, trace_len;
+  u8 *data = 0;
+  svm_fifo_trace_elem_t *trace;
+  u32 offset;
+  svm_fifo_t *dummy_fifo;
+
+  if (!f)
+    return s;
+
+#if SVM_FIFO_TRACE
+  trace = f->trace;
+  trace_len = vec_len (trace);
+#else
+  trace = 0;
+  trace_len = 0;
+#endif
+
+  dummy_fifo = svm_fifo_create (f->nitems);
+  memset (f->data, 0xFF, f->nitems);
+
+  vec_validate (data, f->nitems);
+  for (i = 0; i < vec_len (data); i++)
+    data[i] = i;
+
+  for (i = 0; i < trace_len; i++)
+    {
+      offset = trace[i].offset;
+      if (trace[i].action == 1)
+	{
+	  if (verbose)
+	    s = format (s, "adding [%u, %u]:", trace[i].offset,
+			(trace[i].offset +
+			 trace[i].len) % dummy_fifo->nitems);
+	  svm_fifo_enqueue_with_offset (dummy_fifo, trace[i].offset,
+					trace[i].len, &data[offset]);
+	}
+      else if (trace[i].action == 2)
+	{
+	  if (verbose)
+	    s = format (s, "adding [%u, %u]:", 0, trace[i].len);
+	  svm_fifo_enqueue_nowait (dummy_fifo, trace[i].len, &data[offset]);
+	}
+      else if (!no_read)
+	{
+	  if (verbose)
+	    s = format (s, "read: %u", trace[i].len);
+	  svm_fifo_dequeue_drop (dummy_fifo, trace[i].len);
+	}
+      if (verbose)
+	s = format (s, "%U", format_svm_fifo, dummy_fifo, 1);
+    }
+
+  s = format (s, "result: %U", format_svm_fifo, dummy_fifo, 1);
+
+  return s;
+}
+
+u8 *
 format_ooo_list (u8 * s, va_list * args)
 {
   svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
@@ -73,6 +157,7 @@ format_ooo_list (u8 * s, va_list * args)
       s = format (s, "  %U\n", format_ooo_segment, seg);
       ooo_segment_index = seg->next;
     }
+
   return s;
 }
 
@@ -94,10 +179,10 @@ format_svm_fifo (u8 * s, va_list * args)
 
   if (verbose)
     {
-      s = format (s, " ooo pool %d active elts\n",
-		  pool_elts (f->ooo_segments));
+      s = format (s, " ooo pool %d active elts newest %u\n",
+		  pool_elts (f->ooo_segments), f->ooos_newest);
       if (svm_fifo_has_ooo_data (f))
-	s = format (s, " %U", format_ooo_list, f);
+	s = format (s, " %U", format_ooo_list, f, verbose);
     }
   return s;
 }
@@ -116,7 +201,6 @@ svm_fifo_create (u32 data_size_in_bytes)
   memset (f, 0, sizeof (*f));
   f->nitems = data_size_in_bytes;
   f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
-
   return (f);
 }
 
@@ -178,6 +262,7 @@ ooo_segment_add (svm_fifo_t * f, u32 offset, u32 length)
   u32 new_index, s_end_pos, s_index;
   u32 normalized_position, normalized_end_position;
 
+  ASSERT (offset + length <= ooo_segment_distance_from_tail (f, f->head));
   normalized_position = (f->tail + offset) % f->nitems;
   normalized_end_position = (f->tail + offset + length) % f->nitems;
 
@@ -205,17 +290,9 @@ ooo_segment_add (svm_fifo_t * f, u32 offset, u32 length)
       s = prev;
       s_end_pos = ooo_segment_end_pos (f, s);
 
-      /* Check head and tail now since segment may be wider at both ends so
-       * merge tests lower won't work */
-      if (position_lt (f, normalized_position, s->start))
-	{
-	  s->start = normalized_position;
-	  s->length = position_diff (f, s_end_pos, s->start);
-	}
-      if (position_gt (f, normalized_end_position, s_end_pos))
-	{
-	  s->length = position_diff (f, normalized_end_position, s->start);
-	}
+      /* Since we have previous, normalized start position cannot be smaller
+       * than prev->start. Check tail */
+      ASSERT (position_lt (f, s->start, normalized_position));
       goto check_tail;
     }
 
@@ -256,6 +333,7 @@ ooo_segment_add (svm_fifo_t * f, u32 offset, u32 length)
       /* Pool might've moved, get segment again */
       s = pool_elt_at_index (f->ooo_segments, s_index);
 
+      /* Needs to be last */
       ASSERT (s->next == OOO_SEGMENT_INVALID_INDEX);
 
       new_s->prev = s_index;
@@ -274,32 +352,22 @@ ooo_segment_add (svm_fifo_t * f, u32 offset, u32 length)
     {
       s->start = normalized_position;
       s->length = position_diff (f, s_end_pos, s->start);
-    }
-  /* Overlapping tail */
-  else if (position_gt (f, normalized_end_position, s_end_pos))
-    {
-      s->length = position_diff (f, normalized_end_position, s->start);
-    }
-  /* New segment completely covered by current one */
-  else
-    {
-      /* Do Nothing */
-      s = 0;
-      goto done;
+      f->ooos_newest = s - f->ooo_segments;
     }
 
 check_tail:
-  /* The new segment's tail may cover multiple smaller ones */
+
+  /* Overlapping tail */
   if (position_gt (f, normalized_end_position, s_end_pos))
     {
-      /* Remove the completely overlapped segments */
-      it = (s->next != OOO_SEGMENT_INVALID_INDEX) ?
-	pool_elt_at_index (f->ooo_segments, s->next) : 0;
+      s->length = position_diff (f, normalized_end_position, s->start);
+
+      /* Remove the completely overlapped segments in the tail */
+      it = ooo_segment_next (f, s);
       while (it && position_leq (f, ooo_segment_end_pos (f, it),
 				 normalized_end_position))
 	{
-	  next = (it->next != OOO_SEGMENT_INVALID_INDEX) ?
-	    pool_elt_at_index (f->ooo_segments, it->next) : 0;
+	  next = ooo_segment_next (f, it);
 	  ooo_segment_del (f, it - f->ooo_segments);
 	  it = next;
 	}
@@ -307,16 +375,12 @@ check_tail:
       /* If partial overlap with last, merge */
       if (it && position_leq (f, it->start, normalized_end_position))
 	{
-	  s->length =
-	    position_diff (f, ooo_segment_end_pos (f, it), s->start);
+	  s->length = position_diff (f, ooo_segment_end_pos (f, it),
+				     s->start);
 	  ooo_segment_del (f, it - f->ooo_segments);
 	}
+      f->ooos_newest = s - f->ooo_segments;
     }
-
-done:
-  /* Most recently updated segment */
-  if (s)
-    f->ooos_newest = s - f->ooo_segments;
 }
 
 /**
@@ -422,6 +486,8 @@ svm_fifo_enqueue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_from_here)
       total_copy_bytes = max_bytes;
     }
 
+  svm_fifo_trace_add (f, f->head, total_copy_bytes, 2);
+
   /* Any out-of-order segments to collect? */
   if (PREDICT_FALSE (f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX))
     total_copy_bytes += ooo_segment_try_collect (f, total_copy_bytes);
@@ -499,6 +565,8 @@ svm_fifo_enqueue_with_offset_internal (svm_fifo_t * f,
   if ((required_bytes + offset_from_tail) > (nitems - cursize))
     return -1;
 
+  svm_fifo_trace_add (f, offset, required_bytes, 1);
+
   ooo_segment_add (f, offset, required_bytes);
 
   /* Number of bytes we're going to copy */
@@ -707,6 +775,8 @@ svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes)
   /* Number of bytes we're going to drop */
   total_drop_bytes = (cursize < max_bytes) ? cursize : max_bytes;
 
+  svm_fifo_trace_add (f, f->tail, total_drop_bytes, 3);
+
   /* Number of bytes in first copy segment */
   first_drop_bytes =
     ((nitems - f->head) < total_drop_bytes) ?
diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h
index fe21de476d7..a83cd858f83 100644
--- a/src/svm/svm_fifo.h
+++ b/src/svm/svm_fifo.h
@@ -36,8 +36,16 @@ typedef struct
 format_function_t format_ooo_segment;
 format_function_t format_ooo_list;
 
+#define SVM_FIFO_TRACE (0)
 #define OOO_SEGMENT_INVALID_INDEX ((u32)~0)
 
+typedef struct
+{
+  u32 offset;
+  u32 len;
+  u32 action;
+} svm_fifo_trace_elem_t;
+
 typedef struct _svm_fifo
 {
   volatile u32 cursize;		/**< current fifo size */
@@ -64,9 +72,28 @@ typedef struct _svm_fifo
   u32 ooos_newest;		/**< Last segment to have been updated */
   struct _svm_fifo *next;	/**< next in freelist/active chain */
   struct _svm_fifo *prev;	/**< prev in active chain */
+#if SVM_FIFO_TRACE
+  svm_fifo_trace_elem_t *trace;
+#endif
     CLIB_CACHE_LINE_ALIGN_MARK (data);
 } svm_fifo_t;
 
+#if SVM_FIFO_TRACE
+#define svm_fifo_trace_add(_f, _s, _l, _t)		\
+{							\
+  svm_fifo_trace_elem_t *trace_elt;			\
+  vec_add2(_f->trace, trace_elt, 1);			\
+  trace_elt->offset = _s;				\
+  trace_elt->len = _l;					\
+  trace_elt->action = _t;				\
+}
+#else
+#define svm_fifo_trace_add(_f, _s, _l, _t)
+#endif
+
+u8 *svm_fifo_dump_trace (u8 * s, svm_fifo_t * f);
+u8 *svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose);
+
 static inline u32
 svm_fifo_max_dequeue (svm_fifo_t * f)
 {
@@ -132,6 +159,12 @@ svm_fifo_newest_ooo_segment (svm_fifo_t * f)
   return pool_elt_at_index (f->ooo_segments, f->ooos_newest);
 }
 
+always_inline void
+svm_fifo_newest_ooo_segment_reset (svm_fifo_t * f)
+{
+  f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
+}
+
 always_inline u32
 ooo_segment_distance_from_tail (svm_fifo_t * f, u32 pos)
 {
@@ -174,6 +207,14 @@ ooo_segment_get_prev (svm_fifo_t * f, ooo_segment_t * s)
   return pool_elt_at_index (f->ooo_segments, s->prev);
 }
 
+always_inline ooo_segment_t *
+ooo_segment_next (svm_fifo_t * f, ooo_segment_t * s)
+{
+  if (s->next == OOO_SEGMENT_INVALID_INDEX)
+    return 0;
+  return pool_elt_at_index (f->ooo_segments, s->next);
+}
+
 #endif /* __included_ssvm_fifo_h__ */
 
 /*
diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c
index 0a86d563f0e..2c2a27c1af9 100644
--- a/src/vnet/session/session.c
+++ b/src/vnet/session/session.c
@@ -325,9 +325,9 @@ stream_session_half_open_lookup (session_manager_main_t * smm,
 }
 
 transport_connection_t *
-stream_session_lookup_transport4 (ip4_address_t * lcl, ip4_address_t * rmt,
-				  u16 lcl_port, u16 rmt_port, u8 proto,
-				  u32 my_thread_index)
+stream_session_lookup_transport_wt4 (ip4_address_t * lcl, ip4_address_t * rmt,
+				     u16 lcl_port, u16 rmt_port, u8 proto,
+				     u32 my_thread_index)
 {
   session_manager_main_t *smm = &session_manager_main;
   session_kv4_t kv4;
@@ -358,9 +358,40 @@ stream_session_lookup_transport4 (ip4_address_t * lcl, ip4_address_t * rmt,
 }
 
 transport_connection_t *
-stream_session_lookup_transport6 (ip6_address_t * lcl, ip6_address_t * rmt,
-				  u16 lcl_port, u16 rmt_port, u8 proto,
-				  u32 my_thread_index)
+stream_session_lookup_transport4 (ip4_address_t * lcl, ip4_address_t * rmt,
+				  u16 lcl_port, u16 rmt_port, u8 proto)
+{
+  session_manager_main_t *smm = &session_manager_main;
+  session_kv4_t kv4;
+  stream_session_t *s;
+  int rv;
+
+  /* Lookup session amongst established ones */
+  make_v4_ss_kv (&kv4, lcl, rmt, lcl_port, rmt_port, proto);
+  rv = clib_bihash_search_inline_16_8 (&smm->v4_session_hash, &kv4);
+  if (rv == 0)
+    {
+      s = stream_session_get_from_handle (kv4.value);
+      return tp_vfts[s->session_type].get_connection (s->connection_index,
+						      s->thread_index);
+    }
+
+  /* If nothing is found, check if any listener is available */
+  s = stream_session_lookup_listener4 (lcl, lcl_port, proto);
+  if (s)
+    return tp_vfts[s->session_type].get_listener (s->connection_index);
+
+  /* Finally, try half-open connections */
+  rv = clib_bihash_search_inline_16_8 (&smm->v4_half_open_hash, &kv4);
+  if (rv == 0)
+    return tp_vfts[proto].get_half_open (kv4.value & 0xFFFFFFFF);
+  return 0;
+}
+
+transport_connection_t *
+stream_session_lookup_transport_wt6 (ip6_address_t * lcl, ip6_address_t * rmt,
+				     u16 lcl_port, u16 rmt_port, u8 proto,
+				     u32 my_thread_index)
 {
   session_manager_main_t *smm = &session_manager_main;
   stream_session_t *s;
@@ -390,6 +421,37 @@ stream_session_lookup_transport6 (ip6_address_t * lcl, ip6_address_t * rmt,
   return 0;
 }
 
+transport_connection_t *
+stream_session_lookup_transport6 (ip6_address_t * lcl, ip6_address_t * rmt,
+				  u16 lcl_port, u16 rmt_port, u8 proto)
+{
+  session_manager_main_t *smm = &session_manager_main;
+  stream_session_t *s;
+  session_kv6_t kv6;
+  int rv;
+
+  make_v6_ss_kv (&kv6, lcl, rmt, lcl_port, rmt_port, proto);
+  rv = clib_bihash_search_inline_48_8 (&smm->v6_session_hash, &kv6);
+  if (rv == 0)
+    {
+      s = stream_session_get_from_handle (kv6.value);
+      return tp_vfts[s->session_type].get_connection (s->connection_index,
+						      s->thread_index);
+    }
+
+  /* If nothing is found, check if any listener is available */
+  s = stream_session_lookup_listener6 (lcl, lcl_port, proto);
+  if (s)
+    return tp_vfts[s->session_type].get_listener (s->connection_index);
+
+  /* Finally, try half-open connections */
+  rv = clib_bihash_search_inline_48_8 (&smm->v6_half_open_hash, &kv6);
+  if (rv == 0)
+    return tp_vfts[proto].get_half_open (kv6.value & 0xFFFFFFFF);
+
+  return 0;
+}
+
 int
 stream_session_create_i (segment_manager_t * sm, transport_connection_t * tc,
 			 stream_session_t ** ret_s)
diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h
index b4507d4eaed..6069c5740f7 100644
--- a/src/vnet/session/session.h
+++ b/src/vnet/session/session.h
@@ -263,15 +263,30 @@ stream_session_t *stream_session_lookup6 (ip6_address_t * lcl,
 					  ip6_address_t * rmt, u16 lcl_port,
 					  u16 rmt_port, u8 proto);
 transport_connection_t
-  * stream_session_lookup_transport4 (ip4_address_t * lcl,
-				      ip4_address_t * rmt, u16 lcl_port,
-				      u16 rmt_port, u8 proto,
-				      u32 thread_index);
-transport_connection_t
-  * stream_session_lookup_transport6 (ip6_address_t * lcl,
-				      ip6_address_t * rmt, u16 lcl_port,
-				      u16 rmt_port, u8 proto,
-				      u32 thread_index);
+  * stream_session_lookup_transport_wt4 (ip4_address_t * lcl,
+					 ip4_address_t * rmt, u16 lcl_port,
+					 u16 rmt_port, u8 proto,
+					 u32 thread_index);
+transport_connection_t *stream_session_lookup_transport4 (ip4_address_t * lcl,
+							  ip4_address_t * rmt,
+							  u16 lcl_port,
+							  u16 rmt_port,
+							  u8 proto);
+transport_connection_t *stream_session_lookup_transport_wt6 (ip6_address_t *
+							     lcl,
+							     ip6_address_t *
+							     rmt,
+							     u16 lcl_port,
+							     u16 rmt_port,
+							     u8 proto,
+							     u32
+							     thread_index);
+transport_connection_t *stream_session_lookup_transport6 (ip6_address_t * lcl,
+							  ip6_address_t * rmt,
+							  u16 lcl_port,
+							  u16 rmt_port,
+							  u8 proto);
+
 stream_session_t *stream_session_lookup_listener (ip46_address_t * lcl,
 						  u16 lcl_port, u8 proto);
 void stream_session_table_add_for_tc (transport_connection_t * tc, u64 value);
@@ -415,7 +430,12 @@ void stream_session_cleanup (stream_session_t * s);
 void session_send_session_evt_to_thread (u64 session_handle,
 					 fifo_event_type_t evt_type,
 					 u32 thread_index);
+
 u8 *format_stream_session (u8 * s, va_list * args);
+uword unformat_stream_session (unformat_input_t * input, va_list * args);
+uword unformat_transport_connection (unformat_input_t * input,
+				     va_list * args);
+
 int
 send_session_connected_callback (u32 app_index, u32 api_context,
 				 stream_session_t * s, u8 is_fail);
diff --git a/src/vnet/session/session_cli.c b/src/vnet/session/session_cli.c
index e06bc586be3..e8e6f99c742 100755
--- a/src/vnet/session/session_cli.c
+++ b/src/vnet/session/session_cli.c
@@ -81,12 +81,141 @@ format_stream_session (u8 * s, va_list * args)
     {
       clib_warning ("Session in state: %d!", ss->session_state);
     }
-
   vec_free (str);
 
   return s;
 }
 
+uword
+unformat_stream_session_id (unformat_input_t * input, va_list * args)
+{
+  u8 *proto = va_arg (*args, u8 *);
+  ip46_address_t *lcl = va_arg (*args, ip46_address_t *);
+  ip46_address_t *rmt = va_arg (*args, ip46_address_t *);
+  u16 *lcl_port = va_arg (*args, u16 *);
+  u16 *rmt_port = va_arg (*args, u16 *);
+  u8 *is_ip4 = va_arg (*args, u8 *);
+  u8 tuple_is_set = 0;
+
+  memset (lcl, 0, sizeof (*lcl));
+  memset (rmt, 0, sizeof (*rmt));
+
+  if (unformat (input, "tcp"))
+    {
+      *proto = TRANSPORT_PROTO_TCP;
+    }
+  if (unformat (input, "udp"))
+    {
+      *proto = TRANSPORT_PROTO_UDP;
+    }
+  else if (unformat (input, "%U:%d->%U:%d", unformat_ip4_address, &lcl->ip4,
+		     lcl_port, unformat_ip4_address, &rmt->ip4, rmt_port))
+    {
+      *is_ip4 = 1;
+      tuple_is_set = 1;
+    }
+  else if (unformat (input, "%U:%d->%U:%d", unformat_ip6_address, &lcl->ip6,
+		     lcl_port, unformat_ip6_address, &rmt->ip6, rmt_port))
+    {
+      *is_ip4 = 0;
+      tuple_is_set = 1;
+    }
+  else
+    return 0;
+
+  if (tuple_is_set)
+    return 1;
+
+  return 0;
+}
+
+uword
+unformat_stream_session (unformat_input_t * input, va_list * args)
+{
+  stream_session_t **result = va_arg (*args, stream_session_t **);
+  stream_session_t *s;
+  u8 proto = ~0;
+  ip46_address_t lcl, rmt;
+  u32 lcl_port = 0, rmt_port = 0;
+  u8 is_ip4 = 0, s_type = ~0, id_is_set = 0;
+
+  if (unformat (input, "%U", unformat_stream_session_id, &proto, &lcl, &rmt,
+		&lcl_port, &rmt_port, &is_ip4))
+    {
+      id_is_set = 1;
+    }
+  else
+    return 0;
+
+  if (!id_is_set)
+    {
+      return 0;
+    }
+
+  s_type = session_type_from_proto_and_ip (proto, is_ip4);
+  if (is_ip4)
+    s = stream_session_lookup4 (&lcl.ip4, &rmt.ip4,
+				clib_host_to_net_u16 (lcl_port),
+				clib_host_to_net_u16 (rmt_port), s_type);
+  else
+    s = stream_session_lookup6 (&lcl.ip6, &rmt.ip6,
+				clib_host_to_net_u16 (lcl_port),
+				clib_host_to_net_u16 (rmt_port), s_type);
+  if (s)
+    {
+      *result = s;
+      return 1;
+    }
+  return 0;
+}
+
+uword
+unformat_transport_connection (unformat_input_t * input, va_list * args)
+{
+  transport_connection_t **result = va_arg (*args, transport_connection_t **);
+  u32 suggested_proto = va_arg (*args, u32);
+  transport_connection_t *tc;
+  u8 proto = ~0;
+  ip46_address_t lcl, rmt;
+  u32 lcl_port = 0, rmt_port = 0;
+  u8 is_ip4 = 0, s_type = ~0, id_is_set = 0;
+
+  if (unformat (input, "%U", unformat_stream_session_id, &proto, &lcl, &rmt,
+		&lcl_port, &rmt_port, &is_ip4))
+    {
+      id_is_set = 1;
+    }
+  else
+    return 0;
+
+  if (!id_is_set)
+    {
+      return 0;
+    }
+
+  proto = (proto == (u8) ~ 0) ? suggested_proto : proto;
+  if (proto == (u8) ~ 0)
+    return 0;
+  s_type = session_type_from_proto_and_ip (proto, is_ip4);
+  if (is_ip4)
+    tc = stream_session_lookup_transport4 (&lcl.ip4, &rmt.ip4,
+					   clib_host_to_net_u16 (lcl_port),
+					   clib_host_to_net_u16 (rmt_port),
+					   s_type);
+  else
+    tc = stream_session_lookup_transport6 (&lcl.ip6, &rmt.ip6,
+					   clib_host_to_net_u16 (lcl_port),
+					   clib_host_to_net_u16 (rmt_port),
+					   s_type);
+
+  if (tc)
+    {
+      *result = tc;
+      return 1;
+    }
+  return 0;
+}
+
 static clib_error_t *
 show_session_command_fn (vlib_main_t * vm, unformat_input_t * input,
 			 vlib_cli_command_t * cmd)
@@ -95,13 +224,7 @@ show_session_command_fn (vlib_main_t * vm, unformat_input_t * input,
   int verbose = 0, i;
   stream_session_t *pool;
   stream_session_t *s;
-  u8 *str = 0, one_session = 0, proto_set = 0, proto = 0;
-  u8 is_ip4 = 0, s_type = 0;
-  ip4_address_t lcl_ip4, rmt_ip4;
-  u32 lcl_port = 0, rmt_port = 0;
-
-  memset (&lcl_ip4, 0, sizeof (lcl_ip4));
-  memset (&rmt_ip4, 0, sizeof (rmt_ip4));
+  u8 *str = 0, one_session = 0;
 
   if (!smm->is_enabled)
     {
@@ -114,40 +237,18 @@ show_session_command_fn (vlib_main_t * vm, unformat_input_t * input,
 	;
       else if (unformat (input, "verbose"))
 	verbose = 1;
-      else if (unformat (input, "tcp"))
-	{
-	  proto_set = 1;
-	  proto = TRANSPORT_PROTO_TCP;
-	}
-      else if (unformat (input, "%U:%d->%U:%d",
-			 unformat_ip4_address, &lcl_ip4, &lcl_port,
-			 unformat_ip4_address, &rmt_ip4, &rmt_port))
+      else if (unformat (input, "%U", unformat_stream_session, &s))
 	{
 	  one_session = 1;
-	  is_ip4 = 1;
 	}
-
       else
-	break;
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
     }
 
   if (one_session)
     {
-      if (!proto_set)
-	{
-	  vlib_cli_output (vm, "proto not set");
-	  return clib_error_return (0, "proto not set");
-	}
-
-      s_type = session_type_from_proto_and_ip (proto, is_ip4);
-      s = stream_session_lookup4 (&lcl_ip4, &rmt_ip4,
-				  clib_host_to_net_u16 (lcl_port),
-				  clib_host_to_net_u16 (rmt_port), s_type);
-      if (s)
-	vlib_cli_output (vm, "%U", format_stream_session, s, 2);
-      else
-	vlib_cli_output (vm, "session does not exist");
-
+      vlib_cli_output (vm, "%U", format_stream_session, s, 2);
       return 0;
     }
 
@@ -275,6 +376,103 @@ VLIB_CLI_COMMAND (clear_session_command, static) =
 /* *INDENT-ON* */
 
 static clib_error_t *
+show_session_fifo_trace_command_fn (vlib_main_t * vm,
+				    unformat_input_t * input,
+				    vlib_cli_command_t * cmd)
+{
+  stream_session_t *s = 0;
+  u8 is_rx = 0, *str = 0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "%U", unformat_stream_session, &s))
+	;
+      else if (unformat (input, "rx"))
+	is_rx = 1;
+      else if (unformat (input, "tx"))
+	is_rx = 0;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+
+  if (!SVM_FIFO_TRACE)
+    {
+      vlib_cli_output (vm, "fifo tracing not enabled");
+      return 0;
+    }
+
+  if (!s)
+    {
+      vlib_cli_output (vm, "could not find session");
+      return 0;
+    }
+
+  str = is_rx ?
+    svm_fifo_dump_trace (str, s->server_rx_fifo) :
+    svm_fifo_dump_trace (str, s->server_tx_fifo);
+
+  vlib_cli_output (vm, "%v", str);
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_session_fifo_trace_command, static) =
+{
+  .path = "show session fifo trace",
+  .short_help = "show session fifo trace <session>",
+  .function = show_session_fifo_trace_command_fn,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+session_replay_fifo_command_fn (vlib_main_t * vm, unformat_input_t * input,
+				vlib_cli_command_t * cmd)
+{
+  stream_session_t *s = 0;
+  u8 is_rx = 0, *str = 0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "%U", unformat_stream_session, &s))
+	;
+      else if (unformat (input, "rx"))
+	is_rx = 1;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+
+  if (!SVM_FIFO_TRACE)
+    {
+      vlib_cli_output (vm, "fifo tracing not enabled");
+      return 0;
+    }
+
+  if (!s)
+    {
+      vlib_cli_output (vm, "could not find session");
+      return 0;
+    }
+
+  str = is_rx ?
+    svm_fifo_replay (str, s->server_rx_fifo, 0, 1) :
+    svm_fifo_replay (str, s->server_tx_fifo, 0, 1);
+
+  vlib_cli_output (vm, "%v", str);
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (session_replay_fifo_trace_command, static) =
+{
+  .path = "session replay fifo",
+  .short_help = "session replay fifo <session>",
+  .function = session_replay_fifo_command_fn,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
 session_enable_disable_fn (vlib_main_t * vm, unformat_input_t * input,
 			   vlib_cli_command_t * cmd)
 {
diff --git a/src/vnet/tcp/builtin_client.c b/src/vnet/tcp/builtin_client.c
index a6c8a23582b..a92bacaa37d 100644
--- a/src/vnet/tcp/builtin_client.c
+++ b/src/vnet/tcp/builtin_client.c
@@ -464,6 +464,8 @@ builtin_session_connected_callback (u32 app_index, u32 api_context,
 static void
 builtin_session_reset_callback (stream_session_t * s)
 {
+  if (s->session_state == SESSION_STATE_READY)
+    clib_warning ("Reset active connection %U", format_stream_session, s, 2);
   return;
 }
 
diff --git a/src/vnet/tcp/builtin_server.c b/src/vnet/tcp/builtin_server.c
index 8e958ac0b45..4ecaf56a70c 100644
--- a/src/vnet/tcp/builtin_server.c
+++ b/src/vnet/tcp/builtin_server.c
@@ -99,8 +99,7 @@ builtin_session_disconnect_callback (stream_session_t * s)
 void
 builtin_session_reset_callback (stream_session_t * s)
 {
-  clib_warning ("called.. ");
-
+  clib_warning ("Reset session %U", format_stream_session, s, 2);
   stream_session_cleanup (s);
 }
 
@@ -224,10 +223,6 @@ builtin_server_rx_callback (stream_session_t * s)
 	      clib_warning ("session stuck: %U", format_stream_session, s, 2);
 	    }
 	}
-      else
-	{
-	  bsm->rx_retries[thread_index][s->session_index] = 0;
-	}
 
       return 0;
     }
diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c
index f379e699839..8ed325d2809 100644
--- a/src/vnet/tcp/tcp.c
+++ b/src/vnet/tcp/tcp.c
@@ -732,6 +732,7 @@ format_tcp_connection (u8 * s, va_list * args)
       if (verbose > 1)
 	s = format (s, " %U\n%U", format_tcp_timers, tc, format_tcp_vars, tc);
     }
+
   return s;
 }
 
@@ -792,6 +793,30 @@ format_tcp_sacks (u8 * s, va_list * args)
 }
 
 u8 *
+format_tcp_rcv_sacks (u8 * s, va_list * args)
+{
+  tcp_connection_t *tc = va_arg (*args, tcp_connection_t *);
+  sack_block_t *sacks = tc->rcv_opts.sacks;
+  sack_block_t *block;
+  int i, len = 0;
+
+  len = vec_len (sacks);
+  for (i = 0; i < len - 1; i++)
+    {
+      block = &sacks[i];
+      s = format (s, " start %u end %u\n", block->start - tc->iss,
+		  block->end - tc->iss);
+    }
+  if (len)
+    {
+      block = &sacks[len - 1];
+      s = format (s, " start %u end %u", block->start - tc->iss,
+		  block->end - tc->iss);
+    }
+  return s;
+}
+
+u8 *
 format_tcp_sack_hole (u8 * s, va_list * args)
 {
   sack_scoreboard_hole_t *hole = va_arg (*args, sack_scoreboard_hole_t *);
@@ -820,6 +845,7 @@ format_tcp_scoreboard (u8 * s, va_list * args)
       s = format (s, "%U", format_tcp_sack_hole, hole);
       hole = scoreboard_next_hole (sb, hole);
     }
+
   return s;
 }
 
@@ -1304,7 +1330,189 @@ VLIB_CLI_COMMAND (tcp_src_address_command, static) =
 };
 /* *INDENT-ON* */
 
+static u8 *
+tcp_scoreboard_dump_trace (u8 * s, sack_scoreboard_t * sb)
+{
+#if TCP_SCOREBOARD_TRACE
+
+  scoreboard_trace_elt_t *block;
+  int i = 0;
+
+  if (!sb->trace)
+    return s;
+
+  s = format (s, "scoreboard trace:");
+  vec_foreach (block, sb->trace)
+  {
+    s = format (s, "{%u, %u, %u, %u, %u}, ", block->start, block->end,
+		block->ack, block->snd_una_max, block->group);
+    if ((++i % 3) == 0)
+      s = format (s, "\n");
+  }
+  return s;
+#else
+  return 0;
+#endif
+}
+
+static clib_error_t *
+tcp_show_scoreboard_trace_fn (vlib_main_t * vm, unformat_input_t * input,
+			      vlib_cli_command_t * cmd_arg)
+{
+  transport_connection_t *tconn = 0;
+  tcp_connection_t *tc;
+  u8 *s = 0;
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "%U", unformat_transport_connection, &tconn,
+		    TRANSPORT_PROTO_TCP))
+	;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+
+  if (!TCP_SCOREBOARD_TRACE)
+    {
+      vlib_cli_output (vm, "scoreboard tracing not enabled");
+      return 0;
+    }
+
+  tc = tcp_get_connection_from_transport (tconn);
+  s = tcp_scoreboard_dump_trace (s, &tc->sack_sb);
+  vlib_cli_output (vm, "%v", s);
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (tcp_show_scoreboard_trace_command, static) =
+{
+  .path = "show tcp scoreboard trace",
+  .short_help = "show tcp scoreboard trace <connection>",
+  .function = tcp_show_scoreboard_trace_fn,
+};
+/* *INDENT-ON* */
 
+u8 *
+tcp_scoreboard_replay (u8 * s, tcp_connection_t * tc, u8 verbose)
+{
+  int i, trace_len;
+  scoreboard_trace_elt_t *trace;
+  u32 next_ack, left, group, has_new_ack = 0;
+  tcp_connection_t _dummy_tc, *dummy_tc = &_dummy_tc;
+  sack_block_t *block;
+
+  if (!tc)
+    return s;
+
+  memset (dummy_tc, 0, sizeof (*dummy_tc));
+  tcp_connection_timers_init (dummy_tc);
+  scoreboard_init (&dummy_tc->sack_sb);
+  dummy_tc->rcv_opts.flags |= TCP_OPTS_FLAG_SACK;
+
+#if TCP_SCOREBOARD_TRACE
+  trace = tc->sack_sb.trace;
+  trace_len = vec_len (tc->sack_sb.trace);
+#else
+  trace = 0;
+  trace_len = 0;
+#endif
+
+  for (i = 0; i < trace_len; i++)
+    {
+      if (trace[i].ack != 0)
+	{
+	  dummy_tc->snd_una = trace[i].ack - 1448;
+	  dummy_tc->snd_una_max = trace[i].ack;
+	}
+    }
+
+  left = 0;
+  while (left < trace_len)
+    {
+      group = trace[left].group;
+      vec_reset_length (dummy_tc->rcv_opts.sacks);
+      has_new_ack = 0;
+      while (trace[left].group == group)
+	{
+	  if (trace[left].ack != 0)
+	    {
+	      if (verbose)
+		s = format (s, "Adding ack %u, snd_una_max %u, segs: ",
+			    trace[left].ack, trace[left].snd_una_max);
+	      dummy_tc->snd_una_max = trace[left].snd_una_max;
+	      next_ack = trace[left].ack;
+	      has_new_ack = 1;
+	    }
+	  else
+	    {
+	      if (verbose)
+		s = format (s, "[%u, %u], ", trace[left].start,
+			    trace[left].end);
+	      vec_add2 (dummy_tc->rcv_opts.sacks, block, 1);
+	      block->start = trace[left].start;
+	      block->end = trace[left].end;
+	    }
+	  left++;
+	}
+
+      /* Push segments */
+      tcp_rcv_sacks (dummy_tc, next_ack);
+      if (has_new_ack)
+	dummy_tc->snd_una = next_ack + dummy_tc->sack_sb.snd_una_adv;
+
+      if (verbose)
+	s = format (s, "result: %U", format_tcp_scoreboard,
+		    &dummy_tc->sack_sb);
+
+    }
+  s = format (s, "result: %U", format_tcp_scoreboard, &dummy_tc->sack_sb);
+
+  return s;
+}
+
+static clib_error_t *
+tcp_scoreboard_trace_fn (vlib_main_t * vm, unformat_input_t * input,
+			 vlib_cli_command_t * cmd_arg)
+{
+  transport_connection_t *tconn = 0;
+  tcp_connection_t *tc = 0;
+  u8 *str = 0;
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "%U", unformat_transport_connection, &tconn,
+		    TRANSPORT_PROTO_TCP))
+	;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+
+  if (!TCP_SCOREBOARD_TRACE)
+    {
+      vlib_cli_output (vm, "scoreboard tracing not enabled");
+      return 0;
+    }
+
+  tc = tcp_get_connection_from_transport (tconn);
+  if (!tc)
+    {
+      vlib_cli_output (vm, "connection not found");
+      return 0;
+    }
+  str = tcp_scoreboard_replay (str, tc, 1);
+  vlib_cli_output (vm, "%v", str);
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (tcp_replay_scoreboard_command, static) =
+{
+  .path = "tcp replay scoreboard",
+  .short_help = "tcp replay scoreboard <connection>",
+  .function = tcp_scoreboard_trace_fn,
+};
+/* *INDENT-ON* */
 
 /*
  * fd.io coding-style-patch-verification: ON
diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h
index 37b10fd4753..fd0d02b99d6 100644
--- a/src/vnet/tcp/tcp.h
+++ b/src/vnet/tcp/tcp.h
@@ -62,6 +62,7 @@ typedef enum _tcp_state
 format_function_t format_tcp_state;
 format_function_t format_tcp_flags;
 format_function_t format_tcp_sacks;
+format_function_t format_tcp_rcv_sacks;
 
 /** TCP timers */
 #define foreach_tcp_timer               \
@@ -151,9 +152,19 @@ enum
 #undef _
 };
 
+#define TCP_SCOREBOARD_TRACE (0)
 #define TCP_MAX_SACK_BLOCKS 15	/**< Max number of SACK blocks stored */
 #define TCP_INVALID_SACK_HOLE_INDEX ((u32)~0)
 
+typedef struct _scoreboard_trace_elt
+{
+  u32 start;
+  u32 end;
+  u32 ack;
+  u32 snd_una_max;
+  u32 group;
+} scoreboard_trace_elt_t;
+
 typedef struct _sack_scoreboard_hole
 {
   u32 next;		/**< Index for next entry in linked list */
@@ -177,8 +188,38 @@ typedef struct _sack_scoreboard
   u32 rescue_rxt;			/**< Rescue sequence number */
   u32 lost_bytes;			/**< Bytes lost as per RFC6675 */
   u32 cur_rxt_hole;			/**< Retransmitting from this hole */
+
+#if TCP_SCOREBOARD_TRACE
+  scoreboard_trace_elt_t *trace;
+#endif
+
 } sack_scoreboard_t;
 
+#if TCP_SCOREBOARD_TRACE
+#define tcp_scoreboard_trace_add(_tc, _ack) 				\
+{									\
+    static u64 _group = 0;						\
+    sack_scoreboard_t *_sb = &_tc->sack_sb;				\
+    sack_block_t *_sack, *_sacks;					\
+    scoreboard_trace_elt_t *_elt;					\
+    int i;								\
+    _group++;								\
+    _sacks = _tc->rcv_opts.sacks;					\
+    for (i = 0; i < vec_len (_sacks); i++) 				\
+      {									\
+	_sack = &_sacks[i];						\
+	vec_add2 (_sb->trace, _elt, 1);					\
+	_elt->start = _sack->start;					\
+	_elt->end = _sack->end;						\
+	_elt->ack = _elt->end == _ack ? _ack : 0;			\
+	_elt->snd_una_max = _elt->end == _ack ? _tc->snd_una_max : 0;	\
+	_elt->group = _group;						\
+      }									\
+}
+#else
+#define tcp_scoreboard_trace_add(_tc, _ack)
+#endif
+
 typedef enum _tcp_cc_algorithm_type
 {
   TCP_CC_NEWRENO,
@@ -405,6 +446,12 @@ tcp_connection_get_if_valid (u32 conn_index, u32 thread_index)
   return pool_elt_at_index (tcp_main.connections[thread_index], conn_index);
 }
 
+always_inline tcp_connection_t *
+tcp_get_connection_from_transport (transport_connection_t * tconn)
+{
+  return (tcp_connection_t *) tconn;
+}
+
 void tcp_connection_close (tcp_connection_t * tc);
 void tcp_connection_cleanup (tcp_connection_t * tc);
 void tcp_connection_del (tcp_connection_t * tc);
@@ -414,6 +461,8 @@ u8 *format_tcp_connection_id (u8 * s, va_list * args);
 u8 *format_tcp_connection (u8 * s, va_list * args);
 u8 *format_tcp_scoreboard (u8 * s, va_list * args);
 
+u8 *tcp_scoreboard_replay (u8 * s, tcp_connection_t * tc, u8 verbose);
+
 always_inline tcp_connection_t *
 tcp_listener_get (u32 tli)
 {
@@ -689,7 +738,7 @@ sack_scoreboard_hole_t *scoreboard_next_rxt_hole (sack_scoreboard_t * sb,
 						  start, u8 have_sent_1_smss,
 						  u8 * can_rescue,
 						  u8 * snd_limited);
-void scoreboard_init_high_rxt (sack_scoreboard_t * sb);
+void scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 seq);
 
 always_inline sack_scoreboard_hole_t *
 scoreboard_get_hole (sack_scoreboard_t * sb, u32 index)
@@ -740,6 +789,7 @@ scoreboard_clear (sack_scoreboard_t * sb)
       scoreboard_remove_hole (sb, hole);
     }
   ASSERT (sb->head == sb->tail && sb->head == TCP_INVALID_SACK_HOLE_INDEX);
+  ASSERT (pool_elts (sb->holes) == 0);
   sb->sacked_bytes = 0;
   sb->last_sacked_bytes = 0;
   sb->last_bytes_delivered = 0;
@@ -759,6 +809,7 @@ scoreboard_hole_bytes (sack_scoreboard_hole_t * hole)
 always_inline u32
 scoreboard_hole_index (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
 {
+  ASSERT (!pool_is_free_index (sb->holes, hole - sb->holes));
   return hole - sb->holes;
 }
 
diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c
index 45db0da69c6..bc7d9015789 100644
--- a/src/vnet/tcp/tcp_input.c
+++ b/src/vnet/tcp/tcp_input.c
@@ -206,8 +206,8 @@ tcp_options_parse (tcp_header_t * th, tcp_options_t * to)
 	  vec_reset_length (to->sacks);
 	  for (j = 0; j < to->n_sack_blocks; j++)
 	    {
-	      b.start = clib_net_to_host_u32 (*(u32 *) (data + 2 + 4 * j));
-	      b.end = clib_net_to_host_u32 (*(u32 *) (data + 6 + 4 * j));
+	      b.start = clib_net_to_host_u32 (*(u32 *) (data + 2 + 8 * j));
+	      b.end = clib_net_to_host_u32 (*(u32 *) (data + 6 + 8 * j));
 	      vec_add1 (to->sacks, b);
 	    }
 	  break;
@@ -540,6 +540,10 @@ scoreboard_remove_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
   if (scoreboard_hole_index (sb, hole) == sb->cur_rxt_hole)
     sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
 
+  /* Poison the entry */
+  if (CLIB_DEBUG > 0)
+    memset (hole, 0xfe, sizeof (*hole));
+
   pool_put (sb->holes, hole);
 }
 
@@ -555,7 +559,7 @@ scoreboard_insert_hole (sack_scoreboard_t * sb, u32 prev_index,
 
   hole->start = start;
   hole->end = end;
-  hole_index = hole - sb->holes;
+  hole_index = scoreboard_hole_index (sb, hole);
 
   prev = scoreboard_get_hole (sb, prev_index);
   if (prev)
@@ -680,12 +684,30 @@ scoreboard_next_rxt_hole (sack_scoreboard_t * sb,
 }
 
 void
-scoreboard_init_high_rxt (sack_scoreboard_t * sb)
+scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 seq)
 {
   sack_scoreboard_hole_t *hole;
   hole = scoreboard_first_hole (sb);
-  sb->high_rxt = hole->start;
-  sb->cur_rxt_hole = sb->head;
+  if (hole)
+    {
+      seq = seq_gt (seq, hole->start) ? seq : hole->start;
+      sb->cur_rxt_hole = sb->head;
+    }
+  sb->high_rxt = seq;
+}
+
+/**
+ * Test that scoreboard is sane after recovery
+ *
+ * Returns 1 if scoreboard is empty or if first hole beyond
+ * snd_una.
+ */
+u8
+tcp_scoreboard_is_sane_post_recovery (tcp_connection_t * tc)
+{
+  sack_scoreboard_hole_t *hole;
+  hole = scoreboard_first_hole (&tc->sack_sb);
+  return (!hole || seq_geq (hole->start, tc->snd_una));
 }
 
 void
@@ -712,7 +734,7 @@ tcp_rcv_sacks (tcp_connection_t * tc, u32 ack)
     {
       if (seq_lt (blk->start, blk->end)
 	  && seq_gt (blk->start, tc->snd_una)
-	  && seq_gt (blk->start, ack) && seq_leq (blk->end, tc->snd_nxt))
+	  && seq_gt (blk->start, ack) && seq_leq (blk->end, tc->snd_una_max))
 	{
 	  blk++;
 	  continue;
@@ -731,6 +753,8 @@ tcp_rcv_sacks (tcp_connection_t * tc, u32 ack)
   if (vec_len (tc->rcv_opts.sacks) == 0)
     return;
 
+  tcp_scoreboard_trace_add (tc, ack);
+
   /* Make sure blocks are ordered */
   for (i = 0; i < vec_len (tc->rcv_opts.sacks); i++)
     for (j = i + 1; j < vec_len (tc->rcv_opts.sacks); j++)
@@ -797,7 +821,7 @@ tcp_rcv_sacks (tcp_connection_t * tc, u32 ack)
 		      sb->last_bytes_delivered +=
 			next_hole->start - hole->end;
 		    }
-		  else if (!next_hole)
+		  else
 		    {
 		      ASSERT (seq_geq (sb->high_sacked, ack));
 		      sb->snd_una_adv = sb->high_sacked - ack;
@@ -824,12 +848,14 @@ tcp_rcv_sacks (tcp_connection_t * tc, u32 ack)
 	  if (seq_lt (blk->end, hole->end))
 	    {
 	      hole_index = scoreboard_hole_index (sb, hole);
-	      scoreboard_insert_hole (sb, hole_index, blk->end, hole->end);
+	      next_hole = scoreboard_insert_hole (sb, hole_index, blk->end,
+						  hole->end);
 
 	      /* Pool might've moved */
 	      hole = scoreboard_get_hole (sb, hole_index);
 	      hole->end = blk->start;
 	      blk_index++;
+	      ASSERT (hole->next == scoreboard_hole_index (sb, next_hole));
 	    }
 	  else if (seq_lt (blk->start, hole->end))
 	    {
@@ -957,7 +983,7 @@ tcp_cc_recover (tcp_connection_t * tc)
 
   ASSERT (tc->rto_boff == 0);
   ASSERT (!tcp_in_cong_recovery (tc));
-
+  ASSERT (tcp_scoreboard_is_sane_post_recovery (tc));
   TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 3);
   return 0;
 }
@@ -965,7 +991,7 @@ tcp_cc_recover (tcp_connection_t * tc)
 static void
 tcp_cc_update (tcp_connection_t * tc, vlib_buffer_t * b)
 {
-  ASSERT (!tcp_in_cong_recovery (tc));
+  ASSERT (!tcp_in_cong_recovery (tc) || tcp_is_lost_fin (tc));
 
   /* Congestion avoidance */
   tc->cc_algo->rcv_ack (tc);
@@ -1064,10 +1090,10 @@ tcp_cc_handle_event (tcp_connection_t * tc, u32 is_dack)
 	  ASSERT (tc->cwnd >= tc->snd_mss);
 
 	  /* If cwnd allows, send more data */
-	  if (tcp_opts_sack_permitted (&tc->rcv_opts)
-	      && scoreboard_first_hole (&tc->sack_sb))
+	  if (tcp_opts_sack_permitted (&tc->rcv_opts))
 	    {
-	      scoreboard_init_high_rxt (&tc->sack_sb);
+	      scoreboard_init_high_rxt (&tc->sack_sb,
+					tc->snd_una + tc->snd_mss);
 	      tcp_fast_retransmit_sack (tc);
 	    }
 	  else
@@ -1134,12 +1160,13 @@ partial_ack:
   /* Remove retransmitted bytes that have been delivered */
   ASSERT (tc->bytes_acked + tc->sack_sb.snd_una_adv
 	  >= tc->sack_sb.last_bytes_delivered);
-  rxt_delivered = tc->bytes_acked + tc->sack_sb.snd_una_adv
-    - tc->sack_sb.last_bytes_delivered;
-  if (0 && rxt_delivered && seq_gt (tc->sack_sb.high_rxt, tc->snd_una))
+
+  if (seq_lt (tc->snd_una, tc->sack_sb.high_rxt))
     {
       /* If we have sacks and we haven't gotten an ack beyond high_rxt,
        * remove sacked bytes delivered */
+      rxt_delivered = tc->bytes_acked + tc->sack_sb.snd_una_adv
+	- tc->sack_sb.last_bytes_delivered;
       ASSERT (tc->snd_rxt_bytes >= rxt_delivered);
       tc->snd_rxt_bytes -= rxt_delivered;
     }
@@ -1256,6 +1283,18 @@ tcp_rcv_ack (tcp_connection_t * tc, vlib_buffer_t * b,
   return 0;
 }
 
+static u8
+tcp_sack_vector_is_sane (sack_block_t * sacks)
+{
+  int i;
+  for (i = 1; i < vec_len (sacks); i++)
+    {
+      if (sacks[i - 1].end == sacks[i].start)
+	return 0;
+    }
+  return 1;
+}
+
 /**
  * Build SACK list as per RFC2018.
  *
@@ -1316,6 +1355,9 @@ tcp_update_sack_list (tcp_connection_t * tc, u32 start, u32 end)
   /* Replace old vector with new one */
   vec_free (tc->snd_sacks);
   tc->snd_sacks = new_list;
+
+  /* Segments should not 'touch' */
+  ASSERT (tcp_sack_vector_is_sane (tc->snd_sacks));
 }
 
 /** Enqueue data for delivery to application */
@@ -1330,7 +1372,6 @@ tcp_session_enqueue_data (tcp_connection_t * tc, vlib_buffer_t * b,
   /* Pure ACK. Update rcv_nxt and be done. */
   if (PREDICT_FALSE (data_len == 0))
     {
-      tc->rcv_nxt = vnet_buffer (b)->tcp.seq_end;
       return TCP_ERROR_PURE_ACK;
     }
 
@@ -1385,7 +1426,7 @@ tcp_session_enqueue_ooo (tcp_connection_t * tc, vlib_buffer_t * b,
 			 u16 data_len)
 {
   stream_session_t *s0;
-  int rv;
+  int rv, offset;
 
   ASSERT (seq_gt (vnet_buffer (b)->tcp.seq_number, tc->rcv_nxt));
 
@@ -1421,12 +1462,12 @@ tcp_session_enqueue_ooo (tcp_connection_t * tc, vlib_buffer_t * b,
       newest = svm_fifo_newest_ooo_segment (s0->server_rx_fifo);
       if (newest)
 	{
-	  start =
-	    tc->rcv_nxt + ooo_segment_offset (s0->server_rx_fifo, newest);
+	  offset = ooo_segment_offset (s0->server_rx_fifo, newest);
+	  ASSERT (offset <= vnet_buffer (b)->tcp.seq_number - tc->rcv_nxt);
+	  start = tc->rcv_nxt + offset;
 	  end = start + ooo_segment_length (s0->server_rx_fifo, newest);
 	  tcp_update_sack_list (tc, start, end);
-
-	  ASSERT (seq_gt (start, tc->rcv_nxt));
+	  svm_fifo_newest_ooo_segment_reset (s0->server_rx_fifo);
 	}
     }
 
@@ -2736,12 +2777,12 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      /* lookup session */
 	      tc0 =
 		(tcp_connection_t *)
-		stream_session_lookup_transport4 (&ip40->dst_address,
-						  &ip40->src_address,
-						  tcp0->dst_port,
-						  tcp0->src_port,
-						  SESSION_TYPE_IP4_TCP,
-						  my_thread_index);
+		stream_session_lookup_transport_wt4 (&ip40->dst_address,
+						     &ip40->src_address,
+						     tcp0->dst_port,
+						     tcp0->src_port,
+						     SESSION_TYPE_IP4_TCP,
+						     my_thread_index);
 	    }
 	  else
 	    {
@@ -2754,12 +2795,12 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
 	      tc0 =
 		(tcp_connection_t *)
-		stream_session_lookup_transport6 (&ip60->src_address,
-						  &ip60->dst_address,
-						  tcp0->src_port,
-						  tcp0->dst_port,
-						  SESSION_TYPE_IP6_TCP,
-						  my_thread_index);
+		stream_session_lookup_transport_wt6 (&ip60->src_address,
+						     &ip60->dst_address,
+						     tcp0->src_port,
+						     tcp0->dst_port,
+						     SESSION_TYPE_IP6_TCP,
+						     my_thread_index);
 	    }
 
 	  /* Length check */
@@ -2931,6 +2972,8 @@ do {                                                       	\
   _(ESTABLISHED, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_ESTABLISHED,
     TCP_ERROR_NONE);
   _(ESTABLISHED, TCP_FLAG_SYN, TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
+  _(ESTABLISHED, TCP_FLAG_SYN | TCP_FLAG_ACK, TCP_INPUT_NEXT_ESTABLISHED,
+    TCP_ERROR_NONE);
   /* ACK or FIN-ACK to our FIN */
   _(FIN_WAIT_1, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
   _(FIN_WAIT_1, TCP_FLAG_ACK | TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS,
@@ -2954,6 +2997,7 @@ do {                                                       	\
   _(TIME_WAIT, TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
   _(TIME_WAIT, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
     TCP_ERROR_NONE);
+  _(TIME_WAIT, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
   _(CLOSED, TCP_FLAG_ACK, TCP_INPUT_NEXT_RESET, TCP_ERROR_CONNECTION_CLOSED);
   _(CLOSED, TCP_FLAG_RST, TCP_INPUT_NEXT_DROP, TCP_ERROR_CONNECTION_CLOSED);
   _(CLOSED, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RESET,
diff --git a/src/vnet/tcp/tcp_test.c b/src/vnet/tcp/tcp_test.c
index 510deb4fec0..f37ba96dab8 100644
--- a/src/vnet/tcp/tcp_test.c
+++ b/src/vnet/tcp/tcp_test.c
@@ -34,6 +34,38 @@
     }								\
 }
 
+/* *INDENT-OFF* */
+scoreboard_trace_elt_t sb_trace[] = {};
+/* *INDENT-ON* */
+
+static int
+tcp_test_scoreboard_replay (vlib_main_t * vm, unformat_input_t * input)
+{
+  int verbose = 0;
+  tcp_connection_t _tc, *tc = &_tc;
+  u8 *s = 0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "detail"))
+	verbose = 1;
+      else
+	{
+	  clib_error_t *e = clib_error_return
+	    (0, "unknown input `%U'", format_unformat_error, input);
+	  clib_error_report (e);
+	  return -1;
+	}
+    }
+
+#if TCP_SCOREBOARD_TRACE
+  tc->sack_sb.trace = sb_trace;
+#endif
+  s = tcp_scoreboard_replay (s, tc, verbose);
+  vlib_cli_output (vm, "%v", s);
+  return 0;
+}
+
 static int
 tcp_test_sack_rx (vlib_main_t * vm, unformat_input_t * input)
 {
@@ -47,6 +79,8 @@ tcp_test_sack_rx (vlib_main_t * vm, unformat_input_t * input)
     {
       if (unformat (input, "verbose"))
 	verbose = 1;
+      else if (unformat (input, "replay"))
+	return tcp_test_scoreboard_replay (vm, input);
     }
 
   memset (tc, 0, sizeof (*tc));
@@ -282,6 +316,44 @@ tcp_test_sack_rx (vlib_main_t * vm, unformat_input_t * input)
   TCP_TEST ((sb->last_bytes_delivered == 400),
 	    "last bytes delivered %d", sb->last_bytes_delivered);
 
+  /*
+   * One hole close to head, patch head, split in two and start acking
+   * the lowest part
+   */
+  scoreboard_clear (sb);
+  tc->snd_una = 0;
+  tc->snd_una_max = 1000;
+  tc->snd_nxt = 1000;
+
+  block.start = 500;
+  block.end = 1000;
+  vec_add1 (tc->rcv_opts.sacks, block);
+  tc->rcv_opts.n_sack_blocks = vec_len (tc->rcv_opts.sacks);
+
+  tcp_rcv_sacks (tc, 0);
+  if (verbose)
+    vlib_cli_output (vm, "sb added [500, 1000]:\n%U",
+		     format_tcp_scoreboard, sb);
+
+  vec_reset_length (tc->rcv_opts.sacks);
+  block.start = 300;
+  block.end = 400;
+  vec_add1 (tc->rcv_opts.sacks, block);
+  tc->rcv_opts.n_sack_blocks = vec_len (tc->rcv_opts.sacks);
+  tcp_rcv_sacks (tc, 100);
+  if (verbose)
+    vlib_cli_output (vm, "sb added [0, 100] [300, 400]:\n%U",
+		     format_tcp_scoreboard, sb);
+  TCP_TEST ((pool_elts (sb->holes) == 2),
+	    "scoreboard has %d elements", pool_elts (sb->holes));
+
+  tc->snd_una = 100;
+  tcp_rcv_sacks (tc, 200);
+  tcp_rcv_sacks (tc, 300);
+  if (verbose)
+    vlib_cli_output (vm, "sb added [0, 300]:\n%U", format_tcp_scoreboard, sb);
+  TCP_TEST ((sb->sacked_bytes == 500), "sacked bytes %d", sb->sacked_bytes);
+
   return 0;
 }
 
@@ -390,6 +462,37 @@ tcp_test_sack_tx (vlib_main_t * vm, unformat_input_t * input)
     vlib_cli_output (vm, "advance rcv_nxt to 1200\n%U", format_tcp_sacks, tc);
   TCP_TEST ((vec_len (tc->snd_sacks) == 0), "sack blocks %d expected %d",
 	    vec_len (tc->snd_sacks), 0);
+
+
+  /*
+   * Add 2 blocks, overwrite first and update rcv_nxt to also remove it
+   */
+
+  vec_reset_length (tc->snd_sacks);
+  tc->rcv_nxt = 0;
+
+  tcp_update_sack_list (tc, 100, 200);
+  tcp_update_sack_list (tc, 300, 400);
+
+  if (verbose)
+    vlib_cli_output (vm, "add [100, 200] [300, 400]\n%U",
+		     format_tcp_sacks, tc);
+  TCP_TEST ((vec_len (tc->snd_sacks) == 2),
+	    "sack blocks %d expected %d", vec_len (tc->snd_sacks), 2);
+  TCP_TEST ((tc->snd_sacks[0].start == 300),
+	    "first sack block start %u expected %u", tc->snd_sacks[0].start,
+	    300);
+
+  tc->rcv_nxt = 100;
+  tcp_update_sack_list (tc, 100, 100);
+  if (verbose)
+    vlib_cli_output (vm, "add [100, 200] rcv_nxt = 100\n%U",
+		     format_tcp_sacks, tc);
+  TCP_TEST ((vec_len (tc->snd_sacks) == 1),
+	    "sack blocks %d expected %d", vec_len (tc->snd_sacks), 1);
+  TCP_TEST ((tc->snd_sacks[0].start == 300),
+	    "first sack block start %u expected %u", tc->snd_sacks[0].start,
+	    300);
   return 0;
 }
 
@@ -1188,6 +1291,176 @@ tcp_test_fifo4 (vlib_main_t * vm, unformat_input_t * input)
   return 0;
 }
 
+static u32
+fifo_pos (svm_fifo_t * f, u32 pos)
+{
+  return pos % f->nitems;
+}
+
+static int
+tcp_test_fifo5 (vlib_main_t * vm, unformat_input_t * input)
+{
+  svm_fifo_t *f;
+  u32 fifo_size = 400, j = 0, offset = 200;
+  int i, rv, verbose = 0;
+  u8 *test_data = 0, *data_buf = 0;
+  ooo_segment_t *ooo_seg;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "verbose"))
+	verbose = 1;
+      else
+	{
+	  clib_error_t *e = clib_error_return
+	    (0, "unknown input `%U'", format_unformat_error, input);
+	  clib_error_report (e);
+	  return -1;
+	}
+    }
+
+  f = fifo_prepare (fifo_size);
+  svm_fifo_init_pointers (f, offset);
+
+  vec_validate (test_data, 399);
+  for (i = 0; i < vec_len (test_data); i++)
+    test_data[i] = i % 0xff;
+
+  /*
+   * Start with [100, 200] and [300, 400]
+   */
+  svm_fifo_enqueue_with_offset (f, 100, 100, &test_data[100]);
+  svm_fifo_enqueue_with_offset (f, 300, 100, &test_data[300]);
+
+  TCP_TEST ((svm_fifo_number_ooo_segments (f) == 2),
+	    "number of ooo segments %u", svm_fifo_number_ooo_segments (f));
+  TCP_TEST ((f->ooos_newest == 1), "newest %u", f->ooos_newest);
+  if (verbose)
+    vlib_cli_output (vm, "fifo after [100, 200] and [300, 400] : %U",
+		     format_svm_fifo, f, 2 /* verbose */ );
+
+  /*
+   * Add [225, 275]
+   */
+
+  rv = svm_fifo_enqueue_with_offset (f, 225, 50, &test_data[200]);
+  if (verbose)
+    vlib_cli_output (vm, "fifo after [225, 275] : %U",
+		     format_svm_fifo, f, 2 /* verbose */ );
+  TCP_TEST ((svm_fifo_number_ooo_segments (f) == 3),
+	    "number of ooo segments %u", svm_fifo_number_ooo_segments (f));
+  ooo_seg = svm_fifo_first_ooo_segment (f);
+  TCP_TEST ((ooo_seg->start == fifo_pos (f, 100 + offset)),
+	    "first seg start %u expected %u", ooo_seg->start,
+	    fifo_pos (f, 100 + offset));
+  TCP_TEST ((ooo_seg->length == 100), "first seg length %u expected %u",
+	    ooo_seg->length, 100);
+  ooo_seg = ooo_segment_next (f, ooo_seg);
+  TCP_TEST ((ooo_seg->start == fifo_pos (f, 225 + offset)),
+	    "second seg start %u expected %u",
+	    ooo_seg->start, fifo_pos (f, 225 + offset));
+  TCP_TEST ((ooo_seg->length == 50), "second seg length %u expected %u",
+	    ooo_seg->length, 50);
+  ooo_seg = ooo_segment_next (f, ooo_seg);
+  TCP_TEST ((ooo_seg->start == fifo_pos (f, 300 + offset)),
+	    "third seg start %u expected %u",
+	    ooo_seg->start, fifo_pos (f, 300 + offset));
+  TCP_TEST ((ooo_seg->length == 100), "third seg length %u expected %u",
+	    ooo_seg->length, 100);
+  TCP_TEST ((f->ooos_newest == 2), "newest %u", f->ooos_newest);
+  /*
+   * Add [190, 310]
+   */
+  rv = svm_fifo_enqueue_with_offset (f, 190, 120, &test_data[190]);
+  if (verbose)
+    vlib_cli_output (vm, "fifo after [190, 310] : %U",
+		     format_svm_fifo, f, 1 /* verbose */ );
+  TCP_TEST ((svm_fifo_number_ooo_segments (f) == 1),
+	    "number of ooo segments %u", svm_fifo_number_ooo_segments (f));
+  ooo_seg = svm_fifo_first_ooo_segment (f);
+  TCP_TEST ((ooo_seg->start == fifo_pos (f, offset + 100)),
+	    "first seg start %u expected %u",
+	    ooo_seg->start, fifo_pos (f, offset + 100));
+  TCP_TEST ((ooo_seg->length == 300), "first seg length %u expected %u",
+	    ooo_seg->length, 300);
+
+  /*
+   * Add [0, 150]
+   */
+  rv = svm_fifo_enqueue_nowait (f, 150, test_data);
+
+  if (verbose)
+    vlib_cli_output (vm, "fifo after [0 150] : %U", format_svm_fifo, f,
+		     2 /* verbose */ );
+
+  TCP_TEST ((rv == 400), "managed to enqueue %u expected %u", rv, 400);
+  TCP_TEST ((svm_fifo_number_ooo_segments (f) == 0),
+	    "number of ooo segments %u", svm_fifo_number_ooo_segments (f));
+
+  vec_validate (data_buf, 399);
+  svm_fifo_peek (f, 0, 400, data_buf);
+  if (compare_data (data_buf, test_data, 0, 400, &j))
+    {
+      TCP_TEST (0, "[%d] peeked %u expected %u", j, data_buf[j],
+		test_data[j]);
+    }
+
+  /*
+   * Add [100 200] and overlap it with [50 250]
+   */
+  svm_fifo_free (f);
+  f = fifo_prepare (fifo_size);
+
+  svm_fifo_enqueue_with_offset (f, 100, 100, &test_data[100]);
+  svm_fifo_enqueue_with_offset (f, 50, 200, &test_data[50]);
+  TCP_TEST ((svm_fifo_number_ooo_segments (f) == 1),
+	    "number of ooo segments %u", svm_fifo_number_ooo_segments (f));
+  ooo_seg = svm_fifo_first_ooo_segment (f);
+  TCP_TEST ((ooo_seg->start == 50), "first seg start %u expected %u",
+	    ooo_seg->start, 50);
+  TCP_TEST ((ooo_seg->length == 200), "first seg length %u expected %u",
+	    ooo_seg->length, 200);
+
+  svm_fifo_free (f);
+  vec_free (test_data);
+  return 0;
+}
+
+/* *INDENT-OFF* */
+svm_fifo_trace_elem_t fifo_trace[] = {};
+/* *INDENT-ON* */
+
+static int
+tcp_test_fifo_replay (vlib_main_t * vm, unformat_input_t * input)
+{
+  svm_fifo_t f;
+  int verbose = 0;
+  u8 no_read = 0, *str = 0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "verbose"))
+	verbose = 1;
+      else if (unformat (input, "no-read"))
+	no_read = 1;
+      else
+	{
+	  clib_error_t *e = clib_error_return
+	    (0, "unknown input `%U'", format_unformat_error, input);
+	  clib_error_report (e);
+	  return -1;
+	}
+    }
+
+#if SVMF_FIFO_TRACE
+  f.trace = fifo_trace;
+#endif
+
+  str = svm_fifo_replay (str, &f, no_read, verbose);
+  vlib_cli_output (vm, "%v", str);
+  return 0;
+}
+
 static int
 tcp_test_fifo (vlib_main_t * vm, unformat_input_t * input)
 {
@@ -1237,6 +1510,14 @@ tcp_test_fifo (vlib_main_t * vm, unformat_input_t * input)
       if (tcp_test_fifo3 (vm, input))
 	return -1;
       unformat_free (input);
+
+      res = tcp_test_fifo4 (vm, input);
+      if (res)
+	return res;
+
+      res = tcp_test_fifo5 (vm, input);
+      if (res)
+	return res;
     }
   else
     {
@@ -1256,6 +1537,14 @@ tcp_test_fifo (vlib_main_t * vm, unformat_input_t * input)
 	{
 	  res = tcp_test_fifo4 (vm, input);
 	}
+      else if (unformat (input, "fifo5"))
+	{
+	  res = tcp_test_fifo5 (vm, input);
+	}
+      else if (unformat (input, "replay"))
+	{
+	  res = tcp_test_fifo_replay (vm, input);
+	}
     }
 
   return res;