tcp: cleanup functions

- sprinkle statics for functions
- move some inlines from header files to corresponding .c files
- replace some always_inlines with statics where inlining is not
performance critical

Change-Id: I371dbf63431ce7e27e4ebbbdd844a9546a1f1849
Signed-off-by: Florin Coras <fcoras@cisco.com>

4 files changed, 258 insertions, 263 deletions

diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c
index 30173619e0e..cca9f1c5c90 100644
--- a/src/vnet/tcp/tcp.c
+++ b/src/vnet/tcp/tcp.c
@@ -103,7 +103,7 @@ tcp_connection_bind (u32 session_index, transport_endpoint_t * lcl)
   return listener->c_c_index;
 }
 
-u32
+static u32
 tcp_session_bind (u32 session_index, transport_endpoint_t * tep)
 {
   return tcp_connection_bind (session_index, tep);
@@ -126,14 +126,14 @@ tcp_connection_unbind (u32 listener_index)
   pool_put_index (tm->listener_pool, listener_index);
 }
 
-u32
+static u32
 tcp_session_unbind (u32 listener_index)
 {
   tcp_connection_unbind (listener_index);
   return 0;
 }
 
-transport_connection_t *
+static transport_connection_t *
 tcp_session_get_listener (u32 listener_index)
 {
   tcp_main_t *tm = vnet_get_tcp_main ();
@@ -146,7 +146,7 @@ tcp_session_get_listener (u32 listener_index)
  * Cleanup half-open connection
  *
  */
-void
+static void
 tcp_half_open_connection_del (tcp_connection_t * tc)
 {
   tcp_main_t *tm = vnet_get_tcp_main ();
@@ -178,7 +178,7 @@ tcp_half_open_connection_cleanup (tcp_connection_t * tc)
   return 0;
 }
 
-tcp_connection_t *
+static tcp_connection_t *
 tcp_half_open_connection_new (void)
 {
   tcp_main_t *tm = vnet_get_tcp_main ();
@@ -358,7 +358,7 @@ tcp_connection_close (tcp_connection_t * tc)
     tcp_connection_del (tc);
 }
 
-void
+static void
 tcp_session_close (u32 conn_index, u32 thread_index)
 {
   tcp_connection_t *tc;
@@ -366,7 +366,7 @@ tcp_session_close (u32 conn_index, u32 thread_index)
   tcp_connection_close (tc);
 }
 
-void
+static void
 tcp_session_cleanup (u32 conn_index, u32 thread_index)
 {
   tcp_connection_t *tc;
@@ -504,6 +504,31 @@ tcp_connection_fib_attach (tcp_connection_t * tc)
 }
 #endif /* 0 */
 
+static void
+tcp_cc_init (tcp_connection_t * tc)
+{
+  tc->cc_algo = tcp_cc_algo_get (TCP_CC_NEWRENO);
+  tc->cc_algo->init (tc);
+}
+
+void
+tcp_cc_algo_register (tcp_cc_algorithm_type_e type,
+		      const tcp_cc_algorithm_t * vft)
+{
+  tcp_main_t *tm = vnet_get_tcp_main ();
+  vec_validate (tm->cc_algos, type);
+
+  tm->cc_algos[type] = *vft;
+}
+
+tcp_cc_algorithm_t *
+tcp_cc_algo_get (tcp_cc_algorithm_type_e type)
+{
+  tcp_main_t *tm = vnet_get_tcp_main ();
+  return &tm->cc_algos[type];
+}
+
+
 /**
  * Initialize connection send variables.
  */
@@ -577,7 +602,7 @@ tcp_alloc_custom_local_endpoint (tcp_main_t * tm, ip46_address_t * lcl_addr,
   return 0;
 }
 
-int
+static int
 tcp_connection_open (transport_endpoint_t * rmt)
 {
   tcp_main_t *tm = vnet_get_tcp_main ();
@@ -624,7 +649,7 @@ tcp_connection_open (transport_endpoint_t * rmt)
   return tc->c_c_index;
 }
 
-int
+static int
 tcp_session_open (transport_endpoint_t * tep)
 {
   return tcp_connection_open (tep);
@@ -660,7 +685,7 @@ const char *tcp_connection_flags_str[] = {
 #undef _
 };
 
-u8 *
+static u8 *
 format_tcp_connection_flags (u8 * s, va_list * args)
 {
   tcp_connection_t *tc = va_arg (*args, tcp_connection_t *);
@@ -685,7 +710,7 @@ const char *tcp_conn_timers[] = {
 #undef _
 };
 
-u8 *
+static u8 *
 format_tcp_timers (u8 * s, va_list * args)
 {
   tcp_connection_t *tc = va_arg (*args, tcp_connection_t *);
@@ -710,7 +735,7 @@ format_tcp_timers (u8 * s, va_list * args)
   return s;
 }
 
-u8 *
+static u8 *
 format_tcp_congestion_status (u8 * s, va_list * args)
 {
   tcp_connection_t *tc = va_arg (*args, tcp_connection_t *);
@@ -723,7 +748,13 @@ format_tcp_congestion_status (u8 * s, va_list * args)
   return s;
 }
 
-u8 *
+static i32
+tcp_rcv_wnd_available (tcp_connection_t * tc)
+{
+  return (i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las);
+}
+
+static u8 *
 format_tcp_vars (u8 * s, va_list * args)
 {
   tcp_connection_t *tc = va_arg (*args, tcp_connection_t *);
@@ -763,7 +794,7 @@ format_tcp_vars (u8 * s, va_list * args)
   return s;
 }
 
-u8 *
+static u8 *
 format_tcp_connection_id (u8 * s, va_list * args)
 {
   tcp_connection_t *tc = va_arg (*args, tcp_connection_t *);
@@ -806,7 +837,7 @@ format_tcp_connection (u8 * s, va_list * args)
   return s;
 }
 
-u8 *
+static u8 *
 format_tcp_session (u8 * s, va_list * args)
 {
   u32 tci = va_arg (*args, u32);
@@ -822,7 +853,7 @@ format_tcp_session (u8 * s, va_list * args)
   return s;
 }
 
-u8 *
+static u8 *
 format_tcp_listener_session (u8 * s, va_list * args)
 {
   u32 tci = va_arg (*args, u32);
@@ -830,7 +861,7 @@ format_tcp_listener_session (u8 * s, va_list * args)
   return format (s, "%U", format_tcp_connection_id, tc);
 }
 
-u8 *
+static u8 *
 format_tcp_half_open_session (u8 * s, va_list * args)
 {
   u32 tci = va_arg (*args, u32);
@@ -886,7 +917,7 @@ format_tcp_rcv_sacks (u8 * s, va_list * args)
   return s;
 }
 
-u8 *
+static u8 *
 format_tcp_sack_hole (u8 * s, va_list * args)
 {
   sack_scoreboard_hole_t *hole = va_arg (*args, sack_scoreboard_hole_t *);
@@ -924,14 +955,14 @@ format_tcp_scoreboard (u8 * s, va_list * args)
   return s;
 }
 
-transport_connection_t *
+static transport_connection_t *
 tcp_session_get_transport (u32 conn_index, u32 thread_index)
 {
   tcp_connection_t *tc = tcp_connection_get (conn_index, thread_index);
   return &tc->connection;
 }
 
-transport_connection_t *
+static transport_connection_t *
 tcp_half_open_session_get_transport (u32 conn_index)
 {
   tcp_connection_t *tc = tcp_half_open_connection_get (conn_index);
@@ -945,7 +976,7 @@ tcp_half_open_session_get_transport (u32 conn_index)
  * the tcp options to be used in the next burst and subtracts their
  * length from the connection's snd_mss.
  */
-u16
+static u16
 tcp_session_send_mss (transport_connection_t * trans_conn)
 {
   tcp_connection_t *tc = (tcp_connection_t *) trans_conn;
@@ -985,7 +1016,7 @@ tcp_round_snd_space (tcp_connection_t * tc, u32 snd_space)
  * @param tc tcp connection
  * @return number of bytes session is allowed to write
  */
-u32
+static u32
 tcp_snd_space (tcp_connection_t * tc)
 {
   int snd_space, snt_limited;
@@ -1034,7 +1065,7 @@ tcp_snd_space (tcp_connection_t * tc)
   return 0;
 }
 
-u32
+static u32
 tcp_session_send_space (transport_connection_t * trans_conn)
 {
   tcp_connection_t *tc = (tcp_connection_t *) trans_conn;
@@ -1042,13 +1073,7 @@ tcp_session_send_space (transport_connection_t * trans_conn)
 		   tc->snd_wnd - (tc->snd_nxt - tc->snd_una));
 }
 
-i32
-tcp_rcv_wnd_available (tcp_connection_t * tc)
-{
-  return (i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las);
-}
-
-u32
+static u32
 tcp_session_tx_fifo_offset (transport_connection_t * trans_conn)
 {
   tcp_connection_t *tc = (tcp_connection_t *) trans_conn;
@@ -1059,7 +1084,7 @@ tcp_session_tx_fifo_offset (transport_connection_t * trans_conn)
   return (tc->snd_nxt - tc->snd_una);
 }
 
-void
+static void
 tcp_update_time (f64 now, u8 thread_index)
 {
   tcp_set_time_now (thread_index);
@@ -1069,6 +1094,13 @@ tcp_update_time (f64 now, u8 thread_index)
   tcp_flush_frames_to_output (thread_index);
 }
 
+static u32
+tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b)
+{
+  tcp_connection_t *tc = (tcp_connection_t *) tconn;
+  return tcp_push_header (tc, b);
+}
+
 /* *INDENT-OFF* */
 const static transport_proto_vft_t tcp_proto = {
   .enable = vnet_tcp_enable_disable,
@@ -1093,7 +1125,7 @@ const static transport_proto_vft_t tcp_proto = {
 };
 /* *INDENT-ON* */
 
-void
+static void
 tcp_timer_keep_handler (u32 conn_index)
 {
   u32 thread_index = vlib_get_thread_index ();
@@ -1105,7 +1137,7 @@ tcp_timer_keep_handler (u32 conn_index)
   tcp_connection_close (tc);
 }
 
-void
+static void
 tcp_timer_establish_handler (u32 conn_index)
 {
   tcp_connection_t *tc;
@@ -1133,7 +1165,7 @@ tcp_timer_establish_handler (u32 conn_index)
   tcp_connection_cleanup (tc);
 }
 
-void
+static void
 tcp_timer_waitclose_handler (u32 conn_index)
 {
   u32 thread_index = vlib_get_thread_index ();
@@ -1198,7 +1230,7 @@ tcp_expired_timers_dispatch (u32 * expired_timers)
     }
 }
 
-void
+static void
 tcp_initialize_timer_wheels (tcp_main_t * tm)
 {
   tw_timer_wheel_16t_2w_512sl_t *tw;
@@ -1212,7 +1244,7 @@ tcp_initialize_timer_wheels (tcp_main_t * tm)
   /* *INDENT-ON* */
 }
 
-clib_error_t *
+static clib_error_t *
 tcp_main_enable (vlib_main_t * vm)
 {
   tcp_main_t *tm = vnet_get_tcp_main ();
@@ -1320,7 +1352,7 @@ tcp_punt_unknown (vlib_main_t * vm, u8 is_ip4, u8 is_add)
     tm->punt_unknown6 = is_add;
 }
 
-clib_error_t *
+static clib_error_t *
 tcp_init (vlib_main_t * vm)
 {
   tcp_main_t *tm = vnet_get_tcp_main ();
diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h
index 1fd03c9e7ba..176de2c10ac 100644
--- a/src/vnet/tcp/tcp.h
+++ b/src/vnet/tcp/tcp.h
@@ -225,6 +225,24 @@ typedef struct _sack_scoreboard
 #define tcp_scoreboard_trace_add(_tc, _ack)
 #endif
 
+sack_scoreboard_hole_t *scoreboard_next_rxt_hole (sack_scoreboard_t * sb,
+						  sack_scoreboard_hole_t *
+						  start, u8 have_sent_1_smss,
+						  u8 * can_rescue,
+						  u8 * snd_limited);
+sack_scoreboard_hole_t *scoreboard_get_hole (sack_scoreboard_t * sb,
+					     u32 index);
+
+sack_scoreboard_hole_t *scoreboard_next_hole (sack_scoreboard_t * sb,
+					      sack_scoreboard_hole_t * hole);
+sack_scoreboard_hole_t *scoreboard_prev_hole (sack_scoreboard_t * sb,
+					      sack_scoreboard_hole_t * hole);
+sack_scoreboard_hole_t *scoreboard_first_hole (sack_scoreboard_t * sb);
+sack_scoreboard_hole_t *scoreboard_last_hole (sack_scoreboard_t * sb);
+void scoreboard_clear (sack_scoreboard_t * sb);
+void scoreboard_init (sack_scoreboard_t * sb);
+u8 *format_tcp_scoreboard (u8 * s, va_list * args);
+
 typedef enum _tcp_cc_algorithm_type
 {
   TCP_CC_NEWRENO,
@@ -267,13 +285,13 @@ typedef struct _tcp_connection
   u32 irs;		/**< initial remote sequence */
 
   /* Options */
-  tcp_options_t rcv_opts;	/**< Rx options for connection */
-  tcp_options_t snd_opts;	/**< Tx options for connection */
   u8 snd_opts_len;		/**< Tx options len */
-  u8 rcv_wscale;	/**< Window scale to advertise to peer */
-  u8 snd_wscale;	/**< Window scale to use when sending */
-  u32 tsval_recent;	/**< Last timestamp received */
-  u32 tsval_recent_age;	/**< When last updated tstamp_recent*/
+  u8 rcv_wscale;		/**< Window scale to advertise to peer */
+  u8 snd_wscale;		/**< Window scale to use when sending */
+  u32 tsval_recent;		/**< Last timestamp received */
+  u32 tsval_recent_age;		/**< When last updated tstamp_recent*/
+  tcp_options_t snd_opts;	/**< Tx options for connection */
+  tcp_options_t rcv_opts;	/**< Rx options for connection */
 
   sack_block_t *snd_sacks;	/**< Vector of SACKs to send. XXX Fixed size? */
   sack_scoreboard_t sack_sb;	/**< SACK "scoreboard" that tracks holes */
@@ -338,13 +356,6 @@ tcp_cong_recovery_off (tcp_connection_t * tc)
   tcp_fastrecovery_1_smss_off (tc);
 }
 
-typedef enum
-{
-  TCP_IP4,
-  TCP_IP6,
-  TCP_N_AF,
-} tcp_af_t;
-
 typedef enum _tcp_error
 {
 #define tcp_error(n,s) TCP_ERROR_##n,
@@ -488,11 +499,7 @@ int tcp_configure_v6_source_address_range (vlib_main_t * vm,
 					   ip6_address_t * start,
 					   ip6_address_t * end, u32 table_id);
 void tcp_api_reference (void);
-u8 *format_tcp_connection_id (u8 * s, va_list * args);
 u8 *format_tcp_connection (u8 * s, va_list * args);
-u8 *format_tcp_scoreboard (u8 * s, va_list * args);
-
-u8 *tcp_scoreboard_replay (u8 * s, tcp_connection_t * tc, u8 verbose);
 
 always_inline tcp_connection_t *
 tcp_listener_get (u32 tli)
@@ -640,16 +647,11 @@ tcp_is_lost_fin (tcp_connection_t * tc)
   return 0;
 }
 
-i32 tcp_rcv_wnd_available (tcp_connection_t * tc);
-u32 tcp_snd_space (tcp_connection_t * tc);
-void tcp_update_rcv_wnd (tcp_connection_t * tc);
-
 void tcp_retransmit_first_unacked (tcp_connection_t * tc);
 void tcp_fast_retransmit_no_sack (tcp_connection_t * tc);
 void tcp_fast_retransmit_sack (tcp_connection_t * tc);
 void tcp_fast_retransmit (tcp_connection_t * tc);
 void tcp_cc_init_congestion (tcp_connection_t * tc);
-int tcp_cc_recover (tcp_connection_t * tc);
 void tcp_cc_fastrecovery_exit (tcp_connection_t * tc);
 
 fib_node_index_t tcp_lookup_rmt_in_fib (tcp_connection_t * tc);
@@ -672,12 +674,7 @@ tcp_set_time_now (u32 thread_index)
   return tcp_main.wrk_ctx[thread_index].time_now;
 }
 
-u32 tcp_session_push_header (transport_connection_t * tconn,
-			     vlib_buffer_t * b);
-
-u32
-tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset,
-				u32 max_bytes, vlib_buffer_t ** b);
+u32 tcp_push_header (tcp_connection_t * tconn, vlib_buffer_t * b);
 
 void tcp_connection_timers_init (tcp_connection_t * tc);
 void tcp_connection_timers_reset (tcp_connection_t * tc);
@@ -799,120 +796,13 @@ tcp_timer_is_active (tcp_connection_t * tc, tcp_timers_e timer)
   ASSERT(_tc->state != TCP_STATE_ESTABLISHED 				\
 	 || session_tx_fifo_max_dequeue (&_tc->connection) >= _a)
 
-void
-scoreboard_remove_hole (sack_scoreboard_t * sb,
-			sack_scoreboard_hole_t * hole);
-sack_scoreboard_hole_t *scoreboard_insert_hole (sack_scoreboard_t * sb,
-						u32 prev_index, u32 start,
-						u32 end);
-sack_scoreboard_hole_t *scoreboard_next_rxt_hole (sack_scoreboard_t * sb,
-						  sack_scoreboard_hole_t *
-						  start, u8 have_sent_1_smss,
-						  u8 * can_rescue,
-						  u8 * snd_limited);
-void scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 seq);
-
-always_inline sack_scoreboard_hole_t *
-scoreboard_get_hole (sack_scoreboard_t * sb, u32 index)
-{
-  if (index != TCP_INVALID_SACK_HOLE_INDEX)
-    return pool_elt_at_index (sb->holes, index);
-  return 0;
-}
-
-always_inline sack_scoreboard_hole_t *
-scoreboard_next_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
-{
-  if (hole->next != TCP_INVALID_SACK_HOLE_INDEX)
-    return pool_elt_at_index (sb->holes, hole->next);
-  return 0;
-}
-
-always_inline sack_scoreboard_hole_t *
-scoreboard_prev_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
-{
-  if (hole->prev != TCP_INVALID_SACK_HOLE_INDEX)
-    return pool_elt_at_index (sb->holes, hole->prev);
-  return 0;
-}
-
-always_inline sack_scoreboard_hole_t *
-scoreboard_first_hole (sack_scoreboard_t * sb)
-{
-  if (sb->head != TCP_INVALID_SACK_HOLE_INDEX)
-    return pool_elt_at_index (sb->holes, sb->head);
-  return 0;
-}
-
-always_inline sack_scoreboard_hole_t *
-scoreboard_last_hole (sack_scoreboard_t * sb)
-{
-  if (sb->tail != TCP_INVALID_SACK_HOLE_INDEX)
-    return pool_elt_at_index (sb->holes, sb->tail);
-  return 0;
-}
-
-always_inline void
-scoreboard_clear (sack_scoreboard_t * sb)
-{
-  sack_scoreboard_hole_t *hole;
-  while ((hole = scoreboard_first_hole (sb)))
-    {
-      scoreboard_remove_hole (sb, hole);
-    }
-  ASSERT (sb->head == sb->tail && sb->head == TCP_INVALID_SACK_HOLE_INDEX);
-  ASSERT (pool_elts (sb->holes) == 0);
-  sb->sacked_bytes = 0;
-  sb->last_sacked_bytes = 0;
-  sb->last_bytes_delivered = 0;
-  sb->snd_una_adv = 0;
-  sb->high_sacked = 0;
-  sb->high_rxt = 0;
-  sb->lost_bytes = 0;
-  sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
-}
-
-always_inline u32
-scoreboard_hole_bytes (sack_scoreboard_hole_t * hole)
-{
-  return hole->end - hole->start;
-}
-
-always_inline u32
-scoreboard_hole_index (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
-{
-  ASSERT (!pool_is_free_index (sb->holes, hole - sb->holes));
-  return hole - sb->holes;
-}
-
-always_inline void
-scoreboard_init (sack_scoreboard_t * sb)
-{
-  sb->head = TCP_INVALID_SACK_HOLE_INDEX;
-  sb->tail = TCP_INVALID_SACK_HOLE_INDEX;
-  sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
-}
-
 void tcp_rcv_sacks (tcp_connection_t * tc, u32 ack);
+u8 *tcp_scoreboard_replay (u8 * s, tcp_connection_t * tc, u8 verbose);
 
-always_inline void
-tcp_cc_algo_register (tcp_cc_algorithm_type_e type,
-		      const tcp_cc_algorithm_t * vft)
-{
-  tcp_main_t *tm = vnet_get_tcp_main ();
-  vec_validate (tm->cc_algos, type);
-
-  tm->cc_algos[type] = *vft;
-}
-
-always_inline tcp_cc_algorithm_t *
-tcp_cc_algo_get (tcp_cc_algorithm_type_e type)
-{
-  tcp_main_t *tm = vnet_get_tcp_main ();
-  return &tm->cc_algos[type];
-}
+void tcp_cc_algo_register (tcp_cc_algorithm_type_e type,
+			   const tcp_cc_algorithm_t * vft);
 
-void tcp_cc_init (tcp_connection_t * tc);
+tcp_cc_algorithm_t *tcp_cc_algo_get (tcp_cc_algorithm_type_e type);
 
 /**
  * Push TCP header to buffer
diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c
index 289a6d66e3e..c99d6cca6fe 100644
--- a/src/vnet/tcp/tcp_input.c
+++ b/src/vnet/tcp/tcp_input.c
@@ -123,7 +123,7 @@ tcp_segment_in_rcv_wnd (tcp_connection_t * tc, u32 seq, u32 end_seq)
  * @param to TCP options data structure to be populated
  * @return -1 if parsing failed
  */
-int
+static int
 tcp_options_parse (tcp_header_t * th, tcp_options_t * to)
 {
   const u8 *data;
@@ -534,7 +534,60 @@ tcp_ack_is_cc_event (tcp_connection_t * tc, vlib_buffer_t * b,
   return ((*is_dack || tcp_in_cong_recovery (tc)) && !tcp_is_lost_fin (tc));
 }
 
-void
+static u32
+scoreboard_hole_index (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
+{
+  ASSERT (!pool_is_free_index (sb->holes, hole - sb->holes));
+  return hole - sb->holes;
+}
+
+static u32
+scoreboard_hole_bytes (sack_scoreboard_hole_t * hole)
+{
+  return hole->end - hole->start;
+}
+
+sack_scoreboard_hole_t *
+scoreboard_get_hole (sack_scoreboard_t * sb, u32 index)
+{
+  if (index != TCP_INVALID_SACK_HOLE_INDEX)
+    return pool_elt_at_index (sb->holes, index);
+  return 0;
+}
+
+sack_scoreboard_hole_t *
+scoreboard_next_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
+{
+  if (hole->next != TCP_INVALID_SACK_HOLE_INDEX)
+    return pool_elt_at_index (sb->holes, hole->next);
+  return 0;
+}
+
+sack_scoreboard_hole_t *
+scoreboard_prev_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
+{
+  if (hole->prev != TCP_INVALID_SACK_HOLE_INDEX)
+    return pool_elt_at_index (sb->holes, hole->prev);
+  return 0;
+}
+
+sack_scoreboard_hole_t *
+scoreboard_first_hole (sack_scoreboard_t * sb)
+{
+  if (sb->head != TCP_INVALID_SACK_HOLE_INDEX)
+    return pool_elt_at_index (sb->holes, sb->head);
+  return 0;
+}
+
+sack_scoreboard_hole_t *
+scoreboard_last_hole (sack_scoreboard_t * sb)
+{
+  if (sb->tail != TCP_INVALID_SACK_HOLE_INDEX)
+    return pool_elt_at_index (sb->holes, sb->tail);
+  return 0;
+}
+
+static void
 scoreboard_remove_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
 {
   sack_scoreboard_hole_t *next, *prev;
@@ -569,7 +622,7 @@ scoreboard_remove_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
   pool_put (sb->holes, hole);
 }
 
-sack_scoreboard_hole_t *
+static sack_scoreboard_hole_t *
 scoreboard_insert_hole (sack_scoreboard_t * sb, u32 prev_index,
 			u32 start, u32 end)
 {
@@ -606,7 +659,7 @@ scoreboard_insert_hole (sack_scoreboard_t * sb, u32 prev_index,
   return hole;
 }
 
-void
+static void
 scoreboard_update_bytes (tcp_connection_t * tc, sack_scoreboard_t * sb)
 {
   sack_scoreboard_hole_t *hole, *prev;
@@ -705,7 +758,7 @@ scoreboard_next_rxt_hole (sack_scoreboard_t * sb,
   return hole;
 }
 
-void
+static void
 scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 seq)
 {
   sack_scoreboard_hole_t *hole;
@@ -718,13 +771,41 @@ scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 seq)
   sb->high_rxt = seq;
 }
 
+void
+scoreboard_init (sack_scoreboard_t * sb)
+{
+  sb->head = TCP_INVALID_SACK_HOLE_INDEX;
+  sb->tail = TCP_INVALID_SACK_HOLE_INDEX;
+  sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
+}
+
+void
+scoreboard_clear (sack_scoreboard_t * sb)
+{
+  sack_scoreboard_hole_t *hole;
+  while ((hole = scoreboard_first_hole (sb)))
+    {
+      scoreboard_remove_hole (sb, hole);
+    }
+  ASSERT (sb->head == sb->tail && sb->head == TCP_INVALID_SACK_HOLE_INDEX);
+  ASSERT (pool_elts (sb->holes) == 0);
+  sb->sacked_bytes = 0;
+  sb->last_sacked_bytes = 0;
+  sb->last_bytes_delivered = 0;
+  sb->snd_una_adv = 0;
+  sb->high_sacked = 0;
+  sb->high_rxt = 0;
+  sb->lost_bytes = 0;
+  sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
+}
+
 /**
  * Test that scoreboard is sane after recovery
  *
  * Returns 1 if scoreboard is empty or if first hole beyond
  * snd_una.
  */
-u8
+static u8
 tcp_scoreboard_is_sane_post_recovery (tcp_connection_t * tc)
 {
   sack_scoreboard_hole_t *hole;
@@ -1000,7 +1081,7 @@ tcp_cc_is_spurious_retransmit (tcp_connection_t * tc)
 	  && timestamp_lt (tc->rcv_opts.tsecr, tc->snd_rxt_ts));
 }
 
-int
+static int
 tcp_cc_recover (tcp_connection_t * tc)
 {
   ASSERT (tcp_in_cong_recovery (tc));
@@ -1222,13 +1303,6 @@ partial_ack:
   tcp_fast_retransmit (tc);
 }
 
-void
-tcp_cc_init (tcp_connection_t * tc)
-{
-  tc->cc_algo = tcp_cc_algo_get (TCP_CC_NEWRENO);
-  tc->cc_algo->init (tc);
-}
-
 /**
  * Process incoming ACK
  */
@@ -1415,7 +1489,7 @@ tcp_sack_list_bytes (tcp_connection_t * tc)
 }
 
 /** Enqueue data for delivery to application */
-always_inline int
+static int
 tcp_session_enqueue_data (tcp_connection_t * tc, vlib_buffer_t * b,
 			  u16 data_len)
 {
@@ -1470,7 +1544,7 @@ tcp_session_enqueue_data (tcp_connection_t * tc, vlib_buffer_t * b,
 }
 
 /** Enqueue out-of-order data */
-always_inline int
+static int
 tcp_session_enqueue_ooo (tcp_connection_t * tc, vlib_buffer_t * b,
 			 u16 data_len)
 {
@@ -1648,7 +1722,7 @@ typedef struct
   tcp_connection_t tcp_connection;
 } tcp_rx_trace_t;
 
-u8 *
+static u8 *
 format_tcp_rx_trace (u8 * s, va_list * args)
 {
   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
@@ -1664,7 +1738,7 @@ format_tcp_rx_trace (u8 * s, va_list * args)
   return s;
 }
 
-u8 *
+static u8 *
 format_tcp_rx_trace_short (u8 * s, va_list * args)
 {
   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
@@ -3410,7 +3484,7 @@ do {                                                       	\
 #undef _
 }
 
-clib_error_t *
+static clib_error_t *
 tcp_input_init (vlib_main_t * vm)
 {
   clib_error_t *error = 0;
diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index e29eb6d7f9a..13eac11105c 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -131,30 +131,7 @@ tcp_initial_window_to_advertise (tcp_connection_t * tc)
   return clib_min (tc->rcv_wnd, TCP_WND_MAX);
 }
 
-/**
- * Compute and return window to advertise, scaled as per RFC1323
- */
-u32
-tcp_window_to_advertise (tcp_connection_t * tc, tcp_state_t state)
-{
-  if (state < TCP_STATE_ESTABLISHED)
-    return tcp_initial_window_to_advertise (tc);
-
-  tcp_update_rcv_wnd (tc);
-
-  if (tc->rcv_wnd == 0)
-    {
-      tc->flags |= TCP_CONN_SENT_RCV_WND0;
-    }
-  else
-    {
-      tc->flags &= ~TCP_CONN_SENT_RCV_WND0;
-    }
-
-  return tc->rcv_wnd >> tc->rcv_wscale;
-}
-
-void
+static void
 tcp_update_rcv_wnd (tcp_connection_t * tc)
 {
   i32 observed_wnd;
@@ -201,9 +178,32 @@ tcp_update_rcv_wnd (tcp_connection_t * tc)
 }
 
 /**
+ * Compute and return window to advertise, scaled as per RFC1323
+ */
+static u32
+tcp_window_to_advertise (tcp_connection_t * tc, tcp_state_t state)
+{
+  if (state < TCP_STATE_ESTABLISHED)
+    return tcp_initial_window_to_advertise (tc);
+
+  tcp_update_rcv_wnd (tc);
+
+  if (tc->rcv_wnd == 0)
+    {
+      tc->flags |= TCP_CONN_SENT_RCV_WND0;
+    }
+  else
+    {
+      tc->flags &= ~TCP_CONN_SENT_RCV_WND0;
+    }
+
+  return tc->rcv_wnd >> tc->rcv_wscale;
+}
+
+/**
  * Write TCP options to segment.
  */
-u32
+static u32
 tcp_options_write (u8 * data, tcp_options_t * opts)
 {
   u32 opts_len = 0;
@@ -286,7 +286,7 @@ tcp_options_write (u8 * data, tcp_options_t * opts)
   return opts_len;
 }
 
-always_inline int
+static int
 tcp_make_syn_options (tcp_options_t * opts, u8 wnd_scale)
 {
   u8 len = 0;
@@ -315,7 +315,7 @@ tcp_make_syn_options (tcp_options_t * opts, u8 wnd_scale)
   return len;
 }
 
-always_inline int
+static int
 tcp_make_synack_options (tcp_connection_t * tc, tcp_options_t * opts)
 {
   u8 len = 0;
@@ -350,7 +350,7 @@ tcp_make_synack_options (tcp_connection_t * tc, tcp_options_t * opts)
   return len;
 }
 
-always_inline int
+static int
 tcp_make_established_options (tcp_connection_t * tc, tcp_options_t * opts)
 {
   u8 len = 0;
@@ -444,7 +444,7 @@ tcp_init_mss (tcp_connection_t * tc)
     tc->snd_mss -= TCP_OPTION_LEN_TIMESTAMP;
 }
 
-always_inline int
+static int
 tcp_alloc_tx_buffers (tcp_main_t * tm, u8 thread_index, u16 * n_bufs,
 		      u32 wanted)
 {
@@ -483,7 +483,7 @@ tcp_get_free_buffer_index (tcp_main_t * tm, u32 * bidx)
   return 0;
 }
 
-always_inline void *
+static void *
 tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b)
 {
   if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
@@ -499,7 +499,7 @@ tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b)
   return vlib_buffer_make_headroom (b, MAX_HDRS_LEN);
 }
 
-always_inline void *
+static void *
 tcp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b)
 {
   ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
@@ -516,7 +516,7 @@ tcp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b)
 /**
  * Prepare ACK
  */
-void
+static void
 tcp_make_ack_i (tcp_connection_t * tc, vlib_buffer_t * b, tcp_state_t state,
 		u8 flags)
 {
@@ -666,14 +666,14 @@ tcp_enqueue_to_ip_lookup_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
     }
 }
 
-always_inline void
+static void
 tcp_enqueue_to_ip_lookup_now (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
 			      u8 is_ip4, u32 fib_index)
 {
   tcp_enqueue_to_ip_lookup_i (vm, b, bi, is_ip4, fib_index, 1);
 }
 
-always_inline void
+static void
 tcp_enqueue_to_ip_lookup (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
 			  u8 is_ip4, u32 fib_index)
 {
@@ -716,20 +716,20 @@ tcp_enqueue_to_output_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
     }
 }
 
-always_inline void
+static void
 tcp_enqueue_to_output (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, u8 is_ip4)
 {
   tcp_enqueue_to_output_i (vm, b, bi, is_ip4, 0);
 }
 
-always_inline void
+static void
 tcp_enqueue_to_output_now (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
 			   u8 is_ip4)
 {
   tcp_enqueue_to_output_i (vm, b, bi, is_ip4, 1);
 }
 
-int
+static int
 tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0,
 			 tcp_state_t state, u8 thread_index, u8 is_ip4)
 {
@@ -938,7 +938,7 @@ tcp_send_reset (tcp_connection_t * tc)
   TCP_EVT_DBG (TCP_EVT_RST_SENT, tc);
 }
 
-void
+static void
 tcp_push_ip_hdr (tcp_main_t * tm, tcp_connection_t * tc, vlib_buffer_t * b)
 {
   tcp_header_t *th = vlib_buffer_get_current (b);
@@ -1022,7 +1022,7 @@ tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4)
 /**
  * Flush ip lookup tx frames populated by timer pops
  */
-always_inline void
+static void
 tcp_flush_frame_to_ip_lookup (vlib_main_t * vm, u8 thread_index, u8 is_ip4)
 {
   if (tcp_main.wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4])
@@ -1114,7 +1114,7 @@ tcp_make_state_flags (tcp_connection_t * tc, tcp_state_t next_state)
 /**
  * Push TCP header and update connection variables
  */
-static void
+always_inline void
 tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b,
 		tcp_state_t next_state, u8 compute_opts)
 {
@@ -1153,6 +1153,28 @@ tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b,
   TCP_EVT_DBG (TCP_EVT_PKTIZE, tc);
 }
 
+u32
+tcp_push_header (tcp_connection_t * tc, vlib_buffer_t * b)
+{
+  tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, 0);
+  tc->snd_una_max = tc->snd_nxt;
+  ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd));
+  tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
+  /* If not tracking an ACK, start tracking */
+  if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc))
+    {
+      tc->rtt_ts = tcp_time_now ();
+      tc->rtt_seq = tc->snd_nxt;
+    }
+  if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)))
+    {
+      tcp_retransmit_timer_set (tc);
+      tc->rto_boff = 0;
+    }
+  tcp_trajectory_add_start (b, 3);
+  return 0;
+}
+
 void
 tcp_send_ack (tcp_connection_t * tc)
 {
@@ -1195,7 +1217,7 @@ tcp_timer_delack_handler (u32 index)
  * @return the number of bytes in the segment or 0 if there's nothing to
  *         retransmit
  */
-u32
+static u32
 tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset,
 				u32 max_deq_bytes, vlib_buffer_t ** b)
 {
@@ -1356,7 +1378,7 @@ tcp_rxt_timeout_cc (tcp_connection_t * tc)
   tcp_recovery_on (tc);
 }
 
-static void
+static inline void
 tcp_timer_retransmit_handler_i (u32 index, u8 is_syn)
 {
   tcp_main_t *tm = vnet_get_tcp_main ();
@@ -1755,7 +1777,7 @@ tcp_fast_retransmit (tcp_connection_t * tc)
     tcp_fast_retransmit_no_sack (tc);
 }
 
-always_inline u32
+static u32
 tcp_session_has_ooo_data (tcp_connection_t * tc)
 {
   stream_session_t *s = session_get (tc->c_s_index, tc->c_thread_index);
@@ -1817,7 +1839,7 @@ tcp46_output_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
     }
 }
 
-static inline void
+always_inline void
 tcp_output_push_ip (vlib_main_t * vm, vlib_buffer_t * b0,
 		    tcp_connection_t * tc0, u8 is_ip4)
 {
@@ -1845,7 +1867,7 @@ tcp_output_push_ip (vlib_main_t * vm, vlib_buffer_t * b0,
     }
 }
 
-static inline void
+always_inline void
 tcp_output_handle_packet (tcp_connection_t * tc0, vlib_buffer_t * b0,
 			  u32 * error0, u16 * next0, u8 is_ip4)
 {
@@ -2022,29 +2044,6 @@ VLIB_REGISTER_NODE (tcp6_output_node) =
 
 VLIB_NODE_FUNCTION_MULTIARCH (tcp6_output_node, tcp6_output);
 
-u32
-tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b)
-{
-  tcp_connection_t *tc = (tcp_connection_t *) tconn;
-  tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, 0);
-  tc->snd_una_max = tc->snd_nxt;
-  ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd));
-  tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
-  /* If not tracking an ACK, start tracking */
-  if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc))
-    {
-      tc->rtt_ts = tcp_time_now ();
-      tc->rtt_seq = tc->snd_nxt;
-    }
-  if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)))
-    {
-      tcp_retransmit_timer_set (tc);
-      tc->rto_boff = 0;
-    }
-  tcp_trajectory_add_start (b, 3);
-  return 0;
-}
-
 typedef enum _tcp_reset_next
 {
   TCP_RESET_NEXT_DROP,