Improve svm fifo and tcp tx path performance (VPP-846)

- multiarch on svm fifo
- avoid ip lookup on tx

Change-Id: Iab0d85204a710979417bca1d692cc47877131203
Signed-off-by: Florin Coras <fcoras@cisco.com>
Signed-off-by: Dave Barach <dbarach@cisco.com>

6 files changed, 137 insertions, 25 deletions

diff --git a/src/vnet/session/node.c b/src/vnet/session/node.c
index c0ab1bf096f..b24f5fd994d 100644
--- a/src/vnet/session/node.c
+++ b/src/vnet/session/node.c
@@ -248,6 +248,11 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  ASSERT (bi0);
 	  _vec_len (smm->tx_buffers[thread_index]) = n_bufs;
 
+	  /* usual speculation, or the enqueue_x1 macro will barf */
+	  to_next[0] = bi0;
+	  to_next += 1;
+	  n_left_to_next -= 1;
+
 	  b0 = vlib_get_buffer (vm, bi0);
 	  b0->error = 0;
 	  b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID
@@ -255,10 +260,6 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  b0->current_data = 0;
 	  b0->total_length_not_including_first_buffer = 0;
 
-	  /* RX on the local interface. tx in default fib */
-	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
-	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-
 	  len_to_deq0 = clib_min (left_to_snd0, deq_per_buf);
 
 	  data0 = vlib_buffer_make_headroom (b0, MAX_HDRS_LEN);
@@ -307,10 +308,6 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  }));
 	  /* *INDENT-ON* */
 
-	  /* usual speculation, or the enqueue_x1 macro will barf */
-	  to_next[0] = bi0;
-	  to_next += 1;
-	  n_left_to_next -= 1;
 
 	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
 	  if (PREDICT_FALSE (n_trace > 0))
diff --git a/src/vnet/session/transport.h b/src/vnet/session/transport.h
index 04bd5ca0f29..561a92575cb 100644
--- a/src/vnet/session/transport.h
+++ b/src/vnet/session/transport.h
@@ -30,13 +30,16 @@ typedef struct _transport_connection
   ip46_address_t lcl_ip;	/**< Local IP */
   u16 lcl_port;			/**< Local port */
   u16 rmt_port;			/**< Remote port */
-  u8 proto;			/**< Transport protocol id (also session type) */
+  u8 proto;			/**< Protocol id (also session type) */
 
   u32 s_index;			/**< Parent session index */
   u32 c_index;			/**< Connection index in transport pool */
   u8 is_ip4;			/**< Flag if IP4 connection */
   u32 thread_index;		/**< Worker-thread index */
 
+  fib_node_index_t rmt_fei;	/**< FIB entry index for rmt */
+  dpo_id_t rmt_dpo;		/**< Forwarding DPO for rmt */
+
 #if TRANSPORT_DEBUG
   elog_track_t elog_track;	/**< Event logging */
   u32 cc_stat_tstamp;		/**< CC stats timestamp */
@@ -59,6 +62,8 @@ typedef struct _transport_connection
 #define c_thread_index connection.thread_index
 #define c_elog_track connection.elog_track
 #define c_cc_stat_tstamp connection.cc_stat_tstamp
+#define c_rmt_fei connection.rmt_fei
+#define c_rmt_dpo connection.rmt_dpo
 } transport_connection_t;
 
 /*
diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c
index 5c554bac5a9..4e85eb3fc93 100644
--- a/src/vnet/tcp/tcp.c
+++ b/src/vnet/tcp/tcp.c
@@ -16,6 +16,7 @@
 #include <vnet/tcp/tcp.h>
 #include <vnet/session/session.h>
 #include <vnet/fib/fib.h>
+#include <vnet/dpo/load_balance.h>
 #include <math.h>
 
 tcp_main_t tcp_main;
@@ -342,6 +343,99 @@ tcp_connection_timers_reset (tcp_connection_t * tc)
     }
 }
 
+typedef struct ip4_tcp_hdr
+{
+  ip4_header_t ip;
+  tcp_header_t tcp;
+} ip4_tcp_hdr_t;
+
+typedef struct ip6_tcp_hdr
+{
+  ip6_header_t ip;
+  tcp_header_t tcp;
+} ip6_tcp_hdr_t;
+
+static void
+tcp_connection_select_lb_bucket (tcp_connection_t * tc, const dpo_id_t * dpo,
+				 dpo_id_t * result)
+{
+  const dpo_id_t *choice;
+  load_balance_t *lb;
+  int hash;
+
+  lb = load_balance_get (dpo->dpoi_index);
+  if (tc->c_is_ip4)
+    {
+      ip4_tcp_hdr_t hdr;
+      memset (&hdr, 0, sizeof (hdr));
+      hdr.ip.protocol = IP_PROTOCOL_TCP;
+      hdr.ip.address_pair.src.as_u32 = tc->c_lcl_ip.ip4.as_u32;
+      hdr.ip.address_pair.dst.as_u32 = tc->c_rmt_ip.ip4.as_u32;
+      hdr.tcp.src_port = tc->c_lcl_port;
+      hdr.tcp.dst_port = tc->c_rmt_port;
+      hash = ip4_compute_flow_hash (&hdr.ip, lb->lb_hash_config);
+    }
+  else
+    {
+      ip6_tcp_hdr_t hdr;
+      memset (&hdr, 0, sizeof (hdr));
+      hdr.ip.protocol = IP_PROTOCOL_TCP;
+      clib_memcpy (&hdr.ip.src_address, &tc->c_lcl_ip.ip6,
+		   sizeof (ip6_address_t));
+      clib_memcpy (&hdr.ip.dst_address, &tc->c_rmt_ip.ip6,
+		   sizeof (ip6_address_t));
+      hdr.tcp.src_port = tc->c_lcl_port;
+      hdr.tcp.dst_port = tc->c_rmt_port;
+      hash = ip6_compute_flow_hash (&hdr.ip, lb->lb_hash_config);
+    }
+  choice = load_balance_get_bucket_i (lb, hash & lb->lb_n_buckets_minus_1);
+  dpo_copy (result, choice);
+}
+
+fib_node_index_t
+tcp_lookup_rmt_in_fib (tcp_connection_t * tc)
+{
+  fib_prefix_t prefix;
+
+  clib_memcpy (&prefix.fp_addr, &tc->c_rmt_ip, sizeof (prefix.fp_addr));
+  prefix.fp_proto = tc->c_is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6;
+  prefix.fp_len = tc->c_is_ip4 ? 32 : 128;
+  return fib_table_lookup (0, &prefix);
+}
+
+static int
+tcp_connection_stack_on_fib_entry (tcp_connection_t * tc)
+{
+  dpo_id_t choice = DPO_INVALID;
+  u32 output_node_index;
+  fib_entry_t *fe;
+
+  fe = fib_entry_get (tc->c_rmt_fei);
+  if (fe->fe_lb.dpoi_type != DPO_LOAD_BALANCE)
+    return -1;
+
+  tcp_connection_select_lb_bucket (tc, &fe->fe_lb, &choice);
+
+  output_node_index =
+    tc->c_is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
+  dpo_stack_from_node (output_node_index, &tc->c_rmt_dpo, &choice);
+  return 0;
+}
+
+/** Stack tcp connection on peer's fib entry.
+ *
+ * This ultimately populates the dpo the connection will use to send packets.
+ */
+static void
+tcp_connection_fib_attach (tcp_connection_t * tc)
+{
+  tc->c_rmt_fei = tcp_lookup_rmt_in_fib (tc);
+
+  ASSERT (tc->c_rmt_fei != FIB_NODE_INDEX_INVALID);
+
+  tcp_connection_stack_on_fib_entry (tc);
+}
+
 /** Initialize tcp connection variables
  *
  * Should be called after having received a msg from the peer, i.e., a SYN or
@@ -353,6 +447,7 @@ tcp_connection_init_vars (tcp_connection_t * tc)
   tcp_init_mss (tc);
   scoreboard_init (&tc->sack_sb);
   tcp_cc_init (tc);
+  tcp_connection_fib_attach (tc);
 }
 
 int
@@ -361,7 +456,8 @@ tcp_connection_open (ip46_address_t * rmt_addr, u16 rmt_port, u8 is_ip4)
   tcp_main_t *tm = vnet_get_tcp_main ();
   tcp_connection_t *tc;
   fib_prefix_t prefix;
-  u32 fei, sw_if_index;
+  fib_node_index_t fei;
+  u32 sw_if_index;
   ip46_address_t lcl_addr;
   u16 lcl_port;
 
@@ -985,8 +1081,6 @@ tcp_main_enable (vlib_main_t * vm)
   vec_validate (tm->timer_wheels, num_threads - 1);
   tcp_initialize_timer_wheels (tm);
 
-//  vec_validate (tm->delack_connections, num_threads - 1);
-
   /* Initialize clocks per tick for TCP timestamp. Used to compute
    * monotonically increasing timestamps. */
   tm->tstamp_ticks_per_clock = vm->clib_time.seconds_per_clock
diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h
index e83987182fa..12d804b82f6 100644
--- a/src/vnet/tcp/tcp.h
+++ b/src/vnet/tcp/tcp.h
@@ -28,6 +28,7 @@
 #define THZ (u32) (1/TCP_TICK)		/**< TCP tick frequency */
 #define TCP_TSTAMP_RESOLUTION TCP_TICK	/**< Time stamp resolution */
 #define TCP_PAWS_IDLE 24 * 24 * 60 * 60 * THZ /**< 24 days */
+#define TCP_FIB_RECHECK_PERIOD	1 * THZ	/**< Recheck every 1s */
 #define TCP_MAX_OPTION_SPACE 40
 
 #define TCP_DUPACK_THRESHOLD 	3
@@ -256,6 +257,7 @@ typedef struct _tcp_connection
 
   u16 mss;		/**< Our max seg size that includes options */
   u32 limited_transmit;	/**< snd_nxt when limited transmit starts */
+  u32 last_fib_check;	/**< Last time we checked fib route for peer */
 } tcp_connection_t;
 
 struct _tcp_cc_algorithm
@@ -528,6 +530,8 @@ void tcp_cc_init_congestion (tcp_connection_t * tc);
 int tcp_cc_recover (tcp_connection_t * tc);
 void tcp_cc_fastrecovery_exit (tcp_connection_t * tc);
 
+fib_node_index_t tcp_lookup_rmt_in_fib (tcp_connection_t * tc);
+
 /* Made public for unit testing only */
 void tcp_update_sack_list (tcp_connection_t * tc, u32 start, u32 end);
 
diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index 554a981d924..41bebcb34af 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -22,17 +22,14 @@ vlib_node_registration_t tcp6_output_node;
 typedef enum _tcp_output_nect
 {
   TCP_OUTPUT_NEXT_DROP,
-  TCP_OUTPUT_NEXT_IP_LOOKUP,
   TCP_OUTPUT_N_NEXT
 } tcp_output_next_t;
 
 #define foreach_tcp4_output_next              	\
   _ (DROP, "error-drop")                        \
-  _ (IP_LOOKUP, "ip4-lookup")
 
 #define foreach_tcp6_output_next              	\
   _ (DROP, "error-drop")                        \
-  _ (IP_LOOKUP, "ip6-lookup")
 
 static char *tcp_error_strings[] = {
 #define tcp_error(n,s) s,
@@ -1451,7 +1448,7 @@ tcp46_output_inline (vlib_main_t * vm,
 	  tcp_connection_t *tc0;
 	  tcp_tx_trace_t *t0;
 	  tcp_header_t *th0 = 0;
-	  u32 error0 = TCP_ERROR_PKTS_SENT, next0 = TCP_OUTPUT_NEXT_IP_LOOKUP;
+	  u32 error0 = TCP_ERROR_PKTS_SENT, next0 = TCP_OUTPUT_NEXT_DROP;
 
 	  bi0 = from[0];
 	  to_next[0] = bi0;
@@ -1530,10 +1527,26 @@ tcp46_output_inline (vlib_main_t * vm,
 	      tc0->rto_boff = 0;
 	    }
 
-	  /* set fib index to default and lookup node */
-	  /* XXX network virtualization (vrf/vni) */
-	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
-	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+	  /* Make sure we haven't lost route to our peer */
+	  if (PREDICT_FALSE (tc0->last_fib_check
+			     < tc0->snd_opts.tsval + TCP_FIB_RECHECK_PERIOD))
+	    {
+	      if (PREDICT_TRUE
+		  (tc0->c_rmt_fei == tcp_lookup_rmt_in_fib (tc0)))
+		{
+		  tc0->last_fib_check = tc0->snd_opts.tsval;
+		}
+	      else
+		{
+		  clib_warning ("lost connection to peer");
+		  tcp_connection_reset (tc0);
+		  goto done;
+		}
+	    }
+
+	  /* Use pre-computed dpo to set next node */
+	  next0 = tc0->c_rmt_dpo.dpoi_next_node;
+	  vnet_buffer (b0)->ip.adj_index[VLIB_TX] = tc0->c_rmt_dpo.dpoi_index;
 
 	  b0->flags |= VNET_BUFFER_LOCALLY_ORIGINATED;
 	done:
diff --git a/src/vnet/tcp/tcp_packet.h b/src/vnet/tcp/tcp_packet.h
index 4f28cf32708..a6f62ee16d5 100644
--- a/src/vnet/tcp/tcp_packet.h
+++ b/src/vnet/tcp/tcp_packet.h
@@ -144,12 +144,11 @@ typedef struct
 {
   u8 flags;		/** Option flags, see above */
 
-  /* Received options */
-  u16 mss;		/**< Maximum segment size advertised by peer */
-  u8 wscale;		/**< Window scale advertised by peer */
-  u32 tsval;		/**< Peer's timestamp value */
+  u16 mss;		/**< Maximum segment size advertised */
+  u8 wscale;		/**< Window scale advertised */
+  u32 tsval;		/**< Timestamp value */
   u32 tsecr;		/**< Echoed/reflected time stamp */
-  sack_block_t *sacks;	/**< SACK blocks received */
+  sack_block_t *sacks;	/**< SACK blocks */
   u8 n_sack_blocks;	/**< Number of SACKs blocks */
 } tcp_options_t;