fib: midchain adjacency optimisations

Type: improvement

 - inline some common encap fixup functions into the midchain
   rewrite node so we don't incur the cost of the virtual function call
 - change the copy 'guess' from ethernet_header (which will never happen) to an ip4 header
 - add adj-midchain-tx to multiarch sources
 - don't run adj-midchain-tx as a feature, instead put this node as the
   adj's next and at the end of the feature arc.
 - cache the feature arc config index (to save the cache miss going to fetch it)
 - don't check if features are enabled when taking the arc (since we know they are)

the last two changes will also benefit normal adjacencies taking the arc (i.e. for NAT, ACLs, etc)

for IPSec:
 - don't run esp_encrypt as a feature, instead when required insert this
   node into the adj's next and into the end of the feature arc. this
   implies that encrypt is always 'the last feature' run, which is
   symmetric with decrypt always being the first.
 - esp_encrpyt for tunnels has adj-midchain-tx as next node

Change-Id: Ida0af56a704302cf2d7797ded5f118a781e8acb7
Signed-off-by: Neale Ranns <nranns@cisco.com>

1 files changed, 56 insertions, 28 deletions

diff --git a/src/vnet/adj/adj.h b/src/vnet/adj/adj.h
index e22e7ecd58a..a53122711a8 100644
--- a/src/vnet/adj/adj.h
+++ b/src/vnet/adj/adj.h
@@ -181,6 +181,10 @@ typedef enum adj_attr_t_
      * If the midchain were to stack on its FIB entry a loop would form.
      */
     ADJ_ATTR_MIDCHAIN_LOOPED,
+    /**
+     * the fixup function is standard IP4o4 header
+     */
+    ADJ_ATTR_MIDCHAIN_FIXUP_IP4O4_HDR,
 }  adj_attr_t;
 
 #define ADJ_ATTR_NAMES {                                        \
@@ -188,11 +192,12 @@ typedef enum adj_attr_t_
     [ADJ_ATTR_MIDCHAIN_NO_COUNT] = "midchain-no-count",         \
     [ADJ_ATTR_MIDCHAIN_IP_STACK] = "midchain-ip-stack",         \
     [ADJ_ATTR_MIDCHAIN_LOOPED] = "midchain-looped",             \
+    [ADJ_ATTR_MIDCHAIN_FIXUP_IP4O4_HDR] = "midchain-ip4o4-hdr-fixup",   \
 }
 
-#define FOR_EACH_ADJ_ATTR(_attr)                 \
-    for (_attr = ADJ_ATTR_SYNC_WALK_ACTIVE;      \
-	 _attr <= ADJ_ATTR_MIDCHAIN_LOOPED;      \
+#define FOR_EACH_ADJ_ATTR(_attr)                        \
+    for (_attr = ADJ_ATTR_SYNC_WALK_ACTIVE;             \
+	 _attr <= ADJ_ATTR_MIDCHAIN_FIXUP_IP4O4_HDR;    \
 	 _attr++)
 
 /**
@@ -205,6 +210,7 @@ typedef enum adj_flags_t_
     ADJ_FLAG_MIDCHAIN_NO_COUNT = (1 << ADJ_ATTR_MIDCHAIN_NO_COUNT),
     ADJ_FLAG_MIDCHAIN_IP_STACK = (1 << ADJ_ATTR_MIDCHAIN_IP_STACK),
     ADJ_FLAG_MIDCHAIN_LOOPED = (1 << ADJ_ATTR_MIDCHAIN_LOOPED),
+    ADJ_FLAG_MIDCHAIN_FIXUP_IP4O4_HDR = (1 << ADJ_ATTR_MIDCHAIN_FIXUP_IP4O4_HDR),
 }  __attribute__ ((packed)) adj_flags_t;
 
 /**
@@ -227,32 +233,10 @@ typedef struct ip_adjacency_t_
    * has 8 byte alignment requirements.
    */
   fib_node_t ia_node;
-
-  /**
-   * Next hop after ip4-lookup.
-   *  This is not accessed in the rewrite nodes.
-   * 1-bytes
-   */
-  ip_lookup_next_t lookup_next_index;
-
-  /**
-   * link/ether-type
-   * 1 bytes
-   */
-  vnet_link_t ia_link;
-
   /**
-   * The protocol of the neighbor/peer. i.e. the protocol with
-   * which to interpret the 'next-hop' attributes of the sub-types.
-   * 1-bytes
+   * feature [arc] config index
    */
-  fib_protocol_t ia_nh_proto;
-
-  /**
-   * Flags on the adjacency
-   * 1-bytes
-   */
-  adj_flags_t ia_flags;
+  u32 ia_cfg_index;
 
   union
   {
@@ -298,6 +282,10 @@ typedef struct ip_adjacency_t_
        * loop detection.
        */
       fib_node_index_t fei;
+
+      /** spare space */
+      u8 __ia_midchain_pad[4];
+
     } midchain;
     /**
      * IP_LOOKUP_NEXT_GLEAN
@@ -315,12 +303,14 @@ typedef struct ip_adjacency_t_
 
   CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
 
-  /* Rewrite in second/third cache lines */
+  /** Rewrite in second and third cache lines */
   VNET_DECLARE_REWRITE;
 
   /**
    * more control plane members that do not fit on the first cacheline
    */
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline3);
+
   /**
    * A sorted vector of delegates
    */
@@ -330,6 +320,37 @@ typedef struct ip_adjacency_t_
    * The VLIB node in which this adj is used to forward packets
    */
   u32 ia_node_index;
+
+  /**
+   * Next hop after ip4-lookup.
+   *  This is not accessed in the rewrite nodes.
+   * 1-bytes
+   */
+  ip_lookup_next_t lookup_next_index;
+
+  /**
+   * link/ether-type
+   * 1 bytes
+   */
+  vnet_link_t ia_link;
+
+  /**
+   * The protocol of the neighbor/peer. i.e. the protocol with
+   * which to interpret the 'next-hop' attributes of the sub-types.
+   * 1-bytes
+   */
+  fib_protocol_t ia_nh_proto;
+
+  /**
+   * Flags on the adjacency
+   * 1-bytes
+   */
+  adj_flags_t ia_flags;
+
+  /**
+   * Free space on the fourth cacheline (not used in the DP)
+   */
+  u8 __ia_pad[48];
 } ip_adjacency_t;
 
 STATIC_ASSERT ((STRUCT_OFFSET_OF (ip_adjacency_t, cacheline0) == 0),
@@ -337,6 +358,13 @@ STATIC_ASSERT ((STRUCT_OFFSET_OF (ip_adjacency_t, cacheline0) == 0),
 STATIC_ASSERT ((STRUCT_OFFSET_OF (ip_adjacency_t, cacheline1) ==
 		CLIB_CACHE_LINE_BYTES),
 	       "IP adjacency cacheline 1 is more than one cacheline size offset");
+#if defined __x86_64__
+STATIC_ASSERT ((STRUCT_OFFSET_OF (ip_adjacency_t, cacheline3) ==
+		3 * CLIB_CACHE_LINE_BYTES),
+	       "IP adjacency cacheline 3 is more than one cacheline size offset");
+/* An adj fits into 4 cachelines on your average machine */
+STATIC_ASSERT_SIZEOF (ip_adjacency_t, 4 * 64);
+#endif
 
 /**
  * @brief