MTRIE Optimisations 2

1) 16-8-8 stride. Reduce trie depth walk traded with increased memory in the top PLY.
2) separate the vector of protocol-independent (PI) fib_table_t with the vector of protocol dependent (PD) FIBs. PD FIBs are large structures, we don't want to burn the memory for ech PD type
3) Go straight to the PD FIB in the data-path thus avoiding an indirection through, e.g., a PLY pool.

Change-Id: I800d1ed0b2049040d5da95213f3ed6b12bdd78b7
Signed-off-by: Neale Ranns <nranns@cisco.com>

1 files changed, 87 insertions, 19 deletions

diff --git a/src/vnet/ip/ip4_mtrie.h b/src/vnet/ip/ip4_mtrie.h
index 128195d3..be262c2c 100644
--- a/src/vnet/ip/ip4_mtrie.h
+++ b/src/vnet/ip/ip4_mtrie.h
@@ -47,16 +47,43 @@
 
 /* ip4 fib leafs: 4 ply 8-8-8-8 mtrie.
    1 + 2*adj_index for terminal leaves.
-   0 + 2*next_ply_index for non-terminals.
+   0 + 2*next_ply_index for non-terminals, i.e. PLYs
    1 => empty (adjacency index of zero is special miss adjacency). */
 typedef u32 ip4_fib_mtrie_leaf_t;
 
 #define IP4_FIB_MTRIE_LEAF_EMPTY (1 + 2*0)
 
 /**
+ * @brief the 16 way stride that is the top PLY of the mtrie
+ * We do not maintain the count of 'real' leaves in this PLY, since
+ * it is never removed. The FIB will destroy the mtrie and the ply once
+ * the FIB is destroyed.
+ */
+#define PLY_16_SIZE (1<<16)
+typedef struct ip4_fib_mtrie_16_ply_t_
+{
+  /**
+   * The leaves/slots/buckets to be filed with leafs
+   */
+  union
+  {
+    ip4_fib_mtrie_leaf_t leaves[PLY_16_SIZE];
+
+#ifdef CLIB_HAVE_VEC128
+    u32x4 leaves_as_u32x4[PLY_16_SIZE / 4];
+#endif
+  };
+
+  /**
+   * Prefix length for terminal leaves.
+   */
+  u8 dst_address_bits_of_leaves[PLY_16_SIZE];
+} ip4_fib_mtrie_16_ply_t;
+
+/**
  * @brief One ply of the 4 ply mtrie fib.
  */
-typedef struct
+typedef struct ip4_fib_mtrie_8_ply_t_
 {
   /**
    * The leaves/slots/buckets to be filed with leafs
@@ -90,34 +117,72 @@ typedef struct
   /* Pad to cache line boundary. */
   u8 pad[CLIB_CACHE_LINE_BYTES - 2 * sizeof (i32)];
 }
-ip4_fib_mtrie_ply_t;
+ip4_fib_mtrie_8_ply_t;
 
-STATIC_ASSERT (0 == sizeof (ip4_fib_mtrie_ply_t) % CLIB_CACHE_LINE_BYTES,
+STATIC_ASSERT (0 == sizeof (ip4_fib_mtrie_8_ply_t) % CLIB_CACHE_LINE_BYTES,
 	       "IP4 Mtrie ply cache line");
 
+/**
+ * @brief The mutiway-TRIE.
+ * There is no data associated with the mtrie apart from the top PLY
+ */
 typedef struct
 {
-  /* Pool of plies.  Index zero is root ply. */
-  ip4_fib_mtrie_ply_t *ply_pool;
+  /**
+   * Embed the PLY with the mtrie struct. This means that the Data-plane
+   * 'get me the mtrie' returns the first ply, and not an indirect 'pointer'
+   * to it. therefore no cachline misses in the data-path.
+   */
+  ip4_fib_mtrie_16_ply_t root_ply;
 } ip4_fib_mtrie_t;
 
-void ip4_fib_mtrie_init (ip4_fib_mtrie_t * m);
+/**
+ * @brief Initialise an mtrie
+ */
+void ip4_mtrie_init (ip4_fib_mtrie_t * m);
 
-struct ip4_fib_t;
+/**
+ * @brief Free an mtrie, It must be emty when free'd
+ */
+void ip4_mtrie_free (ip4_fib_mtrie_t * m);
 
-void ip4_fib_mtrie_add_del_route (struct ip4_fib_t *f,
-				  ip4_address_t dst_address,
-				  u32 dst_address_length,
-				  u32 adj_index, u32 is_del);
+/**
+ * @brief Add a route/rntry to the mtrie
+ */
+void ip4_fib_mtrie_route_add (ip4_fib_mtrie_t * m,
+			      const ip4_address_t * dst_address,
+			      u32 dst_address_length, u32 adj_index);
+/**
+ * @brief remove a route/rntry to the mtrie
+ */
+void ip4_fib_mtrie_route_del (ip4_fib_mtrie_t * m,
+			      const ip4_address_t * dst_address,
+			      u32 dst_address_length,
+			      u32 adj_index,
+			      u32 cover_address_length, u32 cover_adj_index);
 
+/**
+ * @brief Format/display the contents of the mtrie
+ */
 format_function_t format_ip4_fib_mtrie;
 
+/**
+ * @brief A global pool of 8bit stride plys
+ */
+extern ip4_fib_mtrie_8_ply_t *ip4_ply_pool;
+
+/**
+ * Is the leaf terminal (i.e. an LB index) or non-terminak (i.e. a PLY index)
+ */
 always_inline u32
 ip4_fib_mtrie_leaf_is_terminal (ip4_fib_mtrie_leaf_t n)
 {
   return n & 1;
 }
 
+/**
+ * From the stored slot value extract the LB index value
+ */
 always_inline u32
 ip4_fib_mtrie_leaf_get_adj_index (ip4_fib_mtrie_leaf_t n)
 {
@@ -125,35 +190,38 @@ ip4_fib_mtrie_leaf_get_adj_index (ip4_fib_mtrie_leaf_t n)
   return n >> 1;
 }
 
-/* Lookup step.  Processes 1 byte of 4 byte ip4 address. */
+/**
+ * @brief Lookup step.  Processes 1 byte of 4 byte ip4 address.
+ */
 always_inline ip4_fib_mtrie_leaf_t
 ip4_fib_mtrie_lookup_step (const ip4_fib_mtrie_t * m,
 			   ip4_fib_mtrie_leaf_t current_leaf,
 			   const ip4_address_t * dst_address,
 			   u32 dst_address_byte_index)
 {
-  ip4_fib_mtrie_ply_t *ply;
+  ip4_fib_mtrie_8_ply_t *ply;
+
   uword current_is_terminal = ip4_fib_mtrie_leaf_is_terminal (current_leaf);
 
   if (!current_is_terminal)
     {
-      ply = m->ply_pool + (current_leaf >> 1);
+      ply = ip4_ply_pool + (current_leaf >> 1);
       return (ply->leaves[dst_address->as_u8[dst_address_byte_index]]);
     }
 
   return current_leaf;
 }
 
-/* Lookup step.  Processes 1 byte of 4 byte ip4 address. */
+/**
+ * @brief Lookup step number 1.  Processes 2 bytes of 4 byte ip4 address.
+ */
 always_inline ip4_fib_mtrie_leaf_t
 ip4_fib_mtrie_lookup_step_one (const ip4_fib_mtrie_t * m,
 			       const ip4_address_t * dst_address)
 {
   ip4_fib_mtrie_leaf_t next_leaf;
-  ip4_fib_mtrie_ply_t *ply;
 
-  ply = m->ply_pool;
-  next_leaf = ply->leaves[dst_address->as_u8[0]];
+  next_leaf = m->root_ply.leaves[dst_address->as_u16[0]];
 
   return next_leaf;
 }