l2: allocate l2fib only when needed

Currently l2 fib allocates 512MB hash table unconditionally on startup.
This patch postpones table creation up to the point where first
interface is put into l2 mode or mac entry is added.

In addition it reduces default table size to 128MB and increases number
of buckets 4 times. This default setting should be enough to keep 1M mac
entries.

Also, new startup.conf section is added which allows user to change memory
and bucket size. .i.e:

l2fib {
  table-size 512M
  num-buckets 524288
}

Type: improvement
Change-Id: I2a29209aa3545181f0087544c97a54d8157b6ec5
Signed-off-by: Damjan Marion <damarion@cisco.com>

5 files changed, 99 insertions, 10 deletions

diff --git a/src/vnet/l2/l2_bd.c b/src/vnet/l2/l2_bd.c
index e6fb6223d93..94852c90769 100644
--- a/src/vnet/l2/l2_bd.c
+++ b/src/vnet/l2/l2_bd.c
@@ -1226,9 +1226,13 @@ int
 bd_add_del (l2_bridge_domain_add_del_args_t * a)
 {
   bd_main_t *bdm = &bd_main;
+  l2fib_main_t *fm = &l2fib_main;
   vlib_main_t *vm = bdm->vlib_main;
   int rv = 0;
 
+  if (fm->mac_table_initialized == 0)
+    l2fib_table_init ();
+
   u32 bd_index = bd_find_index (bdm, a->bd_id);
   if (a->is_add)
     {
diff --git a/src/vnet/l2/l2_fib.c b/src/vnet/l2/l2_fib.c
index 983e0218bfc..0ce4eb84a6b 100644
--- a/src/vnet/l2/l2_fib.c
+++ b/src/vnet/l2/l2_fib.c
@@ -285,6 +285,12 @@ show_l2fib (vlib_main_t * vm,
 	break;
     }
 
+  if (msm->mac_table_initialized == 0)
+    {
+      vlib_cli_output (vm, "no l2fib entries");
+      return 0;
+    }
+
   BV (clib_bihash_foreach_key_value_pair)
     (&msm->mac_table, l2fib_show_walk_cb, &ctx);
 
@@ -341,6 +347,18 @@ VLIB_CLI_COMMAND (show_l2fib_cli, static) = {
 };
 /* *INDENT-ON* */
 
+void
+l2fib_table_init (void)
+{
+  l2fib_main_t *mp = &l2fib_main;
+
+  if (mp->mac_table_initialized == 1)
+    return;
+
+  BV (clib_bihash_init) (&mp->mac_table, "l2fib mac table",
+			 mp->mac_table_n_buckets, mp->mac_table_memory_size);
+  mp->mac_table_initialized = 1;
+}
 
 /* Remove all entries from the l2fib */
 void
@@ -348,10 +366,12 @@ l2fib_clear_table (void)
 {
   l2fib_main_t *mp = &l2fib_main;
 
+  if (mp->mac_table_initialized == 0)
+    return;
+
   /* Remove all entries */
   BV (clib_bihash_free) (&mp->mac_table);
-  BV (clib_bihash_init) (&mp->mac_table, "l2fib mac table",
-			 L2FIB_NUM_BUCKETS, L2FIB_MEMORY_SIZE);
+  l2fib_table_init ();
   l2learn_main.global_learn_count = 0;
 }
 
@@ -412,6 +432,9 @@ l2fib_add_entry (const u8 * mac, u32 bd_index,
   l2learn_main_t *lm = &l2learn_main;
   BVT (clib_bihash_kv) kv;
 
+  if (fm->mac_table_initialized == 0)
+    l2fib_table_init ();
+
   /* set up key */
   key.raw = l2fib_make_key (mac, bd_index);
   kv.key = key.raw;
@@ -555,7 +578,6 @@ static clib_error_t *
 l2fib_test_command_fn (vlib_main_t * vm,
 		       unformat_input_t * input, vlib_cli_command_t * cmd)
 {
-  clib_error_t *error = 0;
   u8 mac[6], save_mac[6];
   u32 bd_index = 0;
   u32 sw_if_index = 8;
@@ -606,6 +628,9 @@ l2fib_test_command_fn (vlib_main_t * vm,
       BVT (clib_bihash_kv) kv;
       l2fib_main_t *mp = &l2fib_main;
 
+      if (mp->mac_table_initialized == 0)
+	return clib_error_return (0, "mac table is not initialized");
+
       clib_memcpy_fast (mac, save_mac, 6);
 
       for (i = 0; i < count; i++)
@@ -631,7 +656,7 @@ l2fib_test_command_fn (vlib_main_t * vm,
 	}
     }
 
-  return error;
+  return 0;
 }
 
 /*?
@@ -692,6 +717,9 @@ l2fib_del_entry (const u8 * mac, u32 bd_index, u32 sw_if_index)
   l2fib_main_t *mp = &l2fib_main;
   BVT (clib_bihash_kv) kv;
 
+  if (mp->mac_table_initialized == 0)
+    return 1;
+
   /* set up key */
   kv.key = l2fib_make_key (mac, bd_index);
 
@@ -1273,10 +1301,11 @@ l2fib_init (vlib_main_t * vm)
 
   mp->vlib_main = vm;
   mp->vnet_main = vnet_get_main ();
-
-  /* Create the hash table  */
-  BV (clib_bihash_init) (&mp->mac_table, "l2fib mac table",
-			 L2FIB_NUM_BUCKETS, L2FIB_MEMORY_SIZE);
+  if (mp->mac_table_n_buckets == 0)
+    mp->mac_table_n_buckets = L2FIB_NUM_BUCKETS;
+  if (mp->mac_table_memory_size == 0)
+    mp->mac_table_memory_size = L2FIB_MEMORY_SIZE;
+  mp->mac_table_initialized = 0;
 
   /* verify the key constructor is good, since it is endian-sensitive */
   clib_memset (test_mac, 0, sizeof (test_mac));
@@ -1291,6 +1320,39 @@ l2fib_init (vlib_main_t * vm)
 
 VLIB_INIT_FUNCTION (l2fib_init);
 
+static clib_error_t *
+lfib_config (vlib_main_t * vm, unformat_input_t * input)
+{
+  l2fib_main_t *lm = &l2fib_main;
+  uword table_size = ~0;
+  u32 n_buckets = ~0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "table-size %U", unformat_memory_size,
+		    &table_size))
+	;
+      else if (unformat (input, "num-buckets %u", &n_buckets))
+	;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+
+  if (n_buckets != ~0)
+    {
+      if (!is_pow2 (n_buckets))
+	return clib_error_return (0, "num-buckets must be power of 2");
+      lm->mac_table_n_buckets = n_buckets;
+    }
+
+  if (table_size != ~0)
+    lm->mac_table_memory_size = table_size;
+  return 0;
+}
+
+VLIB_CONFIG_FUNCTION (lfib_config, "l2fib");
+
 /*
  * fd.io coding-style-patch-verification: ON
  *
diff --git a/src/vnet/l2/l2_fib.h b/src/vnet/l2/l2_fib.h
index a1dbc9db5c7..41ef4ab3461 100644
--- a/src/vnet/l2/l2_fib.h
+++ b/src/vnet/l2/l2_fib.h
@@ -24,8 +24,8 @@
 /*
  * The size of the hash table
  */
-#define L2FIB_NUM_BUCKETS (64 * 1024)
-#define L2FIB_MEMORY_SIZE (512<<20)
+#define L2FIB_NUM_BUCKETS (256 * 1024)
+#define L2FIB_MEMORY_SIZE (128<<20)
 
 /* Ager scan interval is 1 minute for aging */
 #define L2FIB_AGE_SCAN_INTERVAL		(60.0)
@@ -45,6 +45,15 @@ typedef struct
   /* hash table */
   BVT (clib_bihash) mac_table;
 
+  /* number of buckets in the hash table */
+  uword mac_table_n_buckets;
+
+  /* hash table memory size */
+  uword mac_table_memory_size;
+
+  /* hash table initialized */
+  u8 mac_table_initialized;
+
   /* per swif vector of sequence number for interface based flush of MACs */
   u8 *swif_seq_num;
 
@@ -419,6 +428,8 @@ l2fib_lookup_4 (BVT (clib_bihash) * mac_table,
 
 void l2fib_clear_table (void);
 
+void l2fib_table_init (void);
+
 void
 l2fib_add_entry (const u8 * mac,
 		 u32 bd_index,
diff --git a/src/vnet/l2/l2_input.c b/src/vnet/l2/l2_input.c
index 542bd42c4c2..bfb0751f4e2 100644
--- a/src/vnet/l2/l2_input.c
+++ b/src/vnet/l2/l2_input.c
@@ -598,6 +598,9 @@ set_int_l2_mode (vlib_main_t * vm, vnet_main_t * vnet_main,	/*           */
   hi = vnet_get_sup_hw_interface (vnet_main, sw_if_index);
   config = l2input_intf_config (sw_if_index);
 
+  if (l2fib_main.mac_table_initialized == 0)
+    l2fib_table_init ();
+
   if (config->bridge)
     {
       /* Interface is already in bridge mode. Undo the existing config. */
diff --git a/src/vpp/conf/startup.conf b/src/vpp/conf/startup.conf
index 56610e2e315..12679da778c 100644
--- a/src/vpp/conf/startup.conf
+++ b/src/vpp/conf/startup.conf
@@ -189,3 +189,12 @@ cpu {
     # per-node-counters on | off, defaults to none
     # update-interval <f64-seconds>, sets the segment scrape / update interval
 # }
+
+## L2 FIB
+# l2fib {
+    ## l2fib hash table size.
+    #  table-size 512M
+
+    ## l2fib hash table number of buckets. Must be power of 2.
+    #  num-buckets 524288
+# }