1 files changed, 69 insertions, 21 deletions

diff --git a/src/vppinfra/bihash_template.c b/src/vppinfra/bihash_template.c
index a6fa6267c79..d488b1a659c 100644
--- a/src/vppinfra/bihash_template.c
+++ b/src/vppinfra/bihash_template.c
@@ -106,8 +106,10 @@ static inline void *BV (alloc_aligned) (BVT (clib_bihash) * h, uword nbytes)
       void *base, *rv;
       uword alloc = alloc_arena_next (h) - alloc_arena_mapped (h);
       int mmap_flags = MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS;
+#if __linux__
       int mmap_flags_huge = (mmap_flags | MAP_HUGETLB | MAP_LOCKED |
 			     BIHASH_LOG2_HUGEPAGE_SIZE << MAP_HUGE_SHIFT);
+#endif /* __linux__ */
 
       /* new allocation is 25% of existing one */
       if (alloc_arena_mapped (h) >> 2 > alloc)
@@ -118,7 +120,11 @@ static inline void *BV (alloc_aligned) (BVT (clib_bihash) * h, uword nbytes)
 
       base = (void *) (uword) (alloc_arena (h) + alloc_arena_mapped (h));
 
+#if __linux__
       rv = mmap (base, alloc, PROT_READ | PROT_WRITE, mmap_flags_huge, -1, 0);
+#elif __FreeBSD__
+      rv = MAP_FAILED;
+#endif /* __linux__ */
 
       /* fallback - maybe we are still able to allocate normal pages */
       if (rv == MAP_FAILED || mlock (base, alloc) != 0)
@@ -165,19 +171,23 @@ static void BV (clib_bihash_instantiate) (BVT (clib_bihash) * h)
 
   if (BIHASH_KVP_AT_BUCKET_LEVEL)
     {
-      int i;
+      int i, j;
       BVT (clib_bihash_bucket) * b;
 
       b = h->buckets;
 
       for (i = 0; i < h->nbuckets; i++)
 	{
+	  BVT (clib_bihash_kv) * v;
 	  b->offset = BV (clib_bihash_get_offset) (h, (void *) (b + 1));
 	  b->refcnt = 1;
 	  /* Mark all elements free */
-	  clib_memset_u8 ((b + 1), 0xff, BIHASH_KVP_PER_PAGE *
-			  sizeof (BVT (clib_bihash_kv)));
-
+	  v = (void *) (b + 1);
+	  for (j = 0; j < BIHASH_KVP_PER_PAGE; j++)
+	    {
+	      BV (clib_bihash_mark_free) (v);
+	      v++;
+	    }
 	  /* Compute next bucket start address */
 	  b = (void *) (((uword) b) + sizeof (*b) +
 			(BIHASH_KVP_PER_PAGE *
@@ -426,6 +436,7 @@ void BV (clib_bihash_free) (BVT (clib_bihash) * h)
 
   vec_free (h->working_copies);
   vec_free (h->working_copy_lengths);
+  clib_mem_free ((void *) h->alloc_lock);
 #if BIHASH_32_64_SVM == 0
   vec_free (h->freelists);
 #else
@@ -458,6 +469,7 @@ static
 BVT (clib_bihash_value) *
 BV (value_alloc) (BVT (clib_bihash) * h, u32 log2_pages)
 {
+  int i;
   BVT (clib_bihash_value) * rv = 0;
 
   ASSERT (h->alloc_lock[0]);
@@ -477,12 +489,15 @@ BV (value_alloc) (BVT (clib_bihash) * h, u32 log2_pages)
 
 initialize:
   ASSERT (rv);
-  /*
-   * Latest gcc complains that the length arg is zero
-   * if we replace (1<<log2_pages) with vec_len(rv).
-   * No clue.
-   */
-  clib_memset_u8 (rv, 0xff, sizeof (*rv) * (1 << log2_pages));
+
+  BVT (clib_bihash_kv) * v;
+  v = (BVT (clib_bihash_kv) *) rv;
+
+  for (i = 0; i < BIHASH_KVP_PER_PAGE * (1 << log2_pages); i++)
+    {
+      BV (clib_bihash_mark_free) (v);
+      v++;
+    }
   return rv;
 }
 
@@ -671,9 +686,10 @@ BV (split_and_rehash_linear)
   return new_values;
 }
 
-static_always_inline int BV (clib_bihash_add_del_inline_with_hash)
-  (BVT (clib_bihash) * h, BVT (clib_bihash_kv) * add_v, u64 hash, int is_add,
-   int (*is_stale_cb) (BVT (clib_bihash_kv) *, void *), void *arg)
+static_always_inline int BV (clib_bihash_add_del_inline_with_hash) (
+  BVT (clib_bihash) * h, BVT (clib_bihash_kv) * add_v, u64 hash, int is_add,
+  int (*is_stale_cb) (BVT (clib_bihash_kv) *, void *), void *is_stale_arg,
+  void (*overwrite_cb) (BVT (clib_bihash_kv) *, void *), void *overwrite_arg)
 {
   BVT (clib_bihash_bucket) * b, tmp_b;
   BVT (clib_bihash_value) * v, *new_v, *save_new_v, *working_copy;
@@ -684,12 +700,10 @@ static_always_inline int BV (clib_bihash_add_del_inline_with_hash)
   int mark_bucket_linear;
   int resplit_once;
 
-  /* *INDENT-OFF* */
   static const BVT (clib_bihash_bucket) mask = {
     .linear_search = 1,
     .log2_pages = -1
   };
-  /* *INDENT-ON* */
 
 #if BIHASH_LAZY_INSTANTIATE
   /*
@@ -711,6 +725,12 @@ static_always_inline int BV (clib_bihash_add_del_inline_with_hash)
   ASSERT (h->instantiated != 0);
 #endif
 
+  /*
+   * Debug image: make sure that an item being added doesn't accidentally
+   * look like a free item.
+   */
+  ASSERT ((is_add && BV (clib_bihash_is_free) (add_v)) == 0);
+
   b = BV (clib_bihash_get_bucket) (h, hash);
 
   BV (clib_bihash_lock_bucket) (b);
@@ -767,6 +787,8 @@ static_always_inline int BV (clib_bihash_add_del_inline_with_hash)
        */
       for (i = 0; i < limit; i++)
 	{
+	  if (BV (clib_bihash_is_free) (&(v->kvp[i])))
+	    continue;
 	  if (BV (clib_bihash_key_compare) (v->kvp[i].key, add_v->key))
 	    {
 	      /* Add but do not overwrite? */
@@ -775,7 +797,8 @@ static_always_inline int BV (clib_bihash_add_del_inline_with_hash)
 		  BV (clib_bihash_unlock_bucket) (b);
 		  return (-2);
 		}
-
+	      if (overwrite_cb)
+		overwrite_cb (&(v->kvp[i]), overwrite_arg);
 	      clib_memcpy_fast (&(v->kvp[i].value),
 				&add_v->value, sizeof (add_v->value));
 	      BV (clib_bihash_unlock_bucket) (b);
@@ -811,7 +834,7 @@ static_always_inline int BV (clib_bihash_add_del_inline_with_hash)
 	{
 	  for (i = 0; i < limit; i++)
 	    {
-	      if (is_stale_cb (&(v->kvp[i]), arg))
+	      if (is_stale_cb (&(v->kvp[i]), is_stale_arg))
 		{
 		  clib_memcpy_fast (&(v->kvp[i]), add_v, sizeof (*add_v));
 		  CLIB_MEMORY_STORE_BARRIER ();
@@ -827,10 +850,13 @@ static_always_inline int BV (clib_bihash_add_del_inline_with_hash)
     {
       for (i = 0; i < limit; i++)
 	{
+	  /* no sense even looking at this one */
+	  if (BV (clib_bihash_is_free) (&(v->kvp[i])))
+	    continue;
 	  /* Found the key? Kill it... */
 	  if (BV (clib_bihash_key_compare) (v->kvp[i].key, add_v->key))
 	    {
-	      clib_memset_u8 (&(v->kvp[i]), 0xff, sizeof (*(add_v)));
+	      BV (clib_bihash_mark_free) (&(v->kvp[i]));
 	      /* Is the bucket empty? */
 	      if (PREDICT_TRUE (b->refcnt > 1))
 		{
@@ -845,8 +871,13 @@ static_always_inline int BV (clib_bihash_add_del_inline_with_hash)
 		      b->linear_search = 0;
 		      b->log2_pages = 0;
 		      /* Clean up the bucket-level kvp array */
-		      clib_memset_u8 ((b + 1), 0xff, BIHASH_KVP_PER_PAGE *
-				      sizeof (BVT (clib_bihash_kv)));
+		      BVT (clib_bihash_kv) *v = (void *) (b + 1);
+		      int j;
+		      for (j = 0; j < BIHASH_KVP_PER_PAGE; j++)
+			{
+			  BV (clib_bihash_mark_free) (v);
+			  v++;
+			}
 		      CLIB_MEMORY_STORE_BARRIER ();
 		      BV (clib_bihash_unlock_bucket) (b);
 		      BV (clib_bihash_increment_stat) (h, BIHASH_STAT_del, 1);
@@ -993,7 +1024,15 @@ static_always_inline int BV (clib_bihash_add_del_inline)
 {
   u64 hash = BV (clib_bihash_hash) (add_v);
   return BV (clib_bihash_add_del_inline_with_hash) (h, add_v, hash, is_add,
-						    is_stale_cb, arg);
+						    is_stale_cb, arg, 0, 0);
+}
+
+int BV (clib_bihash_add_del_with_hash) (BVT (clib_bihash) * h,
+					BVT (clib_bihash_kv) * add_v, u64 hash,
+					int is_add)
+{
+  return BV (clib_bihash_add_del_inline_with_hash) (h, add_v, hash, is_add, 0,
+						    0, 0, 0);
 }
 
 int BV (clib_bihash_add_del)
@@ -1009,6 +1048,15 @@ int BV (clib_bihash_add_or_overwrite_stale)
   return BV (clib_bihash_add_del_inline) (h, add_v, 1, stale_callback, arg);
 }
 
+int BV (clib_bihash_add_with_overwrite_cb) (
+  BVT (clib_bihash) * h, BVT (clib_bihash_kv) * add_v,
+  void (overwrite_cb) (BVT (clib_bihash_kv) *, void *), void *arg)
+{
+  u64 hash = BV (clib_bihash_hash) (add_v);
+  return BV (clib_bihash_add_del_inline_with_hash) (h, add_v, hash, 1, 0, 0,
+						    overwrite_cb, arg);
+}
+
 int BV (clib_bihash_search)
   (BVT (clib_bihash) * h,
    BVT (clib_bihash_kv) * search_key, BVT (clib_bihash_kv) * valuep)