interface: rx queue infra rework, part one

Type: improvement
Change-Id: I4008cadfd5141f921afbdc09a3ebcd1dcf88eb29
Signed-off-by: Damjan Marion <damarion@cisco.com>

3 files changed, 236 insertions, 0 deletions

diff --git a/src/vppinfra/CMakeLists.txt b/src/vppinfra/CMakeLists.txt
index 200cfb1eded..3dc99cc9db6 100644
--- a/src/vppinfra/CMakeLists.txt
+++ b/src/vppinfra/CMakeLists.txt
@@ -59,6 +59,7 @@ set(VPPINFRA_SRCS
   graph.c
   hash.c
   heap.c
+  interrupt.c
   longjmp.S
   macros.c
   maplog.c
@@ -134,6 +135,7 @@ set(VPPINFRA_HEADERS
   graph.h
   hash.h
   heap.h
+  interrupt.h
   lb_hash_hash.h
   llist.h
   lock.h
diff --git a/src/vppinfra/interrupt.c b/src/vppinfra/interrupt.c
new file mode 100644
index 00000000000..20b7450ceed
--- /dev/null
+++ b/src/vppinfra/interrupt.c
@@ -0,0 +1,92 @@
+
+/*
+ * Copyright (c) 2020 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vppinfra/clib.h>
+#include <vppinfra/bitops.h> /* for count_set_bits */
+#include <vppinfra/vec.h>
+#include <vppinfra/interrupt.h>
+#include <vppinfra/format.h>
+
+__clib_export void
+clib_interrupt_init (void **data, uword n_int)
+{
+  clib_interrupt_header_t *h;
+  uword sz = sizeof (clib_interrupt_header_t);
+  uword data_size = round_pow2 (n_int, CLIB_CACHE_LINE_BYTES * 8) / 8;
+
+  sz += 2 * data_size;
+  h = data[0] = clib_mem_alloc_aligned (sz, CLIB_CACHE_LINE_BYTES);
+  clib_memset (data[0], 0, sz);
+  h->n_int = n_int;
+  h->n_uword_alloc = (data_size * 8) >> log2_uword_bits;
+}
+
+__clib_export void
+clib_interrupt_resize (void **data, uword n_int)
+{
+  clib_interrupt_header_t *h = data[0];
+
+  if (data[0] == 0)
+    {
+      clib_interrupt_init (data, n_int);
+      return;
+    }
+
+  if (n_int < h->n_int)
+    {
+      uword *old_bmp, *old_abp, v;
+      old_bmp = clib_interrupt_get_bitmap (data[0]);
+      old_abp = clib_interrupt_get_atomic_bitmap (data[0]);
+      for (uword i = 0; i < h->n_uword_alloc; i++)
+	{
+	  v = old_abp[i];
+	  old_abp[i] = 0;
+	  if (n_int > ((i + 1) * uword_bits))
+	    old_bmp[i] |= v;
+	  else if (n_int > (i * uword_bits))
+	    old_bmp[i] = (old_bmp[i] | v) & pow2_mask (n_int - i * uword_bits);
+	  else
+	    old_bmp[i] = 0;
+	}
+    }
+  else if (n_int > h->n_uword_alloc * uword_bits)
+    {
+      void *old = data[0];
+      uword *old_bmp, *old_abp, *new_bmp;
+      uword n_uwords = round_pow2 (h->n_int, uword_bits) / uword_bits;
+
+      clib_interrupt_init (data, n_int);
+      h = data[0];
+
+      new_bmp = clib_interrupt_get_bitmap (data[0]);
+      old_bmp = clib_interrupt_get_bitmap (old);
+      old_abp = clib_interrupt_get_atomic_bitmap (old);
+
+      for (uword i = 0; i < n_uwords; i++)
+	new_bmp[i] = old_bmp[i] | old_abp[i];
+
+      clib_mem_free (old);
+    }
+  h->n_int = n_int;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vppinfra/interrupt.h b/src/vppinfra/interrupt.h
new file mode 100644
index 00000000000..60c01fa0248
--- /dev/null
+++ b/src/vppinfra/interrupt.h
@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2020 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef included_clib_interrupt_h
+#define included_clib_interrupt_h
+
+#include <vppinfra/clib.h>
+#include <vppinfra/bitops.h> /* for count_set_bits */
+#include <vppinfra/vec.h>
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  int n_int;
+  uword n_uword_alloc;
+} clib_interrupt_header_t;
+
+void clib_interrupt_init (void **data, uword n_interrupts);
+void clib_interrupt_resize (void **data, uword n_interrupts);
+
+static_always_inline void
+clib_interrupt_free (void **data)
+{
+  if (data[0])
+    {
+      clib_mem_free (data[0]);
+      data[0] = 0;
+    }
+}
+
+static_always_inline int
+clib_interrupt_get_n_int (void *d)
+{
+  clib_interrupt_header_t *h = d;
+  if (h)
+    return h->n_int;
+  return 0;
+}
+
+static_always_inline uword *
+clib_interrupt_get_bitmap (void *d)
+{
+  return d + sizeof (clib_interrupt_header_t);
+}
+
+static_always_inline uword *
+clib_interrupt_get_atomic_bitmap (void *d)
+{
+  clib_interrupt_header_t *h = d;
+  return clib_interrupt_get_bitmap (d) + h->n_uword_alloc;
+}
+
+static_always_inline void
+clib_interrupt_set (void *in, int int_num)
+{
+  uword *bmp = clib_interrupt_get_bitmap (in);
+  uword mask = 1ULL << (int_num & (uword_bits - 1));
+  bmp += int_num >> log2_uword_bits;
+
+  ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
+
+  *bmp |= mask;
+}
+
+static_always_inline void
+clib_interrupt_set_atomic (void *in, int int_num)
+{
+  uword *bmp = clib_interrupt_get_atomic_bitmap (in);
+  uword mask = 1ULL << (int_num & (uword_bits - 1));
+  bmp += int_num >> log2_uword_bits;
+
+  ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
+
+  __atomic_fetch_or (bmp, mask, __ATOMIC_RELAXED);
+}
+
+static_always_inline void
+clib_interrupt_clear (void *in, int int_num)
+{
+  uword *bmp = clib_interrupt_get_bitmap (in);
+  uword *abm = clib_interrupt_get_atomic_bitmap (in);
+  uword mask = 1ULL << (int_num & (uword_bits - 1));
+  uword off = int_num >> log2_uword_bits;
+
+  ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
+
+  bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
+  bmp[off] &= ~mask;
+}
+
+static_always_inline int
+clib_interrupt_get_next (void *in, int last)
+{
+  uword *bmp = clib_interrupt_get_bitmap (in);
+  uword *abm = clib_interrupt_get_atomic_bitmap (in);
+  clib_interrupt_header_t *h = in;
+  uword bmp_uword, off;
+
+  ASSERT (last >= -1 && last < h->n_int);
+
+  off = (last + 1) >> log2_uword_bits;
+
+  last -= off << log2_uword_bits;
+  bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
+  bmp_uword = bmp[off] & ~pow2_mask (last + 1);
+
+next:
+  if (bmp_uword)
+    return (off << log2_uword_bits) + count_trailing_zeros (bmp_uword);
+
+  off++;
+
+  if (off > h->n_int >> log2_uword_bits)
+    return -1;
+
+  bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
+  bmp_uword = bmp[off];
+
+  goto next;
+}
+
+#endif /* included_clib_interrupt_h */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */