vppinfra: refactor interrupt code

Type: improvement
Change-Id: Ie6987736faf7d8a641762e276775da8ee0c03ea4
Signed-off-by: Damjan Marion <damarion@cisco.com>

1 files changed, 67 insertions, 73 deletions

diff --git a/src/vppinfra/interrupt.h b/src/vppinfra/interrupt.h
index 5c39b2183f8..b0d7dde272a 100644
--- a/src/vppinfra/interrupt.h
+++ b/src/vppinfra/interrupt.h
@@ -1,16 +1,5 @@
-/*
- * Copyright (c) 2020 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) 2023 Cisco Systems, Inc.
  */
 
 #ifndef included_clib_interrupt_h
@@ -22,12 +11,15 @@
 typedef struct
 {
   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
-  int n_int;
-  uword n_uword_alloc;
+  u32 n_int;
+  u32 uwords_allocated;
+  u32 uwords_used;
+  uword *local;
+  uword *remote;
 } clib_interrupt_header_t;
 
-void clib_interrupt_init (void **data, uword n_interrupts);
-void clib_interrupt_resize (void **data, uword n_interrupts);
+void clib_interrupt_init (void **data, u32 n_interrupts);
+void clib_interrupt_resize (void **data, u32 n_interrupts);
 
 static_always_inline void
 clib_interrupt_free (void **data)
@@ -48,96 +40,98 @@ clib_interrupt_get_n_int (void *d)
   return 0;
 }
 
-static_always_inline uword *
-clib_interrupt_get_bitmap (void *d)
-{
-  return d + sizeof (clib_interrupt_header_t);
-}
-
-static_always_inline uword *
-clib_interrupt_get_atomic_bitmap (void *d)
-{
-  clib_interrupt_header_t *h = d;
-  return clib_interrupt_get_bitmap (d) + h->n_uword_alloc;
-}
-
 static_always_inline void
 clib_interrupt_set (void *in, int int_num)
 {
-  uword *bmp = clib_interrupt_get_bitmap (in);
-  uword mask = 1ULL << (int_num & (uword_bits - 1));
-  bmp += int_num >> log2_uword_bits;
+  clib_interrupt_header_t *h = in;
+  u32 off = int_num >> log2_uword_bits;
+  uword bit = 1ULL << (int_num & pow2_mask (log2_uword_bits));
 
-  ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
+  ASSERT (int_num < h->n_int);
 
-  *bmp |= mask;
+  h->local[off] |= bit;
 }
 
 static_always_inline void
 clib_interrupt_set_atomic (void *in, int int_num)
 {
-  uword *bmp = clib_interrupt_get_atomic_bitmap (in);
-  uword mask = 1ULL << (int_num & (uword_bits - 1));
-  bmp += int_num >> log2_uword_bits;
+  clib_interrupt_header_t *h = in;
+  u32 off = int_num >> log2_uword_bits;
+  uword bit = 1ULL << (int_num & pow2_mask (log2_uword_bits));
 
-  ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
+  ASSERT (int_num < h->n_int);
 
-  __atomic_fetch_or (bmp, mask, __ATOMIC_RELAXED);
+  __atomic_fetch_or (h->remote + off, bit, __ATOMIC_RELAXED);
 }
 
 static_always_inline void
 clib_interrupt_clear (void *in, int int_num)
 {
-  uword *bmp = clib_interrupt_get_bitmap (in);
-  uword *abm = clib_interrupt_get_atomic_bitmap (in);
-  uword mask = 1ULL << (int_num & (uword_bits - 1));
-  uword off = int_num >> log2_uword_bits;
+  clib_interrupt_header_t *h = in;
+  u32 off = int_num >> log2_uword_bits;
+  uword bit = 1ULL << (int_num & pow2_mask (log2_uword_bits));
+  uword *loc = h->local;
+  uword *rem = h->remote;
+  uword v;
 
-  ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
+  ASSERT (int_num < h->n_int);
 
-  bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
-  bmp[off] &= ~mask;
+  v = loc[off] | __atomic_exchange_n (rem + off, 0, __ATOMIC_SEQ_CST);
+  loc[off] = v & ~bit;
 }
 
 static_always_inline int
-clib_interrupt_get_next (void *in, int last)
+clib_interrupt_get_next_and_clear (void *in, int last)
 {
-  uword *bmp = clib_interrupt_get_bitmap (in);
-  uword *abm = clib_interrupt_get_atomic_bitmap (in);
   clib_interrupt_header_t *h = in;
-  uword bmp_uword, off;
+  uword bit, v;
+  uword *loc = h->local;
+  uword *rem = h->remote;
+  u32 off, n_uwords = h->uwords_used;
 
-  ASSERT (last >= -1 && last < h->n_int);
+  ASSERT (last >= -1 && last < (int) h->n_int);
 
   off = (last + 1) >> log2_uword_bits;
-  if (off > h->n_int >> log2_uword_bits || off >= h->n_uword_alloc)
+
+  if (off >= n_uwords)
     return -1;
 
-  last -= off << log2_uword_bits;
-  bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
-  bmp_uword = bmp[off] & ~pow2_mask (last + 1);
+  v = loc[off] | __atomic_exchange_n (rem + off, 0, __ATOMIC_SEQ_CST);
+  loc[off] = v;
+
+  v &= ~pow2_mask ((last + 1) & pow2_mask (log2_uword_bits));
 
-next:
-  if (bmp_uword)
-    return (off << log2_uword_bits) + count_trailing_zeros (bmp_uword);
+  while (v == 0)
+    {
+      if (++off == n_uwords)
+	return -1;
 
-  off++;
+      v = loc[off] | __atomic_exchange_n (rem + off, 0, __ATOMIC_SEQ_CST);
+      loc[off] = v;
+    }
 
-  if (off > h->n_int >> log2_uword_bits || off >= h->n_uword_alloc)
-    return -1;
+  bit = get_lowest_set_bit (v);
+  loc[off] &= ~bit;
+  return get_lowest_set_bit_index (bit) + (int) (off << log2_uword_bits);
+}
+
+static_always_inline int
+clib_interrupt_is_any_pending (void *in)
+{
+  clib_interrupt_header_t *h = in;
+  u32 n_uwords = h->uwords_used;
+  uword *loc = h->local;
+  uword *rem = h->remote;
+
+  for (u32 i = 0; i < n_uwords; i++)
+    if (loc[i])
+      return 1;
 
-  bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
-  bmp_uword = bmp[off];
+  for (u32 i = 0; i < n_uwords; i++)
+    if (rem[i])
+      return 1;
 
-  goto next;
+  return 0;
 }
 
 #endif /* included_clib_interrupt_h */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */