summaryrefslogtreecommitdiffstats
path: root/src/vppinfra
diff options
context:
space:
mode:
authorDamjan Marion <damarion@cisco.com>2023-11-03 21:57:42 +0000
committerDamjan Marion <damarion@cisco.com>2023-11-03 22:56:29 +0000
commit7f75e80f087d3ff1f5ebcb9eb6a117115baee011 (patch)
treea05daac2f541cf9cca7cb3350206c4b65cea2603 /src/vppinfra
parent236fae462a4ee799b3f05d4b2aff641d5c9486a6 (diff)
vppinfra: refactor interrupt code
Type: improvement Change-Id: Ie6987736faf7d8a641762e276775da8ee0c03ea4 Signed-off-by: Damjan Marion <damarion@cisco.com>
Diffstat (limited to 'src/vppinfra')
-rw-r--r--src/vppinfra/CMakeLists.txt1
-rw-r--r--src/vppinfra/interrupt.c98
-rw-r--r--src/vppinfra/interrupt.h140
-rw-r--r--src/vppinfra/test_interrupt.c11
4 files changed, 112 insertions, 138 deletions
diff --git a/src/vppinfra/CMakeLists.txt b/src/vppinfra/CMakeLists.txt
index 46b2788ddfd..d64847161bd 100644
--- a/src/vppinfra/CMakeLists.txt
+++ b/src/vppinfra/CMakeLists.txt
@@ -249,6 +249,7 @@ if(VPP_BUILD_VPPINFRA_TESTS)
fpool
hash
heap
+ interrupt
longjmp
macros
maplog
diff --git a/src/vppinfra/interrupt.c b/src/vppinfra/interrupt.c
index df242d932b6..c9f0078c5e4 100644
--- a/src/vppinfra/interrupt.c
+++ b/src/vppinfra/interrupt.c
@@ -1,42 +1,33 @@
-
-/*
- * Copyright (c) 2020 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) 2023 Cisco Systems, Inc.
*/
#include <vppinfra/clib.h>
-#include <vppinfra/vec.h>
#include <vppinfra/interrupt.h>
-#include <vppinfra/format.h>
__clib_export void
-clib_interrupt_init (void **data, uword n_int)
+clib_interrupt_init (void **data, u32 n_int)
{
clib_interrupt_header_t *h;
- uword sz = sizeof (clib_interrupt_header_t);
- uword data_size = round_pow2 (n_int, CLIB_CACHE_LINE_BYTES * 8) / 8;
+ const u32 bits_in_cl = 8 << CLIB_LOG2_CACHE_LINE_BYTES;
+ u32 sz = sizeof (clib_interrupt_header_t);
+ u32 n_cl = round_pow2 (n_int, bits_in_cl) / bits_in_cl;
- sz += 2 * data_size;
+ sz += 2 * n_cl * CLIB_CACHE_LINE_BYTES;
h = data[0] = clib_mem_alloc_aligned (sz, CLIB_CACHE_LINE_BYTES);
clib_memset (data[0], 0, sz);
h->n_int = n_int;
- h->n_uword_alloc = (data_size * 8) >> log2_uword_bits;
+ h->uwords_allocated = n_cl * bits_in_cl / uword_bits;
+ h->uwords_used = round_pow2 (n_int, uword_bits) / uword_bits;
+ h->local = (uword *) (h + 1);
+ h->remote = h->local + h->uwords_allocated;
}
__clib_export void
-clib_interrupt_resize (void **data, uword n_int)
+clib_interrupt_resize (void **data, u32 n_int)
{
clib_interrupt_header_t *h = data[0];
+ u32 new_n_uwords, i;
if (data[0] == 0)
{
@@ -44,48 +35,37 @@ clib_interrupt_resize (void **data, uword n_int)
return;
}
- if (n_int < h->n_int)
+ if (n_int == h->n_int)
+ return;
+
+ new_n_uwords = round_pow2 (n_int, uword_bits) / uword_bits;
+
+ if (new_n_uwords > h->uwords_allocated)
{
- uword *old_bmp, *old_abp, v;
- old_bmp = clib_interrupt_get_bitmap (data[0]);
- old_abp = clib_interrupt_get_atomic_bitmap (data[0]);
- for (uword i = 0; i < h->n_uword_alloc; i++)
- {
- v = old_abp[i];
- old_abp[i] = 0;
- if (n_int > ((i + 1) * uword_bits))
- old_bmp[i] |= v;
- else if (n_int > (i * uword_bits))
- old_bmp[i] = (old_bmp[i] | v) & pow2_mask (n_int - i * uword_bits);
- else
- old_bmp[i] = 0;
- }
+ clib_interrupt_header_t *nh;
+ clib_interrupt_init ((void **) &nh, n_int);
+ for (int i = 0; i < h->uwords_used; i++)
+ nh->local[i] = h->local[i] | h->remote[i];
+ clib_mem_free (data[0]);
+ data[0] = nh;
+ return;
}
- else if (n_int > h->n_uword_alloc * uword_bits)
- {
- void *old = data[0];
- uword *old_bmp, *old_abp, *new_bmp;
- uword n_uwords = round_pow2 (h->n_int, uword_bits) / uword_bits;
- clib_interrupt_init (data, n_int);
- h = data[0];
+ h->n_int = n_int;
+ h->uwords_used = new_n_uwords;
- new_bmp = clib_interrupt_get_bitmap (data[0]);
- old_bmp = clib_interrupt_get_bitmap (old);
- old_abp = clib_interrupt_get_atomic_bitmap (old);
+ for (i = 0; i < new_n_uwords; i++)
+ h->local[i] |= h->remote[i];
- for (uword i = 0; i < n_uwords; i++)
- new_bmp[i] = old_bmp[i] | old_abp[i];
+ for (i = 0; i < h->uwords_allocated; i++)
+ h->remote[i] = 0;
- clib_mem_free (old);
- }
- h->n_int = n_int;
+ for (i = new_n_uwords; i < h->uwords_allocated; i++)
+ h->local[i] = 0;
+
+ n_int &= pow2_mask (log2_uword_bits);
+
+ if (n_int)
+ h->local[n_int >> log2_uword_bits] &= pow2_mask (n_int);
}
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vppinfra/interrupt.h b/src/vppinfra/interrupt.h
index 5c39b2183f8..b0d7dde272a 100644
--- a/src/vppinfra/interrupt.h
+++ b/src/vppinfra/interrupt.h
@@ -1,16 +1,5 @@
-/*
- * Copyright (c) 2020 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) 2023 Cisco Systems, Inc.
*/
#ifndef included_clib_interrupt_h
@@ -22,12 +11,15 @@
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- int n_int;
- uword n_uword_alloc;
+ u32 n_int;
+ u32 uwords_allocated;
+ u32 uwords_used;
+ uword *local;
+ uword *remote;
} clib_interrupt_header_t;
-void clib_interrupt_init (void **data, uword n_interrupts);
-void clib_interrupt_resize (void **data, uword n_interrupts);
+void clib_interrupt_init (void **data, u32 n_interrupts);
+void clib_interrupt_resize (void **data, u32 n_interrupts);
static_always_inline void
clib_interrupt_free (void **data)
@@ -48,96 +40,98 @@ clib_interrupt_get_n_int (void *d)
return 0;
}
-static_always_inline uword *
-clib_interrupt_get_bitmap (void *d)
-{
- return d + sizeof (clib_interrupt_header_t);
-}
-
-static_always_inline uword *
-clib_interrupt_get_atomic_bitmap (void *d)
-{
- clib_interrupt_header_t *h = d;
- return clib_interrupt_get_bitmap (d) + h->n_uword_alloc;
-}
-
static_always_inline void
clib_interrupt_set (void *in, int int_num)
{
- uword *bmp = clib_interrupt_get_bitmap (in);
- uword mask = 1ULL << (int_num & (uword_bits - 1));
- bmp += int_num >> log2_uword_bits;
+ clib_interrupt_header_t *h = in;
+ u32 off = int_num >> log2_uword_bits;
+ uword bit = 1ULL << (int_num & pow2_mask (log2_uword_bits));
- ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
+ ASSERT (int_num < h->n_int);
- *bmp |= mask;
+ h->local[off] |= bit;
}
static_always_inline void
clib_interrupt_set_atomic (void *in, int int_num)
{
- uword *bmp = clib_interrupt_get_atomic_bitmap (in);
- uword mask = 1ULL << (int_num & (uword_bits - 1));
- bmp += int_num >> log2_uword_bits;
+ clib_interrupt_header_t *h = in;
+ u32 off = int_num >> log2_uword_bits;
+ uword bit = 1ULL << (int_num & pow2_mask (log2_uword_bits));
- ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
+ ASSERT (int_num < h->n_int);
- __atomic_fetch_or (bmp, mask, __ATOMIC_RELAXED);
+ __atomic_fetch_or (h->remote + off, bit, __ATOMIC_RELAXED);
}
static_always_inline void
clib_interrupt_clear (void *in, int int_num)
{
- uword *bmp = clib_interrupt_get_bitmap (in);
- uword *abm = clib_interrupt_get_atomic_bitmap (in);
- uword mask = 1ULL << (int_num & (uword_bits - 1));
- uword off = int_num >> log2_uword_bits;
+ clib_interrupt_header_t *h = in;
+ u32 off = int_num >> log2_uword_bits;
+ uword bit = 1ULL << (int_num & pow2_mask (log2_uword_bits));
+ uword *loc = h->local;
+ uword *rem = h->remote;
+ uword v;
- ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
+ ASSERT (int_num < h->n_int);
- bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
- bmp[off] &= ~mask;
+ v = loc[off] | __atomic_exchange_n (rem + off, 0, __ATOMIC_SEQ_CST);
+ loc[off] = v & ~bit;
}
static_always_inline int
-clib_interrupt_get_next (void *in, int last)
+clib_interrupt_get_next_and_clear (void *in, int last)
{
- uword *bmp = clib_interrupt_get_bitmap (in);
- uword *abm = clib_interrupt_get_atomic_bitmap (in);
clib_interrupt_header_t *h = in;
- uword bmp_uword, off;
+ uword bit, v;
+ uword *loc = h->local;
+ uword *rem = h->remote;
+ u32 off, n_uwords = h->uwords_used;
- ASSERT (last >= -1 && last < h->n_int);
+ ASSERT (last >= -1 && last < (int) h->n_int);
off = (last + 1) >> log2_uword_bits;
- if (off > h->n_int >> log2_uword_bits || off >= h->n_uword_alloc)
+
+ if (off >= n_uwords)
return -1;
- last -= off << log2_uword_bits;
- bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
- bmp_uword = bmp[off] & ~pow2_mask (last + 1);
+ v = loc[off] | __atomic_exchange_n (rem + off, 0, __ATOMIC_SEQ_CST);
+ loc[off] = v;
+
+ v &= ~pow2_mask ((last + 1) & pow2_mask (log2_uword_bits));
-next:
- if (bmp_uword)
- return (off << log2_uword_bits) + count_trailing_zeros (bmp_uword);
+ while (v == 0)
+ {
+ if (++off == n_uwords)
+ return -1;
- off++;
+ v = loc[off] | __atomic_exchange_n (rem + off, 0, __ATOMIC_SEQ_CST);
+ loc[off] = v;
+ }
- if (off > h->n_int >> log2_uword_bits || off >= h->n_uword_alloc)
- return -1;
+ bit = get_lowest_set_bit (v);
+ loc[off] &= ~bit;
+ return get_lowest_set_bit_index (bit) + (int) (off << log2_uword_bits);
+}
+
+static_always_inline int
+clib_interrupt_is_any_pending (void *in)
+{
+ clib_interrupt_header_t *h = in;
+ u32 n_uwords = h->uwords_used;
+ uword *loc = h->local;
+ uword *rem = h->remote;
+
+ for (u32 i = 0; i < n_uwords; i++)
+ if (loc[i])
+ return 1;
- bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
- bmp_uword = bmp[off];
+ for (u32 i = 0; i < n_uwords; i++)
+ if (rem[i])
+ return 1;
- goto next;
+ return 0;
}
#endif /* included_clib_interrupt_h */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vppinfra/test_interrupt.c b/src/vppinfra/test_interrupt.c
index 519805edc89..133692d1bd0 100644
--- a/src/vppinfra/test_interrupt.c
+++ b/src/vppinfra/test_interrupt.c
@@ -33,10 +33,7 @@ int debug = 0;
void
set_and_check_bits (void *interrupts, int num_ints)
{
-
- int step;
-
- for (step = 1; step < num_ints; step++)
+ for (int step = 1; step < num_ints; step++)
{
int int_num = -1;
int expected = 0;
@@ -48,13 +45,15 @@ set_and_check_bits (void *interrupts, int num_ints)
clib_interrupt_set (interrupts, i);
}
- while ((int_num = clib_interrupt_get_next (interrupts, int_num)) != -1)
+ while ((int_num =
+ clib_interrupt_get_next_and_clear (interrupts, int_num)) != -1)
{
debug (" Got %d, expecting %d\n", int_num, expected);
ASSERT (int_num == expected);
expected += step;
- clib_interrupt_clear (interrupts, int_num);
}
+ int_num = clib_interrupt_get_next_and_clear (interrupts, -1);
+ ASSERT (int_num == -1);
}
}