From 941005336ee8cec614a856089f3d873f7d98135c Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Fri, 6 Nov 2020 23:25:57 +0100 Subject: interface: rx queue infra rework, part one Type: improvement Change-Id: I4008cadfd5141f921afbdc09a3ebcd1dcf88eb29 Signed-off-by: Damjan Marion --- src/vppinfra/CMakeLists.txt | 2 + src/vppinfra/interrupt.c | 92 ++++++++++++++++++++++++++++ src/vppinfra/interrupt.h | 142 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 236 insertions(+) create mode 100644 src/vppinfra/interrupt.c create mode 100644 src/vppinfra/interrupt.h (limited to 'src/vppinfra') diff --git a/src/vppinfra/CMakeLists.txt b/src/vppinfra/CMakeLists.txt index 200cfb1eded..3dc99cc9db6 100644 --- a/src/vppinfra/CMakeLists.txt +++ b/src/vppinfra/CMakeLists.txt @@ -59,6 +59,7 @@ set(VPPINFRA_SRCS graph.c hash.c heap.c + interrupt.c longjmp.S macros.c maplog.c @@ -134,6 +135,7 @@ set(VPPINFRA_HEADERS graph.h hash.h heap.h + interrupt.h lb_hash_hash.h llist.h lock.h diff --git a/src/vppinfra/interrupt.c b/src/vppinfra/interrupt.c new file mode 100644 index 00000000000..20b7450ceed --- /dev/null +++ b/src/vppinfra/interrupt.c @@ -0,0 +1,92 @@ + +/* + * Copyright (c) 2020 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include /* for count_set_bits */ +#include +#include +#include + +__clib_export void +clib_interrupt_init (void **data, uword n_int) +{ + clib_interrupt_header_t *h; + uword sz = sizeof (clib_interrupt_header_t); + uword data_size = round_pow2 (n_int, CLIB_CACHE_LINE_BYTES * 8) / 8; + + sz += 2 * data_size; + h = data[0] = clib_mem_alloc_aligned (sz, CLIB_CACHE_LINE_BYTES); + clib_memset (data[0], 0, sz); + h->n_int = n_int; + h->n_uword_alloc = (data_size * 8) >> log2_uword_bits; +} + +__clib_export void +clib_interrupt_resize (void **data, uword n_int) +{ + clib_interrupt_header_t *h = data[0]; + + if (data[0] == 0) + { + clib_interrupt_init (data, n_int); + return; + } + + if (n_int < h->n_int) + { + uword *old_bmp, *old_abp, v; + old_bmp = clib_interrupt_get_bitmap (data[0]); + old_abp = clib_interrupt_get_atomic_bitmap (data[0]); + for (uword i = 0; i < h->n_uword_alloc; i++) + { + v = old_abp[i]; + old_abp[i] = 0; + if (n_int > ((i + 1) * uword_bits)) + old_bmp[i] |= v; + else if (n_int > (i * uword_bits)) + old_bmp[i] = (old_bmp[i] | v) & pow2_mask (n_int - i * uword_bits); + else + old_bmp[i] = 0; + } + } + else if (n_int > h->n_uword_alloc * uword_bits) + { + void *old = data[0]; + uword *old_bmp, *old_abp, *new_bmp; + uword n_uwords = round_pow2 (h->n_int, uword_bits) / uword_bits; + + clib_interrupt_init (data, n_int); + h = data[0]; + + new_bmp = clib_interrupt_get_bitmap (data[0]); + old_bmp = clib_interrupt_get_bitmap (old); + old_abp = clib_interrupt_get_atomic_bitmap (old); + + for (uword i = 0; i < n_uwords; i++) + new_bmp[i] = old_bmp[i] | old_abp[i]; + + clib_mem_free (old); + } + h->n_int = n_int; +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vppinfra/interrupt.h b/src/vppinfra/interrupt.h new file mode 100644 index 00000000000..60c01fa0248 --- /dev/null +++ b/src/vppinfra/interrupt.h @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2020 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef included_clib_interrupt_h +#define included_clib_interrupt_h + +#include +#include /* for count_set_bits */ +#include + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + int n_int; + uword n_uword_alloc; +} clib_interrupt_header_t; + +void clib_interrupt_init (void **data, uword n_interrupts); +void clib_interrupt_resize (void **data, uword n_interrupts); + +static_always_inline void +clib_interrupt_free (void **data) +{ + if (data[0]) + { + clib_mem_free (data[0]); + data[0] = 0; + } +} + +static_always_inline int +clib_interrupt_get_n_int (void *d) +{ + clib_interrupt_header_t *h = d; + if (h) + return h->n_int; + return 0; +} + +static_always_inline uword * +clib_interrupt_get_bitmap (void *d) +{ + return d + sizeof (clib_interrupt_header_t); +} + +static_always_inline uword * +clib_interrupt_get_atomic_bitmap (void *d) +{ + clib_interrupt_header_t *h = d; + return clib_interrupt_get_bitmap (d) + h->n_uword_alloc; +} + +static_always_inline void +clib_interrupt_set (void *in, int int_num) +{ + uword *bmp = clib_interrupt_get_bitmap (in); + uword mask = 1ULL << (int_num & (uword_bits - 1)); + bmp += int_num >> log2_uword_bits; + + ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int); + + *bmp |= mask; +} + +static_always_inline void +clib_interrupt_set_atomic (void *in, int int_num) +{ + uword *bmp = clib_interrupt_get_atomic_bitmap (in); + uword mask = 1ULL << (int_num & (uword_bits - 1)); + bmp += int_num >> log2_uword_bits; + + ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int); + + __atomic_fetch_or (bmp, mask, __ATOMIC_RELAXED); +} + +static_always_inline void +clib_interrupt_clear (void *in, int int_num) +{ + uword *bmp = clib_interrupt_get_bitmap (in); + uword *abm = clib_interrupt_get_atomic_bitmap (in); + uword mask = 1ULL << (int_num & (uword_bits - 1)); + uword off = int_num >> log2_uword_bits; + + ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int); + + bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST); + bmp[off] &= ~mask; +} + +static_always_inline int +clib_interrupt_get_next (void *in, int last) +{ + uword *bmp = clib_interrupt_get_bitmap (in); + uword *abm = clib_interrupt_get_atomic_bitmap (in); + clib_interrupt_header_t *h = in; + uword bmp_uword, off; + + ASSERT (last >= -1 && last < h->n_int); + + off = (last + 1) >> log2_uword_bits; + + last -= off << log2_uword_bits; + bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST); + bmp_uword = bmp[off] & ~pow2_mask (last + 1); + +next: + if (bmp_uword) + return (off << log2_uword_bits) + count_trailing_zeros (bmp_uword); + + off++; + + if (off > h->n_int >> log2_uword_bits) + return -1; + + bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST); + bmp_uword = bmp[off]; + + goto next; +} + +#endif /* included_clib_interrupt_h */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ -- cgit 1.2.3-korg