From 696e88da9799056036f329676213f3c0c0a1db9c Mon Sep 17 00:00:00 2001 From: Neale Ranns Date: Thu, 16 Mar 2017 07:34:55 -0400 Subject: MPLS performance improvments. 1 - Quad loop lookup and label imposition. 2 - optimise imposition for the 1 label case 3 - input gets TTL from header directly (no byte swap) Change-Id: I59204c9e5d134b0df75d7afa43e360f946d1ffe7 Signed-off-by: Neale Ranns --- src/vnet/mpls/mpls_input.c | 324 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 324 insertions(+) create mode 100644 src/vnet/mpls/mpls_input.c (limited to 'src/vnet/mpls/mpls_input.c') diff --git a/src/vnet/mpls/mpls_input.c b/src/vnet/mpls/mpls_input.c new file mode 100644 index 00000000..893c4511 --- /dev/null +++ b/src/vnet/mpls/mpls_input.c @@ -0,0 +1,324 @@ +/* + * node.c: MPLS input + * + * Copyright (c) 2012-2014 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +typedef struct { + u32 next_index; + u32 label_net_byte_order; +} mpls_input_trace_t; + +#define foreach_mpls_input_next \ +_(DROP, "error-drop") \ +_(LOOKUP, "mpls-lookup") + +typedef enum { +#define _(s,n) MPLS_INPUT_NEXT_##s, + foreach_mpls_input_next +#undef _ + MPLS_INPUT_N_NEXT, +} mpls_input_next_t; + +static u8 * +format_mpls_input_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + mpls_input_trace_t * t = va_arg (*args, mpls_input_trace_t *); + char * next_name; + u32 label; + next_name = "BUG!"; + label = clib_net_to_host_u32(t->label_net_byte_order); + +#define _(a,b) if (t->next_index == MPLS_INPUT_NEXT_##a) next_name = b; + foreach_mpls_input_next; +#undef _ + + s = format (s, "MPLS: next %s[%d] label %d ttl %d", + next_name, t->next_index, + vnet_mpls_uc_get_label(label), + vnet_mpls_uc_get_ttl(label)); + + return s; +} + +vlib_node_registration_t mpls_input_node; + +typedef struct { + u32 last_label; + u32 last_inner_fib_index; + u32 last_outer_fib_index; + mpls_main_t * mpls_main; +} mpls_input_runtime_t; + +static inline uword +mpls_input_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + u32 n_left_from, next_index, * from, * to_next; + mpls_input_runtime_t * rt; + mpls_main_t * mm; + u32 cpu_index = os_get_cpu_number(); + vlib_simple_counter_main_t * cm; + vnet_main_t * vnm = vnet_get_main(); + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + rt = vlib_node_get_runtime_data (vm, mpls_input_node.index); + mm = rt->mpls_main; + /* + * Force an initial lookup every time, in case the control-plane + * changed the label->FIB mapping. + */ + rt->last_label = ~0; + + next_index = node->cached_next_index; + + cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, + VNET_INTERFACE_COUNTER_MPLS); + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, + to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + u32 bi0, next0, sw_if_index0; + u32 bi1, next1, sw_if_index1; + vlib_buffer_t *b0, *b1; + char *h0, *h1; + + /* Prefetch next iteration. */ + { + vlib_buffer_t * p2, * p3; + + p2 = vlib_get_buffer (vm, from[2]); + p3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (p2, STORE); + vlib_prefetch_buffer_header (p3, STORE); + + CLIB_PREFETCH (p2->data, sizeof (h0[0]), STORE); + CLIB_PREFETCH (p3->data, sizeof (h1[0]), STORE); + } + + bi0 = to_next[0] = from[0]; + bi1 = to_next[1] = from[1]; + + from += 2; + to_next += 2; + n_left_from -= 2; + n_left_to_next -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + h0 = vlib_buffer_get_current (b0); + h1 = vlib_buffer_get_current (b1); + + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX]; + + /* TTL expired? */ + if (PREDICT_FALSE(h0[3] == 0)) + { + next0 = MPLS_INPUT_NEXT_DROP; + b0->error = node->errors[MPLS_ERROR_TTL_EXPIRED]; + } + else + { + next0 = MPLS_INPUT_NEXT_LOOKUP; + vnet_feature_arc_start(mm->input_feature_arc_index, + sw_if_index0, &next0, b0); + vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); + } + + if (PREDICT_FALSE(h1[3] == 0)) + { + next1 = MPLS_INPUT_NEXT_DROP; + b1->error = node->errors[MPLS_ERROR_TTL_EXPIRED]; + } + else + { + next1 = MPLS_INPUT_NEXT_LOOKUP; + vnet_feature_arc_start(mm->input_feature_arc_index, + sw_if_index1, &next1, b1); + vlib_increment_simple_counter (cm, cpu_index, sw_if_index1, 1); + } + + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_input_trace_t *tr = vlib_add_trace (vm, node, + b0, sizeof (*tr)); + tr->next_index = next0; + tr->label_net_byte_order = *((u32*)h0); + } + if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_input_trace_t *tr = vlib_add_trace (vm, node, + b1, sizeof (*tr)); + tr->next_index = next1; + tr->label_net_byte_order = *((u32*)h1); + } + + vlib_validate_buffer_enqueue_x2 (vm, node, next_index, + to_next, n_left_to_next, + bi0, bi1, + next0, next1); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 sw_if_index0, next0, bi0; + vlib_buffer_t * b0; + char * h0; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + h0 = vlib_buffer_get_current (b0); + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + + /* TTL expired? */ + if (PREDICT_FALSE(h0[3] == 0)) + { + next0 = MPLS_INPUT_NEXT_DROP; + b0->error = node->errors[MPLS_ERROR_TTL_EXPIRED]; + } + else + { + next0 = MPLS_INPUT_NEXT_LOOKUP; + vnet_feature_arc_start(mm->input_feature_arc_index, sw_if_index0, &next0, b0); + vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); + } + + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_input_trace_t *tr = vlib_add_trace (vm, node, + b0, sizeof (*tr)); + tr->next_index = next0; + tr->label_net_byte_order = *(u32*)h0; + } + + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + vlib_node_increment_counter (vm, mpls_input_node.index, + MPLS_ERROR_PKTS_DECAP, from_frame->n_vectors); + return from_frame->n_vectors; +} + +static uword +mpls_input (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return mpls_input_inline (vm, node, from_frame); +} + +static char * mpls_error_strings[] = { +#define mpls_error(n,s) s, +#include "error.def" +#undef mpls_error +}; + +VLIB_REGISTER_NODE (mpls_input_node) = { + .function = mpls_input, + .name = "mpls-input", + /* Takes a vector of packets. */ + .vector_size = sizeof (u32), + + .runtime_data_bytes = sizeof(mpls_input_runtime_t), + + .n_errors = MPLS_N_ERROR, + .error_strings = mpls_error_strings, + + .n_next_nodes = MPLS_INPUT_N_NEXT, + .next_nodes = { +#define _(s,n) [MPLS_INPUT_NEXT_##s] = n, + foreach_mpls_input_next +#undef _ + }, + + .format_buffer = format_mpls_unicast_header_net_byte_order, + .format_trace = format_mpls_input_trace, +}; + +VLIB_NODE_FUNCTION_MULTIARCH (mpls_input_node, mpls_input) + +static void +mpls_setup_nodes (vlib_main_t * vm) +{ + mpls_input_runtime_t * rt; + pg_node_t * pn; + + pn = pg_get_node (mpls_input_node.index); + pn->unformat_edit = unformat_pg_mpls_header; + + rt = vlib_node_get_runtime_data (vm, mpls_input_node.index); + rt->last_label = (u32) ~0; + rt->last_inner_fib_index = 0; + rt->last_outer_fib_index = 0; + rt->mpls_main = &mpls_main; + + ethernet_register_input_type (vm, ETHERNET_TYPE_MPLS_UNICAST, + mpls_input_node.index); +} + +static clib_error_t * mpls_input_init (vlib_main_t * vm) +{ + clib_error_t * error; + + error = vlib_call_init_function (vm, mpls_init); + if (error) + clib_error_report (error); + + mpls_setup_nodes (vm); + + return 0; +} + +VLIB_INIT_FUNCTION (mpls_input_init); + +static clib_error_t * mpls_input_worker_init (vlib_main_t * vm) +{ + mpls_input_runtime_t * rt; + rt = vlib_node_get_runtime_data (vm, mpls_input_node.index); + rt->last_label = (u32) ~0; + rt->last_inner_fib_index = 0; + rt->last_outer_fib_index = 0; + rt->mpls_main = &mpls_main; + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (mpls_input_worker_init); -- cgit 1.2.3-korg From 586afd762bfa149f5ca167bd5fd5a0cd59ce94fe Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Wed, 5 Apr 2017 19:18:20 +0200 Subject: Use thread local storage for thread index This patch deprecates stack-based thread identification, Also removes requirement that thread stacks are adjacent. Finally, possibly annoying for some folks, it renames all occurences of cpu_index and cpu_number with thread index. Using word "cpu" is misleading here as thread can be migrated ti different CPU, and also it is not related to linux cpu index. Change-Id: I68cdaf661e701d2336fc953dcb9978d10a70f7c1 Signed-off-by: Damjan Marion --- src/examples/srv6-sample-localsid/node.c | 4 +- src/plugins/dpdk/buffer.c | 2 +- src/plugins/dpdk/device/device.c | 8 +- src/plugins/dpdk/device/dpdk_priv.h | 8 +- src/plugins/dpdk/device/init.c | 2 +- src/plugins/dpdk/device/node.c | 32 +++--- src/plugins/dpdk/hqos/hqos.c | 16 +-- src/plugins/dpdk/ipsec/cli.c | 8 +- src/plugins/dpdk/ipsec/crypto_node.c | 4 +- src/plugins/dpdk/ipsec/esp.h | 4 +- src/plugins/dpdk/ipsec/esp_decrypt.c | 4 +- src/plugins/dpdk/ipsec/esp_encrypt.c | 5 +- src/plugins/dpdk/ipsec/ipsec.c | 2 +- src/plugins/dpdk/ipsec/ipsec.h | 4 +- src/plugins/dpdk/main.c | 2 +- src/plugins/flowperpkt/l2_node.c | 2 +- src/plugins/flowperpkt/node.c | 2 +- src/plugins/ioam/export-common/ioam_export.h | 6 +- .../ioam/ip6/ioam_cache_tunnel_select_node.c | 16 +-- src/plugins/ixge/ixge.c | 2 +- src/plugins/lb/lb.c | 8 +- src/plugins/lb/node.c | 22 ++-- src/plugins/lb/refcount.c | 8 +- src/plugins/lb/refcount.h | 4 +- src/plugins/memif/node.c | 35 +++--- src/plugins/snat/in2out.c | 110 +++++++++--------- src/plugins/snat/out2in.c | 102 ++++++++--------- src/plugins/snat/snat.h | 10 +- src/vlib/buffer.c | 6 +- src/vlib/buffer_funcs.h | 4 +- src/vlib/cli.c | 6 +- src/vlib/counter.h | 16 +-- src/vlib/error.c | 2 +- src/vlib/global_funcs.h | 2 +- src/vlib/main.c | 14 +-- src/vlib/main.h | 2 +- src/vlib/node.c | 2 +- src/vlib/node.h | 6 +- src/vlib/node_funcs.h | 8 +- src/vlib/threads.c | 69 ++++------- src/vlib/threads.h | 21 ++-- src/vlib/unix/cj.c | 7 +- src/vlib/unix/cj.h | 2 +- src/vlib/unix/main.c | 43 +++---- src/vnet/adj/adj_l2.c | 4 +- src/vnet/adj/adj_midchain.c | 8 +- src/vnet/adj/adj_nsh.c | 4 +- src/vnet/classify/vnet_classify.c | 16 +-- src/vnet/cop/ip4_whitelist.c | 8 +- src/vnet/cop/ip6_whitelist.c | 8 +- src/vnet/devices/af_packet/node.c | 20 ++-- src/vnet/devices/devices.c | 61 +++++----- src/vnet/devices/devices.h | 18 +-- src/vnet/devices/netmap/node.c | 24 ++-- src/vnet/devices/ssvm/node.c | 6 +- src/vnet/devices/virtio/vhost-user.c | 127 +++++++++++---------- src/vnet/dpo/lookup_dpo.c | 20 ++-- src/vnet/dpo/replicate_dpo.c | 12 +- src/vnet/ethernet/arp.c | 2 +- src/vnet/ethernet/interface.c | 7 +- src/vnet/ethernet/node.c | 14 +-- src/vnet/gre/node.c | 8 +- src/vnet/interface.h | 2 +- src/vnet/interface_output.c | 53 ++++----- src/vnet/ip/ip4_forward.c | 34 +++--- src/vnet/ip/ip4_input.c | 8 +- src/vnet/ip/ip6_forward.c | 24 ++-- src/vnet/ip/ip6_input.c | 8 +- src/vnet/ip/ip6_neighbor.c | 4 +- src/vnet/ipsec/esp.h | 8 +- src/vnet/ipsec/esp_decrypt.c | 13 ++- src/vnet/ipsec/esp_encrypt.c | 13 ++- src/vnet/ipsec/ikev2.c | 64 ++++++----- src/vnet/ipsec/ipsec.h | 12 +- src/vnet/ipsec/ipsec_if.c | 2 +- src/vnet/l2/l2_bvi.h | 2 +- src/vnet/l2/l2_input.c | 14 +-- src/vnet/l2/l2_output.c | 6 +- src/vnet/l2tp/decap.c | 2 +- src/vnet/l2tp/encap.c | 2 +- src/vnet/l2tp/l2tp.c | 6 +- src/vnet/lisp-gpe/decap.c | 16 +-- src/vnet/lldp/lldp_input.c | 2 +- src/vnet/map/ip4_map.c | 14 +-- src/vnet/map/ip4_map_t.c | 12 +- src/vnet/map/ip6_map.c | 19 +-- src/vnet/map/ip6_map_t.c | 12 +- src/vnet/mpls/mpls_input.c | 8 +- src/vnet/mpls/mpls_lookup.c | 20 ++-- src/vnet/mpls/mpls_output.c | 10 +- src/vnet/pg/input.c | 4 +- src/vnet/replication.c | 20 ++-- src/vnet/replication.h | 2 +- src/vnet/session/node.c | 2 +- src/vnet/sr/sr_localsid.c | 44 +++---- src/vnet/tcp/builtin_client.c | 2 +- src/vnet/tcp/tcp.c | 8 +- src/vnet/tcp/tcp_debug.h | 2 +- src/vnet/tcp/tcp_input.c | 10 +- src/vnet/tcp/tcp_output.c | 20 ++-- src/vnet/udp/udp_input.c | 2 +- src/vnet/unix/tapcli.c | 2 +- src/vnet/unix/tuntap.c | 4 +- src/vnet/vxlan-gpe/decap.c | 10 +- src/vnet/vxlan-gpe/encap.c | 12 +- src/vnet/vxlan/decap.c | 10 +- src/vnet/vxlan/encap.c | 12 +- src/vpp/stats/stats.c | 14 +-- src/vpp/stats/stats.h | 2 +- 109 files changed, 790 insertions(+), 791 deletions(-) (limited to 'src/vnet/mpls/mpls_input.c') diff --git a/src/examples/srv6-sample-localsid/node.c b/src/examples/srv6-sample-localsid/node.c index 7bae9cd7..e83e2352 100644 --- a/src/examples/srv6-sample-localsid/node.c +++ b/src/examples/srv6-sample-localsid/node.c @@ -114,7 +114,7 @@ srv6_localsid_sample_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_fram from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; next_index = node->cached_next_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -168,7 +168,7 @@ srv6_localsid_sample_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_fram /* This increments the SRv6 per LocalSID counters.*/ vlib_increment_combined_counter (((next0 == SRV6_SAMPLE_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : &(sm->sr_ls_valid_counters)), - cpu_index, + thread_index, ls0 - sm->localsids, 1, vlib_buffer_length_in_chain (vm, b0)); diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c index 2765c292..c80b3fa8 100644 --- a/src/plugins/dpdk/buffer.c +++ b/src/plugins/dpdk/buffer.c @@ -132,7 +132,7 @@ dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index) u32 merge_index; int i; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); f = vlib_buffer_get_free_list (vm, free_list_index); diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index 50b26689..91661246 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -243,7 +243,7 @@ static_always_inline ASSERT (ring->tx_tail == 0); n_retry = 16; - queue_id = vm->cpu_index; + queue_id = vm->thread_index; do { @@ -266,7 +266,7 @@ static_always_inline { /* no wrap, transmit in one burst */ dpdk_device_hqos_per_worker_thread_t *hqos = - &xd->hqos_wt[vm->cpu_index]; + &xd->hqos_wt[vm->thread_index]; ASSERT (hqos->swq != NULL); @@ -332,7 +332,7 @@ dpdk_buffer_recycle (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp) { dpdk_main_t *dm = &dpdk_main; - u32 my_cpu = vm->cpu_index; + u32 my_cpu = vm->thread_index; struct rte_mbuf *mb_new; if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0) @@ -376,7 +376,7 @@ dpdk_interface_tx (vlib_main_t * vm, tx_ring_hdr_t *ring; u32 n_on_ring; - my_cpu = vm->cpu_index; + my_cpu = vm->thread_index; queue_id = my_cpu; diff --git a/src/plugins/dpdk/device/dpdk_priv.h b/src/plugins/dpdk/device/dpdk_priv.h index dd40ff48..52b4ca4b 100644 --- a/src/plugins/dpdk/device/dpdk_priv.h +++ b/src/plugins/dpdk/device/dpdk_priv.h @@ -79,7 +79,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now) { vlib_simple_counter_main_t *cm; vnet_main_t *vnm = vnet_get_main (); - u32 my_cpu = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u64 rxerrors, last_rxerrors; /* only update counters for PMD interfaces */ @@ -96,7 +96,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now) cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, VNET_INTERFACE_COUNTER_RX_NO_BUF); - vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index, + vlib_increment_simple_counter (cm, thread_index, xd->vlib_sw_if_index, xd->stats.rx_nombuf - xd->last_stats.rx_nombuf); } @@ -107,7 +107,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now) cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, VNET_INTERFACE_COUNTER_RX_MISS); - vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index, + vlib_increment_simple_counter (cm, thread_index, xd->vlib_sw_if_index, xd->stats.imissed - xd->last_stats.imissed); } @@ -119,7 +119,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now) cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, VNET_INTERFACE_COUNTER_RX_ERROR); - vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index, + vlib_increment_simple_counter (cm, thread_index, xd->vlib_sw_if_index, rxerrors - last_rxerrors); } diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index 538db6cb..7eaf8da7 100755 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -324,7 +324,7 @@ dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd) int rv; int j; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) { diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index e740fd18..b10e0fad 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -283,7 +283,7 @@ dpdk_buffer_init_from_template (void *d0, void *d1, void *d2, void *d3, */ static_always_inline u32 dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, - vlib_node_runtime_t * node, u32 cpu_index, u16 queue_id, + vlib_node_runtime_t * node, u32 thread_index, u16 queue_id, int maybe_multiseg) { u32 n_buffers; @@ -294,7 +294,7 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, uword n_rx_bytes = 0; u32 n_trace, trace_cnt __attribute__ ((unused)); vlib_buffer_free_list_t *fl; - vlib_buffer_t *bt = vec_elt_at_index (dm->buffer_templates, cpu_index); + vlib_buffer_t *bt = vec_elt_at_index (dm->buffer_templates, thread_index); if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0) return 0; @@ -306,7 +306,7 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, return 0; } - vec_reset_length (xd->d_trace_buffers[cpu_index]); + vec_reset_length (xd->d_trace_buffers[thread_index]); trace_cnt = n_trace = vlib_get_trace_count (vm, node); if (n_trace > 0) @@ -318,7 +318,7 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, { struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index++]; vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb); - vec_add1 (xd->d_trace_buffers[cpu_index], + vec_add1 (xd->d_trace_buffers[thread_index], vlib_get_buffer_index (vm, b)); } } @@ -546,20 +546,22 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, vlib_put_next_frame (vm, node, next_index, n_left_to_next); } - if (PREDICT_FALSE (vec_len (xd->d_trace_buffers[cpu_index]) > 0)) + if (PREDICT_FALSE (vec_len (xd->d_trace_buffers[thread_index]) > 0)) { - dpdk_rx_trace (dm, node, xd, queue_id, xd->d_trace_buffers[cpu_index], - vec_len (xd->d_trace_buffers[cpu_index])); - vlib_set_trace_count (vm, node, n_trace - - vec_len (xd->d_trace_buffers[cpu_index])); + dpdk_rx_trace (dm, node, xd, queue_id, + xd->d_trace_buffers[thread_index], + vec_len (xd->d_trace_buffers[thread_index])); + vlib_set_trace_count (vm, node, + n_trace - + vec_len (xd->d_trace_buffers[thread_index])); } vlib_increment_combined_counter (vnet_get_main ()->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, xd->vlib_sw_if_index, mb_index, n_rx_bytes); + thread_index, xd->vlib_sw_if_index, mb_index, n_rx_bytes); - vnet_device_increment_rx_packets (cpu_index, mb_index); + vnet_device_increment_rx_packets (thread_index, mb_index); return mb_index; } @@ -630,19 +632,19 @@ dpdk_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f) dpdk_device_t *xd; uword n_rx_packets = 0; dpdk_device_and_queue_t *dq; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); /* * Poll all devices on this cpu for input/interrupts. */ /* *INDENT-OFF* */ - vec_foreach (dq, dm->devices_by_cpu[cpu_index]) + vec_foreach (dq, dm->devices_by_cpu[thread_index]) { xd = vec_elt_at_index(dm->devices, dq->device); if (xd->flags & DPDK_DEVICE_FLAG_MAYBE_MULTISEG) - n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id, /* maybe_multiseg */ 1); + n_rx_packets += dpdk_device_input (dm, xd, node, thread_index, dq->queue_id, /* maybe_multiseg */ 1); else - n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id, /* maybe_multiseg */ 0); + n_rx_packets += dpdk_device_input (dm, xd, node, thread_index, dq->queue_id, /* maybe_multiseg */ 0); } /* *INDENT-ON* */ diff --git a/src/plugins/dpdk/hqos/hqos.c b/src/plugins/dpdk/hqos/hqos.c index a288fca7..8b251beb 100644 --- a/src/plugins/dpdk/hqos/hqos.c +++ b/src/plugins/dpdk/hqos/hqos.c @@ -397,7 +397,7 @@ static_always_inline void dpdk_hqos_thread_internal_hqos_dbg_bypass (vlib_main_t * vm) { dpdk_main_t *dm = &dpdk_main; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; u32 dev_pos; dev_pos = 0; @@ -405,12 +405,12 @@ dpdk_hqos_thread_internal_hqos_dbg_bypass (vlib_main_t * vm) { vlib_worker_thread_barrier_check (); - u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]); + u32 n_devs = vec_len (dm->devices_by_hqos_cpu[thread_index]); if (dev_pos >= n_devs) dev_pos = 0; dpdk_device_and_queue_t *dq = - vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos); + vec_elt_at_index (dm->devices_by_hqos_cpu[thread_index], dev_pos); dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device); dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht; @@ -479,7 +479,7 @@ static_always_inline void dpdk_hqos_thread_internal (vlib_main_t * vm) { dpdk_main_t *dm = &dpdk_main; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; u32 dev_pos; dev_pos = 0; @@ -487,7 +487,7 @@ dpdk_hqos_thread_internal (vlib_main_t * vm) { vlib_worker_thread_barrier_check (); - u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]); + u32 n_devs = vec_len (dm->devices_by_hqos_cpu[thread_index]); if (PREDICT_FALSE (n_devs == 0)) { dev_pos = 0; @@ -497,7 +497,7 @@ dpdk_hqos_thread_internal (vlib_main_t * vm) dev_pos = 0; dpdk_device_and_queue_t *dq = - vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos); + vec_elt_at_index (dm->devices_by_hqos_cpu[thread_index], dev_pos); dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device); dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht; @@ -586,7 +586,7 @@ dpdk_hqos_thread (vlib_worker_thread_t * w) vm = vlib_get_main (); - ASSERT (vm->cpu_index == os_get_cpu_number ()); + ASSERT (vm->thread_index == vlib_get_thread_index ()); clib_time_init (&vm->clib_time); clib_mem_set_heap (w->thread_mheap); @@ -595,7 +595,7 @@ dpdk_hqos_thread (vlib_worker_thread_t * w) while (tm->worker_thread_release == 0) vlib_worker_thread_barrier_check (); - if (vec_len (dm->devices_by_hqos_cpu[vm->cpu_index]) == 0) + if (vec_len (dm->devices_by_hqos_cpu[vm->thread_index]) == 0) return clib_error ("current I/O TX thread does not have any devices assigned to it"); diff --git a/src/plugins/dpdk/ipsec/cli.c b/src/plugins/dpdk/ipsec/cli.c index cd0a6037..3ae8c9b8 100644 --- a/src/plugins/dpdk/ipsec/cli.c +++ b/src/plugins/dpdk/ipsec/cli.c @@ -42,8 +42,8 @@ dpdk_ipsec_show_mapping (vlib_main_t * vm, u16 detail_display) for (i = 0; i < tm->n_vlib_mains; i++) { uword key, data; - u32 cpu_index = vlib_mains[i]->cpu_index; - crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index]; + u32 thread_index = vlib_mains[i]->thread_index; + crypto_worker_main_t *cwm = &dcm->workers_main[thread_index]; u8 *s = 0; if (skip_master) @@ -57,7 +57,7 @@ dpdk_ipsec_show_mapping (vlib_main_t * vm, u16 detail_display) i32 last_cdev = -1; crypto_qp_data_t *qpd; - s = format (s, "%u\t", cpu_index); + s = format (s, "%u\t", thread_index); /* *INDENT-OFF* */ vec_foreach (qpd, cwm->qp_data) @@ -95,7 +95,7 @@ dpdk_ipsec_show_mapping (vlib_main_t * vm, u16 detail_display) cap.sym.auth.algo = p_key->auth_algo; check_algo_is_supported (&cap, auth_str); vlib_cli_output (vm, "%u\t%10s\t%15s\t%3s\t%u\t%u\n", - vlib_mains[i]->cpu_index, cipher_str, auth_str, + vlib_mains[i]->thread_index, cipher_str, auth_str, p_key->is_outbound ? "out" : "in", cwm->qp_data[data].dev_id, cwm->qp_data[data].qp_id); diff --git a/src/plugins/dpdk/ipsec/crypto_node.c b/src/plugins/dpdk/ipsec/crypto_node.c index dc3452b2..a3c45902 100644 --- a/src/plugins/dpdk/ipsec/crypto_node.c +++ b/src/plugins/dpdk/ipsec/crypto_node.c @@ -171,9 +171,9 @@ static uword dpdk_crypto_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); dpdk_crypto_main_t *dcm = &dpdk_crypto_main; - crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index]; + crypto_worker_main_t *cwm = &dcm->workers_main[thread_index]; crypto_qp_data_t *qpd; u32 n_deq = 0; diff --git a/src/plugins/dpdk/ipsec/esp.h b/src/plugins/dpdk/ipsec/esp.h index 320295b1..56f0c756 100644 --- a/src/plugins/dpdk/ipsec/esp.h +++ b/src/plugins/dpdk/ipsec/esp.h @@ -170,9 +170,9 @@ static_always_inline int create_sym_sess (ipsec_sa_t * sa, crypto_sa_session_t * sa_sess, u8 is_outbound) { - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); dpdk_crypto_main_t *dcm = &dpdk_crypto_main; - crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index]; + crypto_worker_main_t *cwm = &dcm->workers_main[thread_index]; struct rte_crypto_sym_xform cipher_xform = { 0 }; struct rte_crypto_sym_xform auth_xform = { 0 }; struct rte_crypto_sym_xform *xfs; diff --git a/src/plugins/dpdk/ipsec/esp_decrypt.c b/src/plugins/dpdk/ipsec/esp_decrypt.c index 286e03f8..bab76e3b 100644 --- a/src/plugins/dpdk/ipsec/esp_decrypt.c +++ b/src/plugins/dpdk/ipsec/esp_decrypt.c @@ -88,7 +88,7 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, { u32 n_left_from, *from, *to_next, next_index; ipsec_main_t *im = &ipsec_main; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); dpdk_crypto_main_t * dcm = &dpdk_crypto_main; dpdk_esp_main_t * em = &dpdk_esp_main; u32 i; @@ -104,7 +104,7 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, return n_left_from; } - crypto_worker_main_t *cwm = vec_elt_at_index(dcm->workers_main, cpu_index); + crypto_worker_main_t *cwm = vec_elt_at_index(dcm->workers_main, thread_index); u32 n_qps = vec_len(cwm->qp_data); struct rte_crypto_op ** cops_to_enq[n_qps]; u32 n_cop_qp[n_qps], * bi_to_enq[n_qps]; diff --git a/src/plugins/dpdk/ipsec/esp_encrypt.c b/src/plugins/dpdk/ipsec/esp_encrypt.c index 5b03de73..f996d7df 100644 --- a/src/plugins/dpdk/ipsec/esp_encrypt.c +++ b/src/plugins/dpdk/ipsec/esp_encrypt.c @@ -93,7 +93,7 @@ dpdk_esp_encrypt_node_fn (vlib_main_t * vm, { u32 n_left_from, *from, *to_next, next_index; ipsec_main_t *im = &ipsec_main; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); dpdk_crypto_main_t *dcm = &dpdk_crypto_main; dpdk_esp_main_t *em = &dpdk_esp_main; u32 i; @@ -111,7 +111,8 @@ dpdk_esp_encrypt_node_fn (vlib_main_t * vm, return n_left_from; } - crypto_worker_main_t *cwm = vec_elt_at_index (dcm->workers_main, cpu_index); + crypto_worker_main_t *cwm = + vec_elt_at_index (dcm->workers_main, thread_index); u32 n_qps = vec_len (cwm->qp_data); struct rte_crypto_op **cops_to_enq[n_qps]; u32 n_cop_qp[n_qps], *bi_to_enq[n_qps]; diff --git a/src/plugins/dpdk/ipsec/ipsec.c b/src/plugins/dpdk/ipsec/ipsec.c index b0aaaaec..5d8f4fba 100644 --- a/src/plugins/dpdk/ipsec/ipsec.c +++ b/src/plugins/dpdk/ipsec/ipsec.c @@ -289,7 +289,7 @@ dpdk_ipsec_process (vlib_main_t * vm, vlib_node_runtime_t * rt, if (!map) { clib_warning ("unable to create hash table for worker %u", - vlib_mains[i]->cpu_index); + vlib_mains[i]->thread_index); goto error; } cwm->algo_qp_map = map; diff --git a/src/plugins/dpdk/ipsec/ipsec.h b/src/plugins/dpdk/ipsec/ipsec.h index 28bffc80..f0f793c0 100644 --- a/src/plugins/dpdk/ipsec/ipsec.h +++ b/src/plugins/dpdk/ipsec/ipsec.h @@ -95,8 +95,8 @@ static_always_inline void crypto_alloc_cops () { dpdk_crypto_main_t *dcm = &dpdk_crypto_main; - u32 cpu_index = os_get_cpu_number (); - crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index]; + u32 thread_index = vlib_get_thread_index (); + crypto_worker_main_t *cwm = &dcm->workers_main[thread_index]; unsigned socket_id = rte_socket_id (); crypto_qp_data_t *qpd; diff --git a/src/plugins/dpdk/main.c b/src/plugins/dpdk/main.c index 7ee2a785..942b8b2d 100644 --- a/src/plugins/dpdk/main.c +++ b/src/plugins/dpdk/main.c @@ -39,7 +39,7 @@ rte_delay_us_override (unsigned us) * thread then do not intercept. (Must not be called from an * independent pthread). */ - if (os_get_cpu_number () == 0) + if (vlib_get_thread_index () == 0) { /* * We're in the vlib main thread or a vlib process. Make sure diff --git a/src/plugins/flowperpkt/l2_node.c b/src/plugins/flowperpkt/l2_node.c index 1c2f681e..fdaf81d1 100644 --- a/src/plugins/flowperpkt/l2_node.c +++ b/src/plugins/flowperpkt/l2_node.c @@ -102,7 +102,7 @@ add_to_flow_record_l2 (vlib_main_t * vm, u8 * src_mac, u8 * dst_mac, u16 ethertype, u64 timestamp, u16 length, int do_flush) { - u32 my_cpu_number = vm->cpu_index; + u32 my_cpu_number = vm->thread_index; flow_report_main_t *frm = &flow_report_main; ip4_header_t *ip; udp_header_t *udp; diff --git a/src/plugins/flowperpkt/node.c b/src/plugins/flowperpkt/node.c index f77f087d..0277682d 100644 --- a/src/plugins/flowperpkt/node.c +++ b/src/plugins/flowperpkt/node.c @@ -101,7 +101,7 @@ add_to_flow_record_ipv4 (vlib_main_t * vm, u32 src_address, u32 dst_address, u8 tos, u64 timestamp, u16 length, int do_flush) { - u32 my_cpu_number = vm->cpu_index; + u32 my_cpu_number = vm->thread_index; flow_report_main_t *frm = &flow_report_main; ip4_header_t *ip; udp_header_t *udp; diff --git a/src/plugins/ioam/export-common/ioam_export.h b/src/plugins/ioam/export-common/ioam_export.h index 2bf3fd54..9de0d13b 100644 --- a/src/plugins/ioam/export-common/ioam_export.h +++ b/src/plugins/ioam/export-common/ioam_export.h @@ -477,8 +477,8 @@ do { \ from = vlib_frame_vector_args (F); \ n_left_from = (F)->n_vectors; \ next_index = (N)->cached_next_index; \ - while (__sync_lock_test_and_set ((EM)->lockp[(VM)->cpu_index], 1)); \ - my_buf = ioam_export_get_my_buffer (EM, (VM)->cpu_index); \ + while (__sync_lock_test_and_set ((EM)->lockp[(VM)->thread_index], 1)); \ + my_buf = ioam_export_get_my_buffer (EM, (VM)->thread_index); \ my_buf->touched_at = vlib_time_now (VM); \ while (n_left_from > 0) \ { \ @@ -620,7 +620,7 @@ do { \ } \ vlib_node_increment_counter (VM, export_node.index, \ EXPORT_ERROR_RECORDED, pkts_recorded); \ - *(EM)->lockp[(VM)->cpu_index] = 0; \ + *(EM)->lockp[(VM)->thread_index] = 0; \ } while(0) #endif /* __included_ioam_export_h__ */ diff --git a/src/plugins/ioam/ip6/ioam_cache_tunnel_select_node.c b/src/plugins/ioam/ip6/ioam_cache_tunnel_select_node.c index a56dc040..0cf742c9 100644 --- a/src/plugins/ioam/ip6/ioam_cache_tunnel_select_node.c +++ b/src/plugins/ioam/ip6/ioam_cache_tunnel_select_node.c @@ -396,7 +396,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, clib_net_to_host_u32 (tcp0->seq_number) + 1, no_of_responses, now, - vm->cpu_index, &pool_index0)) + vm->thread_index, &pool_index0)) { cache_ts_added++; } @@ -419,7 +419,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, e2e = (ioam_e2e_cache_option_t *) ((u8 *) hbh0 + cm->rewrite_pool_index_offset); - e2e->pool_id = (u8) vm->cpu_index; + e2e->pool_id = (u8) vm->thread_index; e2e->pool_index = pool_index0; ioam_e2e_id_rewrite_handler ((ioam_e2e_id_option_t *) ((u8 *) e2e + @@ -455,7 +455,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, clib_net_to_host_u32 (tcp1->seq_number) + 1, no_of_responses, now, - vm->cpu_index, &pool_index1)) + vm->thread_index, &pool_index1)) { cache_ts_added++; } @@ -479,7 +479,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, e2e = (ioam_e2e_cache_option_t *) ((u8 *) hbh1 + cm->rewrite_pool_index_offset); - e2e->pool_id = (u8) vm->cpu_index; + e2e->pool_id = (u8) vm->thread_index; e2e->pool_index = pool_index1; ioam_e2e_id_rewrite_handler ((ioam_e2e_id_option_t *) ((u8 *) e2e + @@ -562,7 +562,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, clib_net_to_host_u32 (tcp0->seq_number) + 1, no_of_responses, now, - vm->cpu_index, &pool_index0)) + vm->thread_index, &pool_index0)) { cache_ts_added++; } @@ -585,7 +585,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, e2e = (ioam_e2e_cache_option_t *) ((u8 *) hbh0 + cm->rewrite_pool_index_offset); - e2e->pool_id = (u8) vm->cpu_index; + e2e->pool_id = (u8) vm->thread_index; e2e->pool_index = pool_index0; ioam_e2e_id_rewrite_handler ((ioam_e2e_id_option_t *) ((u8 *) e2e + @@ -701,7 +701,7 @@ expired_cache_ts_timer_callback (u32 * expired_timers) ioam_cache_main_t *cm = &ioam_cache_main; int i; u32 pool_index; - u32 thread_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 count = 0; for (i = 0; i < vec_len (expired_timers); i++) @@ -724,7 +724,7 @@ ioam_cache_ts_timer_tick_node_fn (vlib_main_t * vm, vlib_frame_t * f) { ioam_cache_main_t *cm = &ioam_cache_main; - u32 my_thread_index = os_get_cpu_number (); + u32 my_thread_index = vlib_get_thread_index (); struct timespec ts, tsrem; tw_timer_expire_timers_16t_2w_512sl (&cm->timer_wheels[my_thread_index], diff --git a/src/plugins/ixge/ixge.c b/src/plugins/ixge/ixge.c index f3c5cc09..08f5b692 100644 --- a/src/plugins/ixge/ixge.c +++ b/src/plugins/ixge/ixge.c @@ -1887,7 +1887,7 @@ done: vlib_increment_combined_counter (vnet_main. interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - 0 /* cpu_index */ , + 0 /* thread_index */ , xd->vlib_sw_if_index, n_packets, dq->rx.n_bytes); diff --git a/src/plugins/lb/lb.c b/src/plugins/lb/lb.c index add81236..addc2a42 100644 --- a/src/plugins/lb/lb.c +++ b/src/plugins/lb/lb.c @@ -63,11 +63,11 @@ u8 *format_lb_main (u8 * s, va_list * args) s = format(s, " #vips: %u\n", pool_elts(lbm->vips)); s = format(s, " #ass: %u\n", pool_elts(lbm->ass) - 1); - u32 cpu_index; - for(cpu_index = 0; cpu_index < tm->n_vlib_mains; cpu_index++ ) { - lb_hash_t *h = lbm->per_cpu[cpu_index].sticky_ht; + u32 thread_index; + for(thread_index = 0; thread_index < tm->n_vlib_mains; thread_index++ ) { + lb_hash_t *h = lbm->per_cpu[thread_index].sticky_ht; if (h) { - s = format(s, "core %d\n", cpu_index); + s = format(s, "core %d\n", thread_index); s = format(s, " timeout: %ds\n", h->timeout); s = format(s, " usage: %d / %d\n", lb_hash_elts(h, lb_hash_time_now(vlib_get_main())), lb_hash_size(h)); } diff --git a/src/plugins/lb/node.c b/src/plugins/lb/node.c index 8b763c53..3171148b 100644 --- a/src/plugins/lb/node.c +++ b/src/plugins/lb/node.c @@ -60,10 +60,10 @@ format_lb_trace (u8 * s, va_list * args) return s; } -lb_hash_t *lb_get_sticky_table(u32 cpu_index) +lb_hash_t *lb_get_sticky_table(u32 thread_index) { lb_main_t *lbm = &lb_main; - lb_hash_t *sticky_ht = lbm->per_cpu[cpu_index].sticky_ht; + lb_hash_t *sticky_ht = lbm->per_cpu[thread_index].sticky_ht; //Check if size changed if (PREDICT_FALSE(sticky_ht && (lbm->per_cpu_sticky_buckets != lb_hash_nbuckets(sticky_ht)))) { @@ -71,8 +71,8 @@ lb_hash_t *lb_get_sticky_table(u32 cpu_index) lb_hash_bucket_t *b; u32 i; lb_hash_foreach_entry(sticky_ht, b, i) { - vlib_refcount_add(&lbm->as_refcount, cpu_index, b->value[i], -1); - vlib_refcount_add(&lbm->as_refcount, cpu_index, 0, 1); + vlib_refcount_add(&lbm->as_refcount, thread_index, b->value[i], -1); + vlib_refcount_add(&lbm->as_refcount, thread_index, 0, 1); } lb_hash_free(sticky_ht); @@ -81,8 +81,8 @@ lb_hash_t *lb_get_sticky_table(u32 cpu_index) //Create if necessary if (PREDICT_FALSE(sticky_ht == NULL)) { - lbm->per_cpu[cpu_index].sticky_ht = lb_hash_alloc(lbm->per_cpu_sticky_buckets, lbm->flow_timeout); - sticky_ht = lbm->per_cpu[cpu_index].sticky_ht; + lbm->per_cpu[thread_index].sticky_ht = lb_hash_alloc(lbm->per_cpu_sticky_buckets, lbm->flow_timeout); + sticky_ht = lbm->per_cpu[thread_index].sticky_ht; clib_warning("Regenerated sticky table %p", sticky_ht); } @@ -153,10 +153,10 @@ lb_node_fn (vlib_main_t * vm, { lb_main_t *lbm = &lb_main; u32 n_left_from, *from, next_index, *to_next, n_left_to_next; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); u32 lb_time = lb_hash_time_now(vm); - lb_hash_t *sticky_ht = lb_get_sticky_table(cpu_index); + lb_hash_t *sticky_ht = lb_get_sticky_table(thread_index); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; next_index = node->cached_next_index; @@ -240,9 +240,9 @@ lb_node_fn (vlib_main_t * vm, //Configuration may be changed, vectors resized, etc... //Dereference previously used - vlib_refcount_add(&lbm->as_refcount, cpu_index, + vlib_refcount_add(&lbm->as_refcount, thread_index, lb_hash_available_value(sticky_ht, hash0, available_index0), -1); - vlib_refcount_add(&lbm->as_refcount, cpu_index, + vlib_refcount_add(&lbm->as_refcount, thread_index, asindex0, 1); //Add sticky entry @@ -260,7 +260,7 @@ lb_node_fn (vlib_main_t * vm, } vlib_increment_simple_counter(&lbm->vip_counters[counter], - cpu_index, + thread_index, vnet_buffer (p0)->ip.adj_index[VLIB_TX], 1); diff --git a/src/plugins/lb/refcount.c b/src/plugins/lb/refcount.c index 22415c88..6f01ab5a 100644 --- a/src/plugins/lb/refcount.c +++ b/src/plugins/lb/refcount.c @@ -31,10 +31,10 @@ u64 vlib_refcount_get(vlib_refcount_t *r, u32 index) { u64 count = 0; vlib_thread_main_t *tm = vlib_get_thread_main (); - u32 cpu_index; - for (cpu_index = 0; cpu_index < tm->n_vlib_mains; cpu_index++) { - if (r->per_cpu[cpu_index].length > index) - count += r->per_cpu[cpu_index].counters[index]; + u32 thread_index; + for (thread_index = 0; thread_index < tm->n_vlib_mains; thread_index++) { + if (r->per_cpu[thread_index].length > index) + count += r->per_cpu[thread_index].counters[index]; } return count; } diff --git a/src/plugins/lb/refcount.h b/src/plugins/lb/refcount.h index 8c26e7be..dcfcb3fe 100644 --- a/src/plugins/lb/refcount.h +++ b/src/plugins/lb/refcount.h @@ -45,9 +45,9 @@ typedef struct { void __vlib_refcount_resize(vlib_refcount_per_cpu_t *per_cpu, u32 size); static_always_inline -void vlib_refcount_add(vlib_refcount_t *r, u32 cpu_index, u32 counter_index, i32 v) +void vlib_refcount_add(vlib_refcount_t *r, u32 thread_index, u32 counter_index, i32 v) { - vlib_refcount_per_cpu_t *per_cpu = &r->per_cpu[cpu_index]; + vlib_refcount_per_cpu_t *per_cpu = &r->per_cpu[thread_index]; if (PREDICT_FALSE(counter_index >= per_cpu->length)) __vlib_refcount_resize(per_cpu, clib_max(counter_index + 16, per_cpu->length * 2)); diff --git a/src/plugins/memif/node.c b/src/plugins/memif/node.c index 659d5dfb..cee1f3d1 100644 --- a/src/plugins/memif/node.c +++ b/src/plugins/memif/node.c @@ -94,7 +94,7 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u32 n_rx_bytes = 0; u32 *to_next = 0; u32 n_free_bufs; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 bi0, bi1; vlib_buffer_t *b0, *b1; u16 ring_size = 1 << mif->log2_ring_size; @@ -105,14 +105,15 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (mif->per_interface_next_index != ~0) next_index = mif->per_interface_next_index; - n_free_bufs = vec_len (nm->rx_buffers[cpu_index]); + n_free_bufs = vec_len (nm->rx_buffers[thread_index]); if (PREDICT_FALSE (n_free_bufs < ring_size)) { - vec_validate (nm->rx_buffers[cpu_index], ring_size + n_free_bufs - 1); + vec_validate (nm->rx_buffers[thread_index], + ring_size + n_free_bufs - 1); n_free_bufs += - vlib_buffer_alloc (vm, &nm->rx_buffers[cpu_index][n_free_bufs], + vlib_buffer_alloc (vm, &nm->rx_buffers[thread_index][n_free_bufs], ring_size); - _vec_len (nm->rx_buffers[cpu_index]) = n_free_bufs; + _vec_len (nm->rx_buffers[thread_index]) = n_free_bufs; } head = ring->head; @@ -158,15 +159,15 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, CLIB_CACHE_LINE_BYTES, LOAD); } /* get empty buffer */ - u32 last_buf = vec_len (nm->rx_buffers[cpu_index]) - 1; - bi0 = nm->rx_buffers[cpu_index][last_buf]; - bi1 = nm->rx_buffers[cpu_index][last_buf - 1]; - _vec_len (nm->rx_buffers[cpu_index]) -= 2; + u32 last_buf = vec_len (nm->rx_buffers[thread_index]) - 1; + bi0 = nm->rx_buffers[thread_index][last_buf]; + bi1 = nm->rx_buffers[thread_index][last_buf - 1]; + _vec_len (nm->rx_buffers[thread_index]) -= 2; if (last_buf > 4) { - memif_prefetch (vm, nm->rx_buffers[cpu_index][last_buf - 2]); - memif_prefetch (vm, nm->rx_buffers[cpu_index][last_buf - 3]); + memif_prefetch (vm, nm->rx_buffers[thread_index][last_buf - 2]); + memif_prefetch (vm, nm->rx_buffers[thread_index][last_buf - 3]); } /* enqueue buffer */ @@ -256,9 +257,9 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, while (num_slots && n_left_to_next) { /* get empty buffer */ - u32 last_buf = vec_len (nm->rx_buffers[cpu_index]) - 1; - bi0 = nm->rx_buffers[cpu_index][last_buf]; - _vec_len (nm->rx_buffers[cpu_index]) = last_buf; + u32 last_buf = vec_len (nm->rx_buffers[thread_index]) - 1; + bi0 = nm->rx_buffers[thread_index][last_buf]; + _vec_len (nm->rx_buffers[thread_index]) = last_buf; /* enqueue buffer */ to_next[0] = bi0; @@ -315,7 +316,7 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, ring->tail = head; vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters - + VNET_INTERFACE_COUNTER_RX, cpu_index, + + VNET_INTERFACE_COUNTER_RX, thread_index, mif->hw_if_index, n_rx_packets, n_rx_bytes); @@ -327,7 +328,7 @@ memif_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { u32 n_rx_packets = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); memif_main_t *nm = &memif_main; memif_if_t *mif; @@ -337,7 +338,7 @@ memif_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, if (mif->flags & MEMIF_IF_FLAG_ADMIN_UP && mif->flags & MEMIF_IF_FLAG_CONNECTED && (mif->if_index % nm->input_cpu_count) == - (cpu_index - nm->input_cpu_first_index)) + (thread_index - nm->input_cpu_first_index)) { if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE) n_rx_packets += diff --git a/src/plugins/snat/in2out.c b/src/plugins/snat/in2out.c index b4961365..e5ee965f 100644 --- a/src/plugins/snat/in2out.c +++ b/src/plugins/snat/in2out.c @@ -212,7 +212,7 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, snat_session_t ** sessionp, vlib_node_runtime_t * node, u32 next0, - u32 cpu_index) + u32 thread_index) { snat_user_t *u; snat_user_key_t user_key; @@ -246,27 +246,27 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0)) { /* no, make a new one */ - pool_get (sm->per_thread_data[cpu_index].users, u); + pool_get (sm->per_thread_data[thread_index].users, u); memset (u, 0, sizeof (*u)); u->addr = ip0->src_address; u->fib_index = rx_fib_index0; - pool_get (sm->per_thread_data[cpu_index].list_pool, per_user_list_head_elt); + pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt); u->sessions_per_user_list_head_index = per_user_list_head_elt - - sm->per_thread_data[cpu_index].list_pool; + sm->per_thread_data[thread_index].list_pool; - clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_init (sm->per_thread_data[thread_index].list_pool, u->sessions_per_user_list_head_index); - kv0.value = u - sm->per_thread_data[cpu_index].users; + kv0.value = u - sm->per_thread_data[thread_index].users; /* add user */ clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */); } else { - u = pool_elt_at_index (sm->per_thread_data[cpu_index].users, + u = pool_elt_at_index (sm->per_thread_data[thread_index].users, value0.value); } @@ -276,25 +276,25 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, /* Remove the oldest dynamic translation */ do { oldest_per_user_translation_list_index = - clib_dlist_remove_head (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool, u->sessions_per_user_list_head_index); ASSERT (oldest_per_user_translation_list_index != ~0); /* add it back to the end of the LRU list */ - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, u->sessions_per_user_list_head_index, oldest_per_user_translation_list_index); /* Get the list element */ oldest_per_user_translation_list_elt = - pool_elt_at_index (sm->per_thread_data[cpu_index].list_pool, + pool_elt_at_index (sm->per_thread_data[thread_index].list_pool, oldest_per_user_translation_list_index); /* Get the session index from the list element */ session_index = oldest_per_user_translation_list_elt->value; /* Get the session */ - s = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, session_index); } while (snat_is_session_static (s)); @@ -346,7 +346,7 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, } /* Create a new session */ - pool_get (sm->per_thread_data[cpu_index].sessions, s); + pool_get (sm->per_thread_data[thread_index].sessions, s); memset (s, 0, sizeof (*s)); s->outside_address_index = address_index; @@ -362,22 +362,22 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, } /* Create list elts */ - pool_get (sm->per_thread_data[cpu_index].list_pool, + pool_get (sm->per_thread_data[thread_index].list_pool, per_user_translation_list_elt); - clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_init (sm->per_thread_data[thread_index].list_pool, per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool); + sm->per_thread_data[thread_index].list_pool); per_user_translation_list_elt->value = - s - sm->per_thread_data[cpu_index].sessions; + s - sm->per_thread_data[thread_index].sessions; s->per_user_index = per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool; + sm->per_thread_data[thread_index].list_pool; s->per_user_list_head_index = u->sessions_per_user_list_head_index; - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s->per_user_list_head_index, per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool); + sm->per_thread_data[thread_index].list_pool); } s->in2out = *key0; @@ -388,12 +388,12 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, /* Add to translation hashes */ kv0.key = s->in2out.as_u64; - kv0.value = s - sm->per_thread_data[cpu_index].sessions; + kv0.value = s - sm->per_thread_data[thread_index].sessions; if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */)) clib_warning ("in2out key add failed"); kv0.key = s->out2in.as_u64; - kv0.value = s - sm->per_thread_data[cpu_index].sessions; + kv0.value = s - sm->per_thread_data[thread_index].sessions; if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */)) clib_warning ("out2in key add failed"); @@ -403,7 +403,7 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, worker_by_out_key.port = s->out2in.port; worker_by_out_key.fib_index = s->out2in.fib_index; kv0.key = worker_by_out_key.as_u64; - kv0.value = cpu_index; + kv0.value = thread_index; clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1); /* log NAT event */ @@ -465,7 +465,7 @@ snat_in2out_error_t icmp_get_key(icmp46_header_t *icmp0, * * @param[in,out] sm SNAT main * @param[in,out] node SNAT node runtime - * @param[in] cpu_index CPU index + * @param[in] thread_index thread index * @param[in,out] b0 buffer containing packet to be translated * @param[out] p_key address and port before NAT translation * @param[out] p_value address and port after NAT translation @@ -473,7 +473,7 @@ snat_in2out_error_t icmp_get_key(icmp46_header_t *icmp0, * @param d optional parameter */ u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d) @@ -524,13 +524,13 @@ u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node, } next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, - &s0, node, next0, cpu_index); + &s0, node, next0, thread_index); if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP)) goto out; } else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value0.value); out: @@ -548,7 +548,7 @@ out: * * @param[in] sm SNAT main * @param[in,out] node SNAT node runtime - * @param[in] cpu_index CPU index + * @param[in] thread_index thread index * @param[in,out] b0 buffer containing packet to be translated * @param[out] p_key address and port before NAT translation * @param[out] p_value address and port after NAT translation @@ -556,7 +556,7 @@ out: * @param d optional parameter */ u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d) @@ -624,7 +624,7 @@ static inline u32 icmp_in2out (snat_main_t *sm, u32 rx_fib_index0, vlib_node_runtime_t * node, u32 next0, - u32 cpu_index, + u32 thread_index, void *d) { snat_session_key_t key0, sm0; @@ -641,7 +641,7 @@ static inline u32 icmp_in2out (snat_main_t *sm, echo0 = (icmp_echo_header_t *)(icmp0+1); - next0_tmp = sm->icmp_match_in2out_cb(sm, node, cpu_index, b0, + next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, &key0, &sm0, &dont_translate, d); if (next0_tmp != ~0) next0 = next0_tmp; @@ -847,11 +847,11 @@ static inline u32 icmp_in2out_slow_path (snat_main_t *sm, vlib_node_runtime_t * node, u32 next0, f64 now, - u32 cpu_index, + u32 thread_index, snat_session_t ** p_s0) { next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, cpu_index, p_s0); + next0, thread_index, p_s0); snat_session_t * s0 = *p_s0; if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0)) { @@ -862,9 +862,9 @@ static inline u32 icmp_in2out_slow_path (snat_main_t *sm, /* Per-user LRU list maintenance for dynamic translations */ if (!snat_is_session_static (s0)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s0->per_user_list_head_index, s0->per_user_index); } @@ -884,7 +884,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; f64 now = vlib_time_now (vm); u32 stats_node_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index : snat_in2out_node.index; @@ -977,7 +977,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, { next0 = icmp_in2out_slow_path (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, - node, next0, now, cpu_index, &s0); + node, next0, now, thread_index, &s0); goto trace00; } } @@ -1006,7 +1006,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, goto trace00; next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, - &s0, node, next0, cpu_index); + &s0, node, next0, thread_index); if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP)) goto trace00; } @@ -1017,7 +1017,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, } } else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value0.value); old_addr0 = ip0->src_address.as_u32; @@ -1063,9 +1063,9 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s0)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s0->per_user_list_head_index, s0->per_user_index); } @@ -1081,7 +1081,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, t->next_index = next0; t->session_index = ~0; if (s0) - t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; + t->session_index = s0 - sm->per_thread_data[thread_index].sessions; } pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP; @@ -1117,7 +1117,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, { next1 = icmp_in2out_slow_path (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, - next1, now, cpu_index, &s1); + next1, now, thread_index, &s1); goto trace01; } } @@ -1146,7 +1146,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, goto trace01; next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1, - &s1, node, next1, cpu_index); + &s1, node, next1, thread_index); if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP)) goto trace01; } @@ -1157,7 +1157,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, } } else - s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value1.value); old_addr1 = ip1->src_address.as_u32; @@ -1203,9 +1203,9 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s1)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s1->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s1->per_user_list_head_index, s1->per_user_index); } @@ -1220,7 +1220,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, t->next_index = next1; t->session_index = ~0; if (s1) - t->session_index = s1 - sm->per_thread_data[cpu_index].sessions; + t->session_index = s1 - sm->per_thread_data[thread_index].sessions; } pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP; @@ -1292,7 +1292,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, { next0 = icmp_in2out_slow_path (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, now, cpu_index, &s0); + next0, now, thread_index, &s0); goto trace0; } } @@ -1321,7 +1321,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, goto trace0; next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, - &s0, node, next0, cpu_index); + &s0, node, next0, thread_index); if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP)) goto trace0; @@ -1333,7 +1333,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, } } else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value0.value); old_addr0 = ip0->src_address.as_u32; @@ -1379,9 +1379,9 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s0)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s0->per_user_list_head_index, s0->per_user_index); } @@ -1397,7 +1397,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, t->next_index = next0; t->session_index = ~0; if (s0) - t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; + t->session_index = s0 - sm->per_thread_data[thread_index].sessions; } pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP; @@ -2010,7 +2010,7 @@ snat_in2out_worker_handoff_fn (vlib_main_t * vm, u32 n_left_to_next_worker = 0, *to_next_worker = 0; u32 next_worker_index = 0; u32 current_worker_index = ~0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); ASSERT (vec_len (sm->workers)); @@ -2048,7 +2048,7 @@ snat_in2out_worker_handoff_fn (vlib_main_t * vm, next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0); - if (PREDICT_FALSE (next_worker_index != cpu_index)) + if (PREDICT_FALSE (next_worker_index != thread_index)) { do_handoff = 1; diff --git a/src/plugins/snat/out2in.c b/src/plugins/snat/out2in.c index 656e42db..5d308d78 100644 --- a/src/plugins/snat/out2in.c +++ b/src/plugins/snat/out2in.c @@ -129,7 +129,7 @@ create_session_for_static_mapping (snat_main_t *sm, snat_session_key_t in2out, snat_session_key_t out2in, vlib_node_runtime_t * node, - u32 cpu_index) + u32 thread_index) { snat_user_t *u; snat_user_key_t user_key; @@ -146,36 +146,36 @@ create_session_for_static_mapping (snat_main_t *sm, if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0)) { /* no, make a new one */ - pool_get (sm->per_thread_data[cpu_index].users, u); + pool_get (sm->per_thread_data[thread_index].users, u); memset (u, 0, sizeof (*u)); u->addr = in2out.addr; u->fib_index = in2out.fib_index; - pool_get (sm->per_thread_data[cpu_index].list_pool, + pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt); u->sessions_per_user_list_head_index = per_user_list_head_elt - - sm->per_thread_data[cpu_index].list_pool; + sm->per_thread_data[thread_index].list_pool; - clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_init (sm->per_thread_data[thread_index].list_pool, u->sessions_per_user_list_head_index); - kv0.value = u - sm->per_thread_data[cpu_index].users; + kv0.value = u - sm->per_thread_data[thread_index].users; /* add user */ clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */); /* add non-traslated packets worker lookup */ - kv0.value = cpu_index; + kv0.value = thread_index; clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1); } else { - u = pool_elt_at_index (sm->per_thread_data[cpu_index].users, + u = pool_elt_at_index (sm->per_thread_data[thread_index].users, value0.value); } - pool_get (sm->per_thread_data[cpu_index].sessions, s); + pool_get (sm->per_thread_data[thread_index].sessions, s); memset (s, 0, sizeof (*s)); s->outside_address_index = ~0; @@ -183,22 +183,22 @@ create_session_for_static_mapping (snat_main_t *sm, u->nstaticsessions++; /* Create list elts */ - pool_get (sm->per_thread_data[cpu_index].list_pool, + pool_get (sm->per_thread_data[thread_index].list_pool, per_user_translation_list_elt); - clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_init (sm->per_thread_data[thread_index].list_pool, per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool); + sm->per_thread_data[thread_index].list_pool); per_user_translation_list_elt->value = - s - sm->per_thread_data[cpu_index].sessions; + s - sm->per_thread_data[thread_index].sessions; s->per_user_index = - per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool; + per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool; s->per_user_list_head_index = u->sessions_per_user_list_head_index; - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s->per_user_list_head_index, per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool); + sm->per_thread_data[thread_index].list_pool); s->in2out = in2out; s->out2in = out2in; @@ -206,12 +206,12 @@ create_session_for_static_mapping (snat_main_t *sm, /* Add to translation hashes */ kv0.key = s->in2out.as_u64; - kv0.value = s - sm->per_thread_data[cpu_index].sessions; + kv0.value = s - sm->per_thread_data[thread_index].sessions; if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */)) clib_warning ("in2out key add failed"); kv0.key = s->out2in.as_u64; - kv0.value = s - sm->per_thread_data[cpu_index].sessions; + kv0.value = s - sm->per_thread_data[thread_index].sessions; if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */)) clib_warning ("out2in key add failed"); @@ -298,7 +298,7 @@ is_interface_addr(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0, * * @param[in,out] sm SNAT main * @param[in,out] node SNAT node runtime - * @param[in] cpu_index CPU index + * @param[in] thread_index thread index * @param[in,out] b0 buffer containing packet to be translated * @param[out] p_key address and port before NAT translation * @param[out] p_value address and port after NAT translation @@ -306,7 +306,7 @@ is_interface_addr(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0, * @param d optional parameter */ u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d) @@ -366,7 +366,7 @@ u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node, /* Create session initiated by host from external network */ s0 = create_session_for_static_mapping(sm, b0, sm0, key0, - node, cpu_index); + node, thread_index); if (!s0) { @@ -375,7 +375,7 @@ u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node, } } else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value0.value); out: @@ -393,7 +393,7 @@ out: * * @param[in] sm SNAT main * @param[in,out] node SNAT node runtime - * @param[in] cpu_index CPU index + * @param[in] thread_index thread index * @param[in,out] b0 buffer containing packet to be translated * @param[out] p_key address and port before NAT translation * @param[out] p_value address and port after NAT translation @@ -401,7 +401,7 @@ out: * @param d optional parameter */ u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d) @@ -460,7 +460,7 @@ static inline u32 icmp_out2in (snat_main_t *sm, u32 rx_fib_index0, vlib_node_runtime_t * node, u32 next0, - u32 cpu_index, + u32 thread_index, void *d) { snat_session_key_t key0, sm0; @@ -477,7 +477,7 @@ static inline u32 icmp_out2in (snat_main_t *sm, echo0 = (icmp_echo_header_t *)(icmp0+1); - next0_tmp = sm->icmp_match_out2in_cb(sm, node, cpu_index, b0, + next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, &key0, &sm0, &dont_translate, d); if (next0_tmp != ~0) next0 = next0_tmp; @@ -589,11 +589,11 @@ static inline u32 icmp_out2in_slow_path (snat_main_t *sm, u32 rx_fib_index0, vlib_node_runtime_t * node, u32 next0, f64 now, - u32 cpu_index, + u32 thread_index, snat_session_t ** p_s0) { next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, cpu_index, p_s0); + next0, thread_index, p_s0); snat_session_t * s0 = *p_s0; if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0)) { @@ -604,9 +604,9 @@ static inline u32 icmp_out2in_slow_path (snat_main_t *sm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s0)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s0->per_user_list_head_index, s0->per_user_index); } @@ -624,7 +624,7 @@ snat_out2in_node_fn (vlib_main_t * vm, u32 pkts_processed = 0; snat_main_t * sm = &snat_main; f64 now = vlib_time_now (vm); - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -712,7 +712,7 @@ snat_out2in_node_fn (vlib_main_t * vm, { next0 = icmp_out2in_slow_path (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, now, cpu_index, &s0); + next0, now, thread_index, &s0); goto trace0; } @@ -743,7 +743,7 @@ snat_out2in_node_fn (vlib_main_t * vm, /* Create session initiated by host from external network */ s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node, - cpu_index); + thread_index); if (!s0) { b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; @@ -752,7 +752,7 @@ snat_out2in_node_fn (vlib_main_t * vm, } } else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value0.value); old_addr0 = ip0->dst_address.as_u32; @@ -796,9 +796,9 @@ snat_out2in_node_fn (vlib_main_t * vm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s0)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s0->per_user_list_head_index, s0->per_user_index); } @@ -813,7 +813,7 @@ snat_out2in_node_fn (vlib_main_t * vm, t->next_index = next0; t->session_index = ~0; if (s0) - t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; + t->session_index = s0 - sm->per_thread_data[thread_index].sessions; } pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP; @@ -847,7 +847,7 @@ snat_out2in_node_fn (vlib_main_t * vm, { next1 = icmp_out2in_slow_path (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, - next1, now, cpu_index, &s1); + next1, now, thread_index, &s1); goto trace1; } @@ -878,7 +878,7 @@ snat_out2in_node_fn (vlib_main_t * vm, /* Create session initiated by host from external network */ s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node, - cpu_index); + thread_index); if (!s1) { b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; @@ -887,7 +887,7 @@ snat_out2in_node_fn (vlib_main_t * vm, } } else - s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value1.value); old_addr1 = ip1->dst_address.as_u32; @@ -931,9 +931,9 @@ snat_out2in_node_fn (vlib_main_t * vm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s1)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s1->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s1->per_user_list_head_index, s1->per_user_index); } @@ -948,7 +948,7 @@ snat_out2in_node_fn (vlib_main_t * vm, t->next_index = next1; t->session_index = ~0; if (s1) - t->session_index = s1 - sm->per_thread_data[cpu_index].sessions; + t->session_index = s1 - sm->per_thread_data[thread_index].sessions; } pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP; @@ -1016,7 +1016,7 @@ snat_out2in_node_fn (vlib_main_t * vm, { next0 = icmp_out2in_slow_path (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, now, cpu_index, &s0); + next0, now, thread_index, &s0); goto trace00; } @@ -1048,7 +1048,7 @@ snat_out2in_node_fn (vlib_main_t * vm, /* Create session initiated by host from external network */ s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node, - cpu_index); + thread_index); if (!s0) { b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; @@ -1057,7 +1057,7 @@ snat_out2in_node_fn (vlib_main_t * vm, } } else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value0.value); old_addr0 = ip0->dst_address.as_u32; @@ -1101,9 +1101,9 @@ snat_out2in_node_fn (vlib_main_t * vm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s0)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s0->per_user_list_head_index, s0->per_user_index); } @@ -1118,7 +1118,7 @@ snat_out2in_node_fn (vlib_main_t * vm, t->next_index = next0; t->session_index = ~0; if (s0) - t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; + t->session_index = s0 - sm->per_thread_data[thread_index].sessions; } pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP; @@ -1599,7 +1599,7 @@ snat_out2in_worker_handoff_fn (vlib_main_t * vm, u32 n_left_to_next_worker = 0, *to_next_worker = 0; u32 next_worker_index = 0; u32 current_worker_index = ~0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); ASSERT (vec_len (sm->workers)); @@ -1637,7 +1637,7 @@ snat_out2in_worker_handoff_fn (vlib_main_t * vm, next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0); - if (PREDICT_FALSE (next_worker_index != cpu_index)) + if (PREDICT_FALSE (next_worker_index != thread_index)) { do_handoff = 1; diff --git a/src/plugins/snat/snat.h b/src/plugins/snat/snat.h index 017825c0..f4e1c5c0 100644 --- a/src/plugins/snat/snat.h +++ b/src/plugins/snat/snat.h @@ -221,7 +221,7 @@ struct snat_main_s; typedef u32 snat_icmp_match_function_t (struct snat_main_s *sm, vlib_node_runtime_t *node, - u32 cpu_index, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, @@ -402,22 +402,22 @@ typedef struct { } tcp_udp_header_t; u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d); u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d); u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d); u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d); diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index a517a597..be3b41ef 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -299,7 +299,7 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm, if (CLIB_DEBUG == 0) return; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); /* smp disaster check */ if (vec_len (vlib_mains) > 1) @@ -355,7 +355,7 @@ vlib_buffer_create_free_list_helper (vlib_main_t * vm, vlib_buffer_free_list_t *f; int i; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); if (!is_default && pool_elts (bm->buffer_free_list_pool) == 0) { @@ -474,7 +474,7 @@ vlib_buffer_delete_free_list_internal (vlib_main_t * vm, u32 free_list_index) u32 merge_index; int i; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); f = vlib_buffer_get_free_list (vm, free_list_index); diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index 394c336a..328660a3 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -209,7 +209,7 @@ always_inline vlib_buffer_known_state_t vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index) { vlib_buffer_main_t *bm = vm->buffer_main; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); uword *p = hash_get (bm->buffer_known_hash, buffer_index); return p ? p[0] : VLIB_BUFFER_UNKNOWN; @@ -221,7 +221,7 @@ vlib_buffer_set_known_state (vlib_main_t * vm, vlib_buffer_known_state_t state) { vlib_buffer_main_t *bm = vm->buffer_main; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); hash_set (bm->buffer_known_hash, buffer_index, state); } diff --git a/src/vlib/cli.c b/src/vlib/cli.c index f853f655..3cc95076 100644 --- a/src/vlib/cli.c +++ b/src/vlib/cli.c @@ -709,7 +709,7 @@ test_heap_validate (vlib_main_t * vm, unformat_input_t * input, { /* *INDENT-OFF* */ foreach_vlib_main({ - heap = clib_per_cpu_mheaps[this_vlib_main->cpu_index]; + heap = clib_per_cpu_mheaps[this_vlib_main->thread_index]; mheap = mheap_header(heap); mheap->flags |= MHEAP_FLAG_VALIDATE; // Turn off small object cache because it delays detection of errors @@ -722,7 +722,7 @@ test_heap_validate (vlib_main_t * vm, unformat_input_t * input, { /* *INDENT-OFF* */ foreach_vlib_main({ - heap = clib_per_cpu_mheaps[this_vlib_main->cpu_index]; + heap = clib_per_cpu_mheaps[this_vlib_main->thread_index]; mheap = mheap_header(heap); mheap->flags &= ~MHEAP_FLAG_VALIDATE; mheap->flags |= MHEAP_FLAG_SMALL_OBJECT_CACHE; @@ -733,7 +733,7 @@ test_heap_validate (vlib_main_t * vm, unformat_input_t * input, { /* *INDENT-OFF* */ foreach_vlib_main({ - heap = clib_per_cpu_mheaps[this_vlib_main->cpu_index]; + heap = clib_per_cpu_mheaps[this_vlib_main->thread_index]; mheap = mheap_header(heap); mheap_validate(heap); }); diff --git a/src/vlib/counter.h b/src/vlib/counter.h index 17a85217..60e2055d 100644 --- a/src/vlib/counter.h +++ b/src/vlib/counter.h @@ -70,17 +70,17 @@ u32 vlib_simple_counter_n_counters (const vlib_simple_counter_main_t * cm); /** Increment a simple counter @param cm - (vlib_simple_counter_main_t *) simple counter main pointer - @param cpu_index - (u32) the current cpu index + @param thread_index - (u32) the current cpu index @param index - (u32) index of the counter to increment @param increment - (u64) quantitiy to add to the counter */ always_inline void vlib_increment_simple_counter (vlib_simple_counter_main_t * cm, - u32 cpu_index, u32 index, u64 increment) + u32 thread_index, u32 index, u64 increment) { counter_t *my_counters; - my_counters = cm->counters[cpu_index]; + my_counters = cm->counters[thread_index]; my_counters[index] += increment; } @@ -201,7 +201,7 @@ void vlib_clear_combined_counters (vlib_combined_counter_main_t * cm); /** Increment a combined counter @param cm - (vlib_combined_counter_main_t *) comined counter main pointer - @param cpu_index - (u32) the current cpu index + @param thread_index - (u32) the current cpu index @param index - (u32) index of the counter to increment @param packet_increment - (u64) number of packets to add to the counter @param byte_increment - (u64) number of bytes to add to the counter @@ -209,13 +209,13 @@ void vlib_clear_combined_counters (vlib_combined_counter_main_t * cm); always_inline void vlib_increment_combined_counter (vlib_combined_counter_main_t * cm, - u32 cpu_index, + u32 thread_index, u32 index, u64 n_packets, u64 n_bytes) { vlib_counter_t *my_counters; /* Use this CPU's counter array */ - my_counters = cm->counters[cpu_index]; + my_counters = cm->counters[thread_index]; my_counters[index].packets += n_packets; my_counters[index].bytes += n_bytes; @@ -224,14 +224,14 @@ vlib_increment_combined_counter (vlib_combined_counter_main_t * cm, /** Pre-fetch a per-thread combined counter for the given object index */ always_inline void vlib_prefetch_combined_counter (const vlib_combined_counter_main_t * cm, - u32 cpu_index, u32 index) + u32 thread_index, u32 index) { vlib_counter_t *cpu_counters; /* * This CPU's index is assumed to already be in cache */ - cpu_counters = cm->counters[cpu_index]; + cpu_counters = cm->counters[thread_index]; CLIB_PREFETCH (cpu_counters + index, CLIB_CACHE_LINE_BYTES, STORE); } diff --git a/src/vlib/error.c b/src/vlib/error.c index a2c23176..e4ed4ee3 100644 --- a/src/vlib/error.c +++ b/src/vlib/error.c @@ -149,7 +149,7 @@ vlib_register_errors (vlib_main_t * vm, vlib_node_t *n = vlib_get_node (vm, node_index); uword l; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); /* Free up any previous error strings. */ if (n->n_errors > 0) diff --git a/src/vlib/global_funcs.h b/src/vlib/global_funcs.h index f51ec381..9dd01fbf 100644 --- a/src/vlib/global_funcs.h +++ b/src/vlib/global_funcs.h @@ -23,7 +23,7 @@ always_inline vlib_main_t * vlib_get_main (void) { vlib_main_t *vm; - vm = vlib_mains[os_get_cpu_number ()]; + vm = vlib_mains[vlib_get_thread_index ()]; ASSERT (vm); return vm; } diff --git a/src/vlib/main.c b/src/vlib/main.c index b22203f0..422d3e26 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -136,18 +136,18 @@ vlib_frame_alloc_to_node (vlib_main_t * vm, u32 to_node_index, else { f = clib_mem_alloc_aligned_no_fail (n, VLIB_FRAME_ALIGN); - f->cpu_index = vm->cpu_index; + f->thread_index = vm->thread_index; fi = vlib_frame_index_no_check (vm, f); } /* Poison frame when debugging. */ if (CLIB_DEBUG > 0) { - u32 save_cpu_index = f->cpu_index; + u32 save_thread_index = f->thread_index; memset (f, 0xfe, n); - f->cpu_index = save_cpu_index; + f->thread_index = save_thread_index; } /* Insert magic number. */ @@ -517,7 +517,7 @@ vlib_put_next_frame (vlib_main_t * vm, * a dangling frame reference. Each thread has its own copy of * the next_frames vector. */ - if (0 && r->cpu_index != next_runtime->cpu_index) + if (0 && r->thread_index != next_runtime->thread_index) { nf->frame_index = ~0; nf->flags &= ~(VLIB_FRAME_PENDING | VLIB_FRAME_IS_ALLOCATED); @@ -866,7 +866,7 @@ vlib_elog_main_loop_event (vlib_main_t * vm, : evm->node_call_elog_event_types, node_index), /* track */ - (vm->cpu_index ? &vlib_worker_threads[vm->cpu_index]. + (vm->thread_index ? &vlib_worker_threads[vm->thread_index]. elog_track : &em->default_track), /* data to log */ n_vectors); } @@ -963,7 +963,7 @@ dispatch_node (vlib_main_t * vm, vm->cpu_time_last_node_dispatch = last_time_stamp; - if (1 /* || vm->cpu_index == node->cpu_index */ ) + if (1 /* || vm->thread_index == node->thread_index */ ) { vlib_main_t *stat_vm; @@ -1029,7 +1029,7 @@ dispatch_node (vlib_main_t * vm, { u32 node_name, vector_length, is_polling; } *ed; - vlib_worker_thread_t *w = vlib_worker_threads + vm->cpu_index; + vlib_worker_thread_t *w = vlib_worker_threads + vm->thread_index; #endif if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT diff --git a/src/vlib/main.h b/src/vlib/main.h index 0197b4f3..329bf073 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -156,7 +156,7 @@ typedef struct vlib_main_t uword *init_functions_called; /* to compare with node runtime */ - u32 cpu_index; + u32 thread_index; void **mbuf_alloc_list; diff --git a/src/vlib/node.c b/src/vlib/node.c index dc0a4de5..bbd3a42e 100644 --- a/src/vlib/node.c +++ b/src/vlib/node.c @@ -99,7 +99,7 @@ vlib_node_runtime_update (vlib_main_t * vm, u32 node_index, u32 next_index) vlib_pending_frame_t *pf; i32 i, j, n_insert; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); vlib_worker_thread_barrier_sync (vm); diff --git a/src/vlib/node.h b/src/vlib/node.h index fc7e7da2..1e2f4c38 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -344,8 +344,8 @@ typedef struct vlib_frame_t /* Number of vector elements currently in frame. */ u16 n_vectors; - /* Owner cpuid / heap id */ - u16 cpu_index; + /* Owner thread / heap id */ + u16 thread_index; /* Scalar and vector arguments to next node. */ u8 arguments[0]; @@ -459,7 +459,7 @@ typedef struct vlib_node_runtime_t zero before first run of this node. */ - u16 cpu_index; /**< CPU this node runs on */ + u16 thread_index; /**< thread this node runs on */ u8 runtime_data[0]; /**< Function dependent node-runtime data. This data is diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h index 1f7d94e1..54e36874 100644 --- a/src/vlib/node_funcs.h +++ b/src/vlib/node_funcs.h @@ -201,9 +201,9 @@ always_inline vlib_frame_t * vlib_get_frame_no_check (vlib_main_t * vm, uword frame_index) { vlib_frame_t *f; - u32 cpu_index = frame_index & VLIB_CPU_MASK; + u32 thread_index = frame_index & VLIB_CPU_MASK; u32 offset = frame_index & VLIB_OFFSET_MASK; - vm = vlib_mains[cpu_index]; + vm = vlib_mains[thread_index]; f = vm->heap_base + offset; return f; } @@ -215,10 +215,10 @@ vlib_frame_index_no_check (vlib_main_t * vm, vlib_frame_t * f) ASSERT (((uword) f & VLIB_CPU_MASK) == 0); - vm = vlib_mains[f->cpu_index]; + vm = vlib_mains[f->thread_index]; i = ((u8 *) f - (u8 *) vm->heap_base); - return i | f->cpu_index; + return i | f->thread_index; } always_inline vlib_frame_t * diff --git a/src/vlib/threads.c b/src/vlib/threads.c index ef3a24d3..4a111f8d 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -35,27 +35,12 @@ vl (void *p) vlib_worker_thread_t *vlib_worker_threads; vlib_thread_main_t vlib_thread_main; +__thread uword vlib_thread_index = 0; + uword os_get_cpu_number (void) { - void *sp; - uword n; - u32 len; - - len = vec_len (vlib_thread_stacks); - if (len == 0) - return 0; - - /* Get any old stack address. */ - sp = &sp; - - n = ((uword) sp - (uword) vlib_thread_stacks[0]) - >> VLIB_LOG2_THREAD_STACK_SIZE; - - /* "processes" have their own stacks, and they always run in thread 0 */ - n = n >= len ? 0 : n; - - return n; + return vlib_thread_index; } uword @@ -275,21 +260,6 @@ vlib_thread_init (vlib_main_t * vm) return 0; } -vlib_worker_thread_t * -vlib_alloc_thread (vlib_main_t * vm) -{ - vlib_worker_thread_t *w; - - if (vec_len (vlib_worker_threads) >= vec_len (vlib_thread_stacks)) - { - clib_warning ("out of worker threads... Quitting..."); - exit (1); - } - vec_add2 (vlib_worker_threads, w, 1); - w->thread_stack = vlib_thread_stacks[w - vlib_worker_threads]; - return w; -} - vlib_frame_queue_t * vlib_frame_queue_alloc (int nelts) { @@ -427,7 +397,7 @@ vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index, f64 b4 = vlib_time_now_ticks (vm, before); vlib_worker_thread_barrier_check (vm, b4); /* Bad idea. Dequeue -> enqueue -> dequeue -> trouble */ - // vlib_frame_queue_dequeue (vm->cpu_index, vm, nm); + // vlib_frame_queue_dequeue (vm->thread_index, vm, nm); } elt = fq->elts + (new_tail & (fq->nelts - 1)); @@ -497,6 +467,8 @@ vlib_worker_thread_bootstrap_fn (void *arg) w->lwp = syscall (SYS_gettid); w->thread_id = pthread_self (); + vlib_thread_index = w - vlib_worker_threads; + rv = (void *) clib_calljmp ((uword (*)(uword)) w->thread_function, (uword) arg, w->thread_stack + VLIB_THREAD_STACK_SIZE); @@ -610,7 +582,9 @@ start_workers (vlib_main_t * vm) mheap_alloc (0 /* use VM */ , tr->mheap_size); else w->thread_mheap = main_heap; - w->thread_stack = vlib_thread_stacks[w - vlib_worker_threads]; + + w->thread_stack = + vlib_thread_stack_init (w - vlib_worker_threads); w->thread_function = tr->function; w->thread_function_arg = w; w->instance_id = k; @@ -630,7 +604,7 @@ start_workers (vlib_main_t * vm) vm_clone = clib_mem_alloc (sizeof (*vm_clone)); clib_memcpy (vm_clone, vlib_mains[0], sizeof (*vm_clone)); - vm_clone->cpu_index = worker_thread_index; + vm_clone->thread_index = worker_thread_index; vm_clone->heap_base = w->thread_mheap; vm_clone->mbuf_alloc_list = 0; vm_clone->init_functions_called = @@ -679,7 +653,7 @@ start_workers (vlib_main_t * vm) vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) { vlib_node_t *n = vlib_get_node (vm, rt->node_index); - rt->cpu_index = vm_clone->cpu_index; + rt->thread_index = vm_clone->thread_index; /* copy initial runtime_data from node */ if (n->runtime_data && n->runtime_data_bytes > 0) clib_memcpy (rt->runtime_data, n->runtime_data, @@ -692,7 +666,7 @@ start_workers (vlib_main_t * vm) vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) { vlib_node_t *n = vlib_get_node (vm, rt->node_index); - rt->cpu_index = vm_clone->cpu_index; + rt->thread_index = vm_clone->thread_index; /* copy initial runtime_data from node */ if (n->runtime_data && n->runtime_data_bytes > 0) clib_memcpy (rt->runtime_data, n->runtime_data, @@ -756,7 +730,8 @@ start_workers (vlib_main_t * vm) mheap_alloc (0 /* use VM */ , tr->mheap_size); else w->thread_mheap = main_heap; - w->thread_stack = vlib_thread_stacks[w - vlib_worker_threads]; + w->thread_stack = + vlib_thread_stack_init (w - vlib_worker_threads); w->thread_function = tr->function; w->thread_function_arg = w; w->instance_id = j; @@ -827,7 +802,7 @@ vlib_worker_thread_node_runtime_update (void) uword n_calls, uword n_vectors, uword n_clocks); - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); if (vec_len (vlib_mains) == 1) return; @@ -835,7 +810,7 @@ vlib_worker_thread_node_runtime_update (void) vm = vlib_mains[0]; nm = &vm->node_main; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); ASSERT (*vlib_worker_threads->wait_at_barrier == 1); /* @@ -955,7 +930,7 @@ vlib_worker_thread_node_runtime_update (void) vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]) { vlib_node_t *n = vlib_get_node (vm, rt->node_index); - rt->cpu_index = vm_clone->cpu_index; + rt->thread_index = vm_clone->thread_index; /* copy runtime_data, will be overwritten later for existing rt */ if (n->runtime_data && n->runtime_data_bytes > 0) clib_memcpy (rt->runtime_data, n->runtime_data, @@ -981,7 +956,7 @@ vlib_worker_thread_node_runtime_update (void) vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) { vlib_node_t *n = vlib_get_node (vm, rt->node_index); - rt->cpu_index = vm_clone->cpu_index; + rt->thread_index = vm_clone->thread_index; /* copy runtime_data, will be overwritten later for existing rt */ if (n->runtime_data && n->runtime_data_bytes > 0) clib_memcpy (rt->runtime_data, n->runtime_data, @@ -1180,7 +1155,7 @@ vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which) if (vlib_mains == 0) return; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); vlib_worker_thread_barrier_sync (vm); switch (which) @@ -1212,7 +1187,7 @@ vlib_worker_thread_barrier_sync (vlib_main_t * vm) vlib_worker_threads[0].barrier_sync_count++; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); deadline = vlib_time_now (vm) + BARRIER_SYNC_TIMEOUT; @@ -1260,7 +1235,7 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm) int vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm) { - u32 thread_id = vm->cpu_index; + u32 thread_id = vm->thread_index; vlib_frame_queue_t *fq = fqm->vlib_frame_queues[thread_id]; vlib_frame_queue_elt_t *elt; u32 *from, *to; @@ -1393,7 +1368,7 @@ vlib_worker_thread_fn (void *arg) vlib_main_t *vm = vlib_get_main (); clib_error_t *e; - ASSERT (vm->cpu_index == os_get_cpu_number ()); + ASSERT (vm->thread_index == vlib_get_thread_index ()); vlib_worker_thread_init (w); clib_time_init (&vm->clib_time); diff --git a/src/vlib/threads.h b/src/vlib/threads.h index eca4fc26..101d3d4a 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -153,8 +153,6 @@ typedef struct /* Called early, in thread 0's context */ clib_error_t *vlib_thread_init (vlib_main_t * vm); -vlib_worker_thread_t *vlib_alloc_thread (vlib_main_t * vm); - int vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t * frame, vlib_frame_queue_msg_type_t type); @@ -183,12 +181,19 @@ u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts); void vlib_worker_thread_barrier_sync (vlib_main_t * vm); void vlib_worker_thread_barrier_release (vlib_main_t * vm); +extern __thread uword vlib_thread_index; +static_always_inline uword +vlib_get_thread_index (void) +{ + return vlib_thread_index; +} + always_inline void vlib_smp_unsafe_warning (void) { if (CLIB_DEBUG > 0) { - if (os_get_cpu_number ()) + if (vlib_get_thread_index ()) fformat (stderr, "%s: SMP unsafe warning...\n", __FUNCTION__); } } @@ -331,21 +336,21 @@ vlib_num_workers () } always_inline u32 -vlib_get_worker_cpu_index (u32 worker_index) +vlib_get_worker_thread_index (u32 worker_index) { return worker_index + 1; } always_inline u32 -vlib_get_worker_index (u32 cpu_index) +vlib_get_worker_index (u32 thread_index) { - return cpu_index - 1; + return thread_index - 1; } always_inline u32 vlib_get_current_worker_index () { - return os_get_cpu_number () - 1; + return vlib_get_thread_index () - 1; } static inline void @@ -467,6 +472,8 @@ vlib_get_worker_handoff_queue_elt (u32 frame_queue_index, return elt; } +u8 *vlib_thread_stack_init (uword thread_index); + int vlib_thread_cb_register (struct vlib_main_t *vm, vlib_thread_callbacks_t * cb); diff --git a/src/vlib/unix/cj.c b/src/vlib/unix/cj.c index 33ba163a..7c1e9475 100644 --- a/src/vlib/unix/cj.c +++ b/src/vlib/unix/cj.c @@ -48,7 +48,7 @@ cj_log (u32 type, void *data0, void *data1) r = (cj_record_t *) & (cjm->records[new_tail & (cjm->num_records - 1)]); r->time = vlib_time_now (cjm->vlib_main); - r->cpu = os_get_cpu_number (); + r->thread_index = vlib_get_thread_index (); r->type = type; r->data[0] = pointer_to_uword (data0); r->data[1] = pointer_to_uword (data1); @@ -133,7 +133,8 @@ static inline void cj_dump_one_record (cj_record_t * r) { fprintf (stderr, "[%d]: %10.6f T%02d %llx %llx\n", - r->cpu, r->time, r->type, (long long unsigned int) r->data[0], + r->thread_index, r->time, r->type, + (long long unsigned int) r->data[0], (long long unsigned int) r->data[1]); } @@ -161,7 +162,7 @@ cj_dump_internal (u8 filter0_enable, u64 filter0, index = (cjm->tail + 1) & (cjm->num_records - 1); r = &(cjm->records[index]); - if (r->cpu != (u32) ~ 0) + if (r->thread_index != (u32) ~ 0) { /* Yes, dump from tail + 1 to the end */ for (i = index; i < cjm->num_records; i++) diff --git a/src/vlib/unix/cj.h b/src/vlib/unix/cj.h index 67626afe..d0a1d46e 100644 --- a/src/vlib/unix/cj.h +++ b/src/vlib/unix/cj.h @@ -23,7 +23,7 @@ typedef struct { f64 time; - u32 cpu; + u32 thread_index; u32 type; u64 data[2]; } cj_record_t; diff --git a/src/vlib/unix/main.c b/src/vlib/unix/main.c index 6b96cc0d..db5ddd64 100644 --- a/src/vlib/unix/main.c +++ b/src/vlib/unix/main.c @@ -510,13 +510,28 @@ thread0 (uword arg) return i; } +u8 * +vlib_thread_stack_init (uword thread_index) +{ + vec_validate (vlib_thread_stacks, thread_index); + vlib_thread_stacks[thread_index] = clib_mem_alloc_aligned + (VLIB_THREAD_STACK_SIZE, VLIB_THREAD_STACK_SIZE); + + /* + * Disallow writes to the bottom page of the stack, to + * catch stack overflows. + */ + if (mprotect (vlib_thread_stacks[thread_index], + clib_mem_get_page_size (), PROT_READ) < 0) + clib_unix_warning ("thread stack"); + return vlib_thread_stacks[thread_index]; +} + int vlib_unix_main (int argc, char *argv[]) { vlib_main_t *vm = &vlib_global_main; /* one and only time for this! */ - vlib_thread_main_t *tm = &vlib_thread_main; unformat_input_t input; - u8 *thread_stacks; clib_error_t *e; int i; @@ -548,29 +563,9 @@ vlib_unix_main (int argc, char *argv[]) } unformat_free (&input); - /* - * allocate n x VLIB_THREAD_STACK_SIZE stacks, aligned to a - * VLIB_THREAD_STACK_SIZE boundary - * See also: os_get_cpu_number() in vlib/vlib/threads.c - */ - thread_stacks = clib_mem_alloc_aligned - ((uword) tm->n_thread_stacks * VLIB_THREAD_STACK_SIZE, - VLIB_THREAD_STACK_SIZE); - - vec_validate (vlib_thread_stacks, tm->n_thread_stacks - 1); - for (i = 0; i < vec_len (vlib_thread_stacks); i++) - { - vlib_thread_stacks[i] = thread_stacks; - - /* - * Disallow writes to the bottom page of the stack, to - * catch stack overflows. - */ - if (mprotect (thread_stacks, clib_mem_get_page_size (), PROT_READ) < 0) - clib_unix_warning ("thread stack"); + vlib_thread_stack_init (0); - thread_stacks += VLIB_THREAD_STACK_SIZE; - } + vlib_thread_index = 0; i = clib_calljmp (thread0, (uword) vm, (void *) (vlib_thread_stacks[0] + diff --git a/src/vnet/adj/adj_l2.c b/src/vnet/adj/adj_l2.c index f68e54e0..20d70dd4 100644 --- a/src/vnet/adj/adj_l2.c +++ b/src/vnet/adj/adj_l2.c @@ -52,7 +52,7 @@ adj_l2_rewrite_inline (vlib_main_t * vm, { u32 * from = vlib_frame_vector_args (frame); u32 n_left_from, n_left_to_next, * to_next, next_index; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); ethernet_main_t * em = ðernet_main; n_left_from = frame->n_vectors; @@ -93,7 +93,7 @@ adj_l2_rewrite_inline (vlib_main_t * vm, vnet_buffer(p0)->sw_if_index[VLIB_TX] = adj0->rewrite_header.sw_if_index; vlib_increment_combined_counter(&adjacency_counters, - cpu_index, + thread_index, adj_index0, /* packet increment */ 0, /* byte increment */ rw_len0); diff --git a/src/vnet/adj/adj_midchain.c b/src/vnet/adj/adj_midchain.c index e8087f08..5756de43 100644 --- a/src/vnet/adj/adj_midchain.c +++ b/src/vnet/adj/adj_midchain.c @@ -49,7 +49,7 @@ adj_midchain_tx_inline (vlib_main_t * vm, u32 next_index; vnet_main_t *vnm = vnet_get_main (); vnet_interface_main_t *im = &vnm->interface_main; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; /* Vector of buffer / pkt indices we're supposed to process */ from = vlib_frame_vector_args (frame); @@ -124,13 +124,13 @@ adj_midchain_tx_inline (vlib_main_t * vm, { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, + thread_index, adj0->rewrite_header.sw_if_index, 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, + thread_index, adj1->rewrite_header.sw_if_index, 1, vlib_buffer_length_in_chain (vm, b1)); @@ -181,7 +181,7 @@ adj_midchain_tx_inline (vlib_main_t * vm, { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, + thread_index, adj0->rewrite_header.sw_if_index, 1, vlib_buffer_length_in_chain (vm, b0)); diff --git a/src/vnet/adj/adj_nsh.c b/src/vnet/adj/adj_nsh.c index 9a0f9d8b..128570b0 100644 --- a/src/vnet/adj/adj_nsh.c +++ b/src/vnet/adj/adj_nsh.c @@ -53,7 +53,7 @@ adj_nsh_rewrite_inline (vlib_main_t * vm, { u32 * from = vlib_frame_vector_args (frame); u32 n_left_from, n_left_to_next, * to_next, next_index; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); n_left_from = frame->n_vectors; next_index = node->cached_next_index; @@ -94,7 +94,7 @@ adj_nsh_rewrite_inline (vlib_main_t * vm, vnet_buffer(p0)->ip.save_rewrite_length = rw_len0; vlib_increment_combined_counter(&adjacency_counters, - cpu_index, + thread_index, adj_index0, /* packet increment */ 0, /* byte increment */ rw_len0); diff --git a/src/vnet/classify/vnet_classify.c b/src/vnet/classify/vnet_classify.c index 98842a48..70a189b0 100644 --- a/src/vnet/classify/vnet_classify.c +++ b/src/vnet/classify/vnet_classify.c @@ -251,12 +251,12 @@ static inline void make_working_copy vnet_classify_entry_##size##_t * working_copy##size = 0; foreach_size_in_u32x4; #undef _ - u32 cpu_number = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); - if (cpu_number >= vec_len (t->working_copies)) + if (thread_index >= vec_len (t->working_copies)) { oldheap = clib_mem_set_heap (t->mheap); - vec_validate (t->working_copies, cpu_number); + vec_validate (t->working_copies, thread_index); clib_mem_set_heap (oldheap); } @@ -265,7 +265,7 @@ static inline void make_working_copy * updates from multiple threads will not result in sporadic, spurious * lookup failures. */ - working_copy = t->working_copies[cpu_number]; + working_copy = t->working_copies[thread_index]; t->saved_bucket.as_u64 = b->as_u64; oldheap = clib_mem_set_heap (t->mheap); @@ -290,7 +290,7 @@ static inline void make_working_copy default: abort(); } - t->working_copies[cpu_number] = working_copy; + t->working_copies[thread_index] = working_copy; } _vec_len(working_copy) = (1<log2_pages)*t->entries_per_page; @@ -318,7 +318,7 @@ static inline void make_working_copy working_bucket.offset = vnet_classify_get_offset (t, working_copy); CLIB_MEMORY_BARRIER(); b->as_u64 = working_bucket.as_u64; - t->working_copies[cpu_number] = working_copy; + t->working_copies[thread_index] = working_copy; } static vnet_classify_entry_t * @@ -387,7 +387,7 @@ int vnet_classify_add_del (vnet_classify_table_t * t, int i; u64 hash, new_hash; u32 new_log2_pages; - u32 cpu_number = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); u8 * key_minus_skip; ASSERT ((add_v->flags & VNET_CLASSIFY_ENTRY_FREE) == 0); @@ -498,7 +498,7 @@ int vnet_classify_add_del (vnet_classify_table_t * t, new_log2_pages = t->saved_bucket.log2_pages + 1; expand_again: - working_copy = t->working_copies[cpu_number]; + working_copy = t->working_copies[thread_index]; new_v = split_and_rehash (t, working_copy, new_log2_pages); if (new_v == 0) diff --git a/src/vnet/cop/ip4_whitelist.c b/src/vnet/cop/ip4_whitelist.c index 6ef3d7d7..1b5e336b 100644 --- a/src/vnet/cop/ip4_whitelist.c +++ b/src/vnet/cop/ip4_whitelist.c @@ -60,7 +60,7 @@ ip4_cop_whitelist_node_fn (vlib_main_t * vm, cop_feature_type_t next_index; cop_main_t *cm = &cop_main; vlib_combined_counter_main_t * vcm = &load_balance_main.lbm_via_counters; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -177,12 +177,12 @@ ip4_cop_whitelist_node_fn (vlib_main_t * vm, dpo1 = load_balance_get_bucket_i(lb1, 0); vlib_increment_combined_counter - (vcm, cpu_index, lb_index0, 1, + (vcm, thread_index, lb_index0, 1, vlib_buffer_length_in_chain (vm, b0) + sizeof(ethernet_header_t)); vlib_increment_combined_counter - (vcm, cpu_index, lb_index1, 1, + (vcm, thread_index, lb_index1, 1, vlib_buffer_length_in_chain (vm, b1) + sizeof(ethernet_header_t)); @@ -273,7 +273,7 @@ ip4_cop_whitelist_node_fn (vlib_main_t * vm, dpo0 = load_balance_get_bucket_i(lb0, 0); vlib_increment_combined_counter - (vcm, cpu_index, lb_index0, 1, + (vcm, thread_index, lb_index0, 1, vlib_buffer_length_in_chain (vm, b0) + sizeof(ethernet_header_t)); diff --git a/src/vnet/cop/ip6_whitelist.c b/src/vnet/cop/ip6_whitelist.c index c2e16ccf..f3fe62e3 100644 --- a/src/vnet/cop/ip6_whitelist.c +++ b/src/vnet/cop/ip6_whitelist.c @@ -61,7 +61,7 @@ ip6_cop_whitelist_node_fn (vlib_main_t * vm, cop_main_t *cm = &cop_main; ip6_main_t * im6 = &ip6_main; vlib_combined_counter_main_t * vcm = &load_balance_main.lbm_via_counters; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -153,12 +153,12 @@ ip6_cop_whitelist_node_fn (vlib_main_t * vm, dpo1 = load_balance_get_bucket_i(lb1, 0); vlib_increment_combined_counter - (vcm, cpu_index, lb_index0, 1, + (vcm, thread_index, lb_index0, 1, vlib_buffer_length_in_chain (vm, b0) + sizeof(ethernet_header_t)); vlib_increment_combined_counter - (vcm, cpu_index, lb_index1, 1, + (vcm, thread_index, lb_index1, 1, vlib_buffer_length_in_chain (vm, b1) + sizeof(ethernet_header_t)); @@ -233,7 +233,7 @@ ip6_cop_whitelist_node_fn (vlib_main_t * vm, dpo0 = load_balance_get_bucket_i(lb0, 0); vlib_increment_combined_counter - (vcm, cpu_index, lb_index0, 1, + (vcm, thread_index, lb_index0, 1, vlib_buffer_length_in_chain (vm, b0) + sizeof(ethernet_header_t)); diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c index ba337f3f..76980102 100644 --- a/src/vnet/devices/af_packet/node.c +++ b/src/vnet/devices/af_packet/node.c @@ -124,7 +124,7 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, u32 frame_num = apif->rx_req->tp_frame_nr; u8 *block_start = apif->rx_ring + block * block_size; uword n_trace = vlib_get_trace_count (vm, node); - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); u32 min_bufs = apif->rx_req->tp_frame_size / n_buffer_bytes; @@ -132,15 +132,15 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, if (apif->per_interface_next_index != ~0) next_index = apif->per_interface_next_index; - n_free_bufs = vec_len (apm->rx_buffers[cpu_index]); + n_free_bufs = vec_len (apm->rx_buffers[thread_index]); if (PREDICT_FALSE (n_free_bufs < VLIB_FRAME_SIZE)) { - vec_validate (apm->rx_buffers[cpu_index], + vec_validate (apm->rx_buffers[thread_index], VLIB_FRAME_SIZE + n_free_bufs - 1); n_free_bufs += - vlib_buffer_alloc (vm, &apm->rx_buffers[cpu_index][n_free_bufs], + vlib_buffer_alloc (vm, &apm->rx_buffers[thread_index][n_free_bufs], VLIB_FRAME_SIZE); - _vec_len (apm->rx_buffers[cpu_index]) = n_free_bufs; + _vec_len (apm->rx_buffers[thread_index]) = n_free_bufs; } rx_frame = apif->next_rx_frame; @@ -163,11 +163,11 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, { /* grab free buffer */ u32 last_empty_buffer = - vec_len (apm->rx_buffers[cpu_index]) - 1; + vec_len (apm->rx_buffers[thread_index]) - 1; prev_bi0 = bi0; - bi0 = apm->rx_buffers[cpu_index][last_empty_buffer]; + bi0 = apm->rx_buffers[thread_index][last_empty_buffer]; b0 = vlib_get_buffer (vm, bi0); - _vec_len (apm->rx_buffers[cpu_index]) = last_empty_buffer; + _vec_len (apm->rx_buffers[thread_index]) = last_empty_buffer; n_free_bufs--; /* copy data */ @@ -236,9 +236,9 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_increment_combined_counter (vnet_get_main ()->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number (), apif->hw_if_index, n_rx_packets, n_rx_bytes); + vlib_get_thread_index (), apif->hw_if_index, n_rx_packets, n_rx_bytes); - vnet_device_increment_rx_packets (cpu_index, n_rx_packets); + vnet_device_increment_rx_packets (thread_index, n_rx_packets); return n_rx_packets; } diff --git a/src/vnet/devices/devices.c b/src/vnet/devices/devices.c index 41645220..5e5e812c 100644 --- a/src/vnet/devices/devices.c +++ b/src/vnet/devices/devices.c @@ -104,7 +104,7 @@ vnet_device_queue_sort (void *a1, void *a2) void vnet_device_input_assign_thread (u32 hw_if_index, - u16 queue_id, uword cpu_index) + u16 queue_id, uword thread_index) { vnet_main_t *vnm = vnet_get_main (); vnet_device_main_t *vdm = &vnet_device_main; @@ -115,19 +115,19 @@ vnet_device_input_assign_thread (u32 hw_if_index, ASSERT (hw->input_node_index > 0); - if (vdm->first_worker_cpu_index == 0) - cpu_index = 0; + if (vdm->first_worker_thread_index == 0) + thread_index = 0; - if (cpu_index != 0 && - (cpu_index < vdm->first_worker_cpu_index || - cpu_index > vdm->last_worker_cpu_index)) + if (thread_index != 0 && + (thread_index < vdm->first_worker_thread_index || + thread_index > vdm->last_worker_thread_index)) { - cpu_index = vdm->next_worker_cpu_index++; - if (vdm->next_worker_cpu_index > vdm->last_worker_cpu_index) - vdm->next_worker_cpu_index = vdm->first_worker_cpu_index; + thread_index = vdm->next_worker_thread_index++; + if (vdm->next_worker_thread_index > vdm->last_worker_thread_index) + vdm->next_worker_thread_index = vdm->first_worker_thread_index; } - vm = vlib_mains[cpu_index]; + vm = vlib_mains[thread_index]; rt = vlib_node_get_runtime_data (vm, hw->input_node_index); vec_add2 (rt->devices_and_queues, dq, 1); @@ -136,33 +136,33 @@ vnet_device_input_assign_thread (u32 hw_if_index, dq->queue_id = queue_id; vec_sort_with_function (rt->devices_and_queues, vnet_device_queue_sort); - vec_validate (hw->input_node_cpu_index_by_queue, queue_id); - hw->input_node_cpu_index_by_queue[queue_id] = cpu_index; + vec_validate (hw->input_node_thread_index_by_queue, queue_id); + hw->input_node_thread_index_by_queue[queue_id] = thread_index; } static int vnet_device_input_unassign_thread (u32 hw_if_index, u16 queue_id, - uword cpu_index) + uword thread_index) { vnet_main_t *vnm = vnet_get_main (); vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); vnet_device_input_runtime_t *rt; vnet_device_and_queue_t *dq; - uword old_cpu_index; + uword old_thread_index; - if (hw->input_node_cpu_index_by_queue == 0) + if (hw->input_node_thread_index_by_queue == 0) return VNET_API_ERROR_INVALID_INTERFACE; - if (vec_len (hw->input_node_cpu_index_by_queue) < queue_id + 1) + if (vec_len (hw->input_node_thread_index_by_queue) < queue_id + 1) return VNET_API_ERROR_INVALID_INTERFACE; - old_cpu_index = hw->input_node_cpu_index_by_queue[queue_id]; + old_thread_index = hw->input_node_thread_index_by_queue[queue_id]; - if (old_cpu_index == cpu_index) + if (old_thread_index == thread_index) return 0; rt = - vlib_node_get_runtime_data (vlib_mains[old_cpu_index], + vlib_node_get_runtime_data (vlib_mains[old_thread_index], hw->input_node_index); vec_foreach (dq, rt->devices_and_queues) @@ -240,7 +240,7 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input, vnet_device_main_t *vdm = &vnet_device_main; u32 hw_if_index = (u32) ~ 0; u32 queue_id = (u32) 0; - u32 cpu_index = (u32) ~ 0; + u32 thread_index = (u32) ~ 0; int rv; if (!unformat_user (input, unformat_line_input, line_input)) @@ -253,10 +253,10 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input, ; else if (unformat (line_input, "queue %d", &queue_id)) ; - else if (unformat (line_input, "main", &cpu_index)) - cpu_index = 0; - else if (unformat (line_input, "worker %d", &cpu_index)) - cpu_index += vdm->first_worker_cpu_index; + else if (unformat (line_input, "main", &thread_index)) + thread_index = 0; + else if (unformat (line_input, "worker %d", &thread_index)) + thread_index += vdm->first_worker_thread_index; else { error = clib_error_return (0, "parse error: '%U'", @@ -271,16 +271,17 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input, if (hw_if_index == (u32) ~ 0) return clib_error_return (0, "please specify valid interface name"); - if (cpu_index > vdm->last_worker_cpu_index) + if (thread_index > vdm->last_worker_thread_index) return clib_error_return (0, "please specify valid worker thread or main"); - rv = vnet_device_input_unassign_thread (hw_if_index, queue_id, cpu_index); + rv = + vnet_device_input_unassign_thread (hw_if_index, queue_id, thread_index); if (rv) return clib_error_return (0, "not found"); - vnet_device_input_assign_thread (hw_if_index, queue_id, cpu_index); + vnet_device_input_assign_thread (hw_if_index, queue_id, thread_index); return 0; } @@ -326,9 +327,9 @@ vnet_device_init (vlib_main_t * vm) tr = p ? (vlib_thread_registration_t *) p[0] : 0; if (tr && tr->count > 0) { - vdm->first_worker_cpu_index = tr->first_index; - vdm->next_worker_cpu_index = tr->first_index; - vdm->last_worker_cpu_index = tr->first_index + tr->count - 1; + vdm->first_worker_thread_index = tr->first_index; + vdm->next_worker_thread_index = tr->first_index; + vdm->last_worker_thread_index = tr->first_index + tr->count - 1; } return 0; } diff --git a/src/vnet/devices/devices.h b/src/vnet/devices/devices.h index bbb29fe3..966f8302 100644 --- a/src/vnet/devices/devices.h +++ b/src/vnet/devices/devices.h @@ -50,9 +50,9 @@ typedef struct typedef struct { vnet_device_per_worker_data_t *workers; - uword first_worker_cpu_index; - uword last_worker_cpu_index; - uword next_worker_cpu_index; + uword first_worker_thread_index; + uword last_worker_thread_index; + uword next_worker_thread_index; } vnet_device_main_t; typedef struct @@ -80,7 +80,7 @@ vnet_set_device_input_node (u32 hw_if_index, u32 node_index) } void vnet_device_input_assign_thread (u32 hw_if_index, u16 queue_id, - uword cpu_index); + uword thread_index); static inline u64 vnet_get_aggregate_rx_packets (void) @@ -95,12 +95,12 @@ vnet_get_aggregate_rx_packets (void) } static inline void -vnet_device_increment_rx_packets (u32 cpu_index, u64 count) +vnet_device_increment_rx_packets (u32 thread_index, u64 count) { vnet_device_main_t *vdm = &vnet_device_main; vnet_device_per_worker_data_t *pwd; - pwd = vec_elt_at_index (vdm->workers, cpu_index); + pwd = vec_elt_at_index (vdm->workers, thread_index); pwd->aggregate_rx_packets += count; } @@ -117,9 +117,9 @@ vnet_device_input_set_interrupt_pending (vnet_main_t * vnm, u32 hw_if_index, { vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); - ASSERT (queue_id < vec_len (hw->input_node_cpu_index_by_queue)); - u32 cpu_index = hw->input_node_cpu_index_by_queue[queue_id]; - vlib_node_set_interrupt_pending (vlib_mains[cpu_index], + ASSERT (queue_id < vec_len (hw->input_node_thread_index_by_queue)); + u32 thread_index = hw->input_node_thread_index_by_queue[queue_id]; + vlib_node_set_interrupt_pending (vlib_mains[thread_index], hw->input_node_index); } diff --git a/src/vnet/devices/netmap/node.c b/src/vnet/devices/netmap/node.c index 68ea7832..e120eeae 100644 --- a/src/vnet/devices/netmap/node.c +++ b/src/vnet/devices/netmap/node.c @@ -98,22 +98,22 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, u32 n_free_bufs; struct netmap_ring *ring; int cur_ring; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); if (nif->per_interface_next_index != ~0) next_index = nif->per_interface_next_index; - n_free_bufs = vec_len (nm->rx_buffers[cpu_index]); + n_free_bufs = vec_len (nm->rx_buffers[thread_index]); if (PREDICT_FALSE (n_free_bufs < VLIB_FRAME_SIZE)) { - vec_validate (nm->rx_buffers[cpu_index], + vec_validate (nm->rx_buffers[thread_index], VLIB_FRAME_SIZE + n_free_bufs - 1); n_free_bufs += - vlib_buffer_alloc (vm, &nm->rx_buffers[cpu_index][n_free_bufs], + vlib_buffer_alloc (vm, &nm->rx_buffers[thread_index][n_free_bufs], VLIB_FRAME_SIZE); - _vec_len (nm->rx_buffers[cpu_index]) = n_free_bufs; + _vec_len (nm->rx_buffers[thread_index]) = n_free_bufs; } cur_ring = nif->first_rx_ring; @@ -163,11 +163,11 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_buffer_t *b0; /* grab free buffer */ u32 last_empty_buffer = - vec_len (nm->rx_buffers[cpu_index]) - 1; + vec_len (nm->rx_buffers[thread_index]) - 1; prev_bi0 = bi0; - bi0 = nm->rx_buffers[cpu_index][last_empty_buffer]; + bi0 = nm->rx_buffers[thread_index][last_empty_buffer]; b0 = vlib_get_buffer (vm, bi0); - _vec_len (nm->rx_buffers[cpu_index]) = last_empty_buffer; + _vec_len (nm->rx_buffers[thread_index]) = last_empty_buffer; n_free_bufs--; /* copy data */ @@ -247,9 +247,9 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_increment_combined_counter (vnet_get_main ()->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number (), nif->hw_if_index, n_rx_packets, n_rx_bytes); + vlib_get_thread_index (), nif->hw_if_index, n_rx_packets, n_rx_bytes); - vnet_device_increment_rx_packets (cpu_index, n_rx_packets); + vnet_device_increment_rx_packets (thread_index, n_rx_packets); return n_rx_packets; } @@ -260,7 +260,7 @@ netmap_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, { int i; u32 n_rx_packets = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); netmap_main_t *nm = &netmap_main; netmap_if_t *nmi; @@ -269,7 +269,7 @@ netmap_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, nmi = vec_elt_at_index (nm->interfaces, i); if (nmi->is_admin_up && (i % nm->input_cpu_count) == - (cpu_index - nm->input_cpu_first_index)) + (thread_index - nm->input_cpu_first_index)) n_rx_packets += netmap_device_input_fn (vm, node, frame, nmi); } diff --git a/src/vnet/devices/ssvm/node.c b/src/vnet/devices/ssvm/node.c index a6c9dfd7..539b4161 100644 --- a/src/vnet/devices/ssvm/node.c +++ b/src/vnet/devices/ssvm/node.c @@ -89,7 +89,7 @@ ssvm_eth_device_input (ssvm_eth_main_t * em, ethernet_header_t *eh0; u16 type0; u32 n_rx_bytes = 0, l3_offset0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 trace_cnt __attribute__ ((unused)) = vlib_get_trace_count (vm, node); volatile u32 *lock; u32 *elt_indices; @@ -284,10 +284,10 @@ out: vlib_increment_combined_counter (vnet_get_main ()->interface_main.combined_sw_if_counters - + VNET_INTERFACE_COUNTER_RX, cpu_index, + + VNET_INTERFACE_COUNTER_RX, thread_index, intfc->vlib_hw_if_index, rx_queue_index, n_rx_bytes); - vnet_device_increment_rx_packets (cpu_index, rx_queue_index); + vnet_device_increment_rx_packets (thread_index, rx_queue_index); return rx_queue_index; } diff --git a/src/vnet/devices/virtio/vhost-user.c b/src/vnet/devices/virtio/vhost-user.c index 00807dc0..5e720f65 100644 --- a/src/vnet/devices/virtio/vhost-user.c +++ b/src/vnet/devices/virtio/vhost-user.c @@ -331,7 +331,7 @@ vhost_user_tx_thread_placement (vhost_user_intf_t * vui) { //Let's try to assign one queue to each thread u32 qid = 0; - u32 cpu_index = 0; + u32 thread_index = 0; vui->use_tx_spinlock = 0; while (1) { @@ -341,20 +341,21 @@ vhost_user_tx_thread_placement (vhost_user_intf_t * vui) if (!rxvq->started || !rxvq->enabled) continue; - vui->per_cpu_tx_qid[cpu_index] = qid; - cpu_index++; - if (cpu_index == vlib_get_thread_main ()->n_vlib_mains) + vui->per_cpu_tx_qid[thread_index] = qid; + thread_index++; + if (thread_index == vlib_get_thread_main ()->n_vlib_mains) return; } //We need to loop, meaning the spinlock has to be used vui->use_tx_spinlock = 1; - if (cpu_index == 0) + if (thread_index == 0) { //Could not find a single valid one - for (cpu_index = 0; - cpu_index < vlib_get_thread_main ()->n_vlib_mains; cpu_index++) + for (thread_index = 0; + thread_index < vlib_get_thread_main ()->n_vlib_mains; + thread_index++) { - vui->per_cpu_tx_qid[cpu_index] = 0; + vui->per_cpu_tx_qid[thread_index] = 0; } return; } @@ -368,7 +369,7 @@ vhost_user_rx_thread_placement () vhost_user_intf_t *vui; vhost_cpu_t *vhc; u32 *workers = 0; - u32 cpu_index; + u32 thread_index; vlib_main_t *vm; //Let's list all workers cpu indexes @@ -400,9 +401,9 @@ vhost_user_rx_thread_placement () continue; i %= vec_len (vui_workers); - cpu_index = vui_workers[i]; + thread_index = vui_workers[i]; i++; - vhc = &vum->cpus[cpu_index]; + vhc = &vum->cpus[thread_index]; iaq.qid = qid; iaq.vhost_iface_index = vui - vum->vhost_user_interfaces; @@ -429,14 +430,14 @@ vhost_user_rx_thread_placement () vhc->operation_mode = mode; } - for (cpu_index = vum->input_cpu_first_index; - cpu_index < vum->input_cpu_first_index + vum->input_cpu_count; - cpu_index++) + for (thread_index = vum->input_cpu_first_index; + thread_index < vum->input_cpu_first_index + vum->input_cpu_count; + thread_index++) { vlib_node_state_t state = VLIB_NODE_STATE_POLLING; - vhc = &vum->cpus[cpu_index]; - vm = vlib_mains ? vlib_mains[cpu_index] : &vlib_global_main; + vhc = &vum->cpus[thread_index]; + vm = vlib_mains ? vlib_mains[thread_index] : &vlib_global_main; switch (vhc->operation_mode) { case VHOST_USER_INTERRUPT_MODE: @@ -532,7 +533,7 @@ vhost_user_set_interrupt_pending (vhost_user_intf_t * vui, u32 ifq) { vhost_user_main_t *vum = &vhost_user_main; vhost_cpu_t *vhc; - u32 cpu_index; + u32 thread_index; vhost_iface_and_queue_t *vhiq; vlib_main_t *vm; u32 ifq2; @@ -553,8 +554,8 @@ vhost_user_set_interrupt_pending (vhost_user_intf_t * vui, u32 ifq) if ((vhiq->vhost_iface_index == (ifq >> 8)) && (VHOST_VRING_IDX_TX (vhiq->qid) == (ifq & 0xff))) { - cpu_index = vhc - vum->cpus; - vm = vlib_mains ? vlib_mains[cpu_index] : &vlib_global_main; + thread_index = vhc - vum->cpus; + vm = vlib_mains ? vlib_mains[thread_index] : &vlib_global_main; /* * Convert RX virtqueue number in the lower byte to vring * queue index for the input node process. Top bytes contain @@ -1592,7 +1593,7 @@ vhost_user_if_input (vlib_main_t * vm, u32 n_trace = vlib_get_trace_count (vm, node); u16 qsz_mask; u32 map_hint = 0; - u16 cpu_index = os_get_cpu_number (); + u16 thread_index = vlib_get_thread_index (); u16 copy_len = 0; { @@ -1651,32 +1652,32 @@ vhost_user_if_input (vlib_main_t * vm, * in the loop and come back later. This is not an issue as for big packet, * processing cost really comes from the memory copy. */ - if (PREDICT_FALSE (vum->cpus[cpu_index].rx_buffers_len < n_left + 1)) + if (PREDICT_FALSE (vum->cpus[thread_index].rx_buffers_len < n_left + 1)) { - u32 curr_len = vum->cpus[cpu_index].rx_buffers_len; - vum->cpus[cpu_index].rx_buffers_len += + u32 curr_len = vum->cpus[thread_index].rx_buffers_len; + vum->cpus[thread_index].rx_buffers_len += vlib_buffer_alloc_from_free_list (vm, - vum->cpus[cpu_index].rx_buffers + + vum->cpus[thread_index].rx_buffers + curr_len, VHOST_USER_RX_BUFFERS_N - curr_len, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); if (PREDICT_FALSE - (vum->cpus[cpu_index].rx_buffers_len < + (vum->cpus[thread_index].rx_buffers_len < VHOST_USER_RX_BUFFER_STARVATION)) { /* In case of buffer starvation, discard some packets from the queue * and log the event. * We keep doing best effort for the remaining packets. */ - u32 flush = (n_left + 1 > vum->cpus[cpu_index].rx_buffers_len) ? - n_left + 1 - vum->cpus[cpu_index].rx_buffers_len : 1; + u32 flush = (n_left + 1 > vum->cpus[thread_index].rx_buffers_len) ? + n_left + 1 - vum->cpus[thread_index].rx_buffers_len : 1; flush = vhost_user_rx_discard_packet (vm, vui, txvq, flush); n_left -= flush; vlib_increment_simple_counter (vnet_main. interface_main.sw_if_counters + VNET_INTERFACE_COUNTER_DROP, - os_get_cpu_number (), + vlib_get_thread_index (), vui->sw_if_index, flush); vlib_error_count (vm, vhost_user_input_node.index, @@ -1696,7 +1697,7 @@ vhost_user_if_input (vlib_main_t * vm, u32 desc_data_offset; vring_desc_t *desc_table = txvq->desc; - if (PREDICT_FALSE (vum->cpus[cpu_index].rx_buffers_len <= 1)) + if (PREDICT_FALSE (vum->cpus[thread_index].rx_buffers_len <= 1)) { /* Not enough rx_buffers * Note: We yeld on 1 so we don't need to do an additional @@ -1707,17 +1708,18 @@ vhost_user_if_input (vlib_main_t * vm, } desc_current = txvq->avail->ring[txvq->last_avail_idx & qsz_mask]; - vum->cpus[cpu_index].rx_buffers_len--; - bi_current = (vum->cpus[cpu_index].rx_buffers) - [vum->cpus[cpu_index].rx_buffers_len]; + vum->cpus[thread_index].rx_buffers_len--; + bi_current = (vum->cpus[thread_index].rx_buffers) + [vum->cpus[thread_index].rx_buffers_len]; b_head = b_current = vlib_get_buffer (vm, bi_current); to_next[0] = bi_current; //We do that now so we can forget about bi_current to_next++; n_left_to_next--; vlib_prefetch_buffer_with_index (vm, - (vum->cpus[cpu_index].rx_buffers) - [vum->cpus[cpu_index]. + (vum-> + cpus[thread_index].rx_buffers) + [vum->cpus[thread_index]. rx_buffers_len - 1], LOAD); /* Just preset the used descriptor id and length for later */ @@ -1791,7 +1793,7 @@ vhost_user_if_input (vlib_main_t * vm, (b_current->current_length == VLIB_BUFFER_DATA_SIZE)) { if (PREDICT_FALSE - (vum->cpus[cpu_index].rx_buffers_len == 0)) + (vum->cpus[thread_index].rx_buffers_len == 0)) { /* Cancel speculation */ to_next--; @@ -1805,17 +1807,18 @@ vhost_user_if_input (vlib_main_t * vm, * but valid. */ vhost_user_input_rewind_buffers (vm, - &vum->cpus[cpu_index], + &vum->cpus + [thread_index], b_head); n_left = 0; goto stop; } /* Get next output */ - vum->cpus[cpu_index].rx_buffers_len--; + vum->cpus[thread_index].rx_buffers_len--; u32 bi_next = - (vum->cpus[cpu_index].rx_buffers)[vum->cpus - [cpu_index].rx_buffers_len]; + (vum->cpus[thread_index].rx_buffers)[vum->cpus + [thread_index].rx_buffers_len]; b_current->next_buffer = bi_next; b_current->flags |= VLIB_BUFFER_NEXT_PRESENT; bi_current = bi_next; @@ -1823,7 +1826,7 @@ vhost_user_if_input (vlib_main_t * vm, } /* Prepare a copy order executed later for the data */ - vhost_copy_t *cpy = &vum->cpus[cpu_index].copy[copy_len]; + vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len]; copy_len++; u32 desc_data_l = desc_table[desc_current].len - desc_data_offset; @@ -1880,7 +1883,7 @@ vhost_user_if_input (vlib_main_t * vm, if (PREDICT_FALSE (copy_len >= VHOST_USER_RX_COPY_THRESHOLD)) { if (PREDICT_FALSE - (vhost_user_input_copy (vui, vum->cpus[cpu_index].copy, + (vhost_user_input_copy (vui, vum->cpus[thread_index].copy, copy_len, &map_hint))) { clib_warning @@ -1905,7 +1908,7 @@ vhost_user_if_input (vlib_main_t * vm, /* Do the memory copies */ if (PREDICT_FALSE - (vhost_user_input_copy (vui, vum->cpus[cpu_index].copy, + (vhost_user_input_copy (vui, vum->cpus[thread_index].copy, copy_len, &map_hint))) { clib_warning ("Memory mapping error on interface hw_if_index=%d " @@ -1933,9 +1936,9 @@ vhost_user_if_input (vlib_main_t * vm, vlib_increment_combined_counter (vnet_main.interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number (), vui->sw_if_index, n_rx_packets, n_rx_bytes); + vlib_get_thread_index (), vui->sw_if_index, n_rx_packets, n_rx_bytes); - vnet_device_increment_rx_packets (cpu_index, n_rx_packets); + vnet_device_increment_rx_packets (thread_index, n_rx_packets); return n_rx_packets; } @@ -1946,15 +1949,15 @@ vhost_user_input (vlib_main_t * vm, { vhost_user_main_t *vum = &vhost_user_main; uword n_rx_packets = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); vhost_iface_and_queue_t *vhiq; vhost_user_intf_t *vui; vhost_cpu_t *vhc; - vhc = &vum->cpus[cpu_index]; + vhc = &vum->cpus[thread_index]; if (PREDICT_TRUE (vhc->operation_mode == VHOST_USER_POLLING_MODE)) { - vec_foreach (vhiq, vum->cpus[cpu_index].rx_queues) + vec_foreach (vhiq, vum->cpus[thread_index].rx_queues) { vui = &vum->vhost_user_interfaces[vhiq->vhost_iface_index]; n_rx_packets += vhost_user_if_input (vm, vum, vui, vhiq->qid, node); @@ -2096,7 +2099,7 @@ vhost_user_tx (vlib_main_t * vm, vhost_user_vring_t *rxvq; u16 qsz_mask; u8 error; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 map_hint = 0; u8 retry = 8; u16 copy_len; @@ -2116,7 +2119,7 @@ vhost_user_tx (vlib_main_t * vm, qid = VHOST_VRING_IDX_RX (*vec_elt_at_index - (vui->per_cpu_tx_qid, os_get_cpu_number ())); + (vui->per_cpu_tx_qid, vlib_get_thread_index ())); rxvq = &vui->vrings[qid]; if (PREDICT_FALSE (vui->use_tx_spinlock)) vhost_user_vring_lock (vui, qid); @@ -2143,10 +2146,10 @@ retry: if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { - vum->cpus[cpu_index].current_trace = + vum->cpus[thread_index].current_trace = vlib_add_trace (vm, node, b0, - sizeof (*vum->cpus[cpu_index].current_trace)); - vhost_user_tx_trace (vum->cpus[cpu_index].current_trace, + sizeof (*vum->cpus[thread_index].current_trace)); + vhost_user_tx_trace (vum->cpus[thread_index].current_trace, vui, qid / 2, b0, rxvq); } @@ -2188,14 +2191,14 @@ retry: { // Get a header from the header array virtio_net_hdr_mrg_rxbuf_t *hdr = - &vum->cpus[cpu_index].tx_headers[tx_headers_len]; + &vum->cpus[thread_index].tx_headers[tx_headers_len]; tx_headers_len++; hdr->hdr.flags = 0; hdr->hdr.gso_type = 0; hdr->num_buffers = 1; //This is local, no need to check // Prepare a copy order executed later for the header - vhost_copy_t *cpy = &vum->cpus[cpu_index].copy[copy_len]; + vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len]; copy_len++; cpy->len = vui->virtio_net_hdr_sz; cpy->dst = buffer_map_addr; @@ -2220,7 +2223,7 @@ retry: else if (vui->virtio_net_hdr_sz == 12) //MRG is available { virtio_net_hdr_mrg_rxbuf_t *hdr = - &vum->cpus[cpu_index].tx_headers[tx_headers_len - 1]; + &vum->cpus[thread_index].tx_headers[tx_headers_len - 1]; //Move from available to used buffer rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id = @@ -2282,7 +2285,7 @@ retry: } { - vhost_copy_t *cpy = &vum->cpus[cpu_index].copy[copy_len]; + vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len]; copy_len++; cpy->len = bytes_left; cpy->len = (cpy->len > buffer_len) ? buffer_len : cpy->len; @@ -2325,8 +2328,8 @@ retry: if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { - vum->cpus[cpu_index].current_trace->hdr = - vum->cpus[cpu_index].tx_headers[tx_headers_len - 1]; + vum->cpus[thread_index].current_trace->hdr = + vum->cpus[thread_index].tx_headers[tx_headers_len - 1]; } n_left--; //At the end for error counting when 'goto done' is invoked @@ -2336,7 +2339,7 @@ retry: done: //Do the memory copies if (PREDICT_FALSE - (vhost_user_tx_copy (vui, vum->cpus[cpu_index].copy, + (vhost_user_tx_copy (vui, vum->cpus[thread_index].copy, copy_len, &map_hint))) { clib_warning ("Memory mapping error on interface hw_if_index=%d " @@ -2386,7 +2389,7 @@ done3: vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters + VNET_INTERFACE_COUNTER_DROP, - os_get_cpu_number (), vui->sw_if_index, n_left); + vlib_get_thread_index (), vui->sw_if_index, n_left); } vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors); @@ -2773,11 +2776,11 @@ vhost_user_send_interrupt_process (vlib_main_t * vm, case ~0: vec_foreach (vhc, vum->cpus) { - u32 cpu_index = vhc - vum->cpus; + u32 thread_index = vhc - vum->cpus; f64 next_timeout; next_timeout = timeout; - vec_foreach (vhiq, vum->cpus[cpu_index].rx_queues) + vec_foreach (vhiq, vum->cpus[thread_index].rx_queues) { vui = &vum->vhost_user_interfaces[vhiq->vhost_iface_index]; vhost_user_vring_t *rxvq = diff --git a/src/vnet/dpo/lookup_dpo.c b/src/vnet/dpo/lookup_dpo.c index e94e871c..97ad0a44 100644 --- a/src/vnet/dpo/lookup_dpo.c +++ b/src/vnet/dpo/lookup_dpo.c @@ -266,7 +266,7 @@ lookup_dpo_ip4_inline (vlib_main_t * vm, int table_from_interface) { u32 n_left_from, next_index, * from, * to_next; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters; from = vlib_frame_vector_args (from_frame); @@ -407,10 +407,10 @@ lookup_dpo_ip4_inline (vlib_main_t * vm, vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, b1)); if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) @@ -511,7 +511,7 @@ lookup_dpo_ip4_inline (vlib_main_t * vm, vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) @@ -606,7 +606,7 @@ lookup_dpo_ip6_inline (vlib_main_t * vm, { vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters; u32 n_left_from, next_index, * from, * to_next; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -749,10 +749,10 @@ lookup_dpo_ip6_inline (vlib_main_t * vm, vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, b1)); if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) @@ -853,7 +853,7 @@ lookup_dpo_ip6_inline (vlib_main_t * vm, vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) @@ -930,7 +930,7 @@ lookup_dpo_mpls_inline (vlib_main_t * vm, int table_from_interface) { u32 n_left_from, next_index, * from, * to_next; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters; from = vlib_frame_vector_args (from_frame); @@ -994,7 +994,7 @@ lookup_dpo_mpls_inline (vlib_main_t * vm, vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) diff --git a/src/vnet/dpo/replicate_dpo.c b/src/vnet/dpo/replicate_dpo.c index a9f334be..e25ceae9 100644 --- a/src/vnet/dpo/replicate_dpo.c +++ b/src/vnet/dpo/replicate_dpo.c @@ -627,7 +627,7 @@ replicate_inline (vlib_main_t * vm, vlib_combined_counter_main_t * cm = &replicate_main.repm_counters; replicate_main_t * rm = &replicate_main; u32 n_left_from, * from, * to_next, next_index; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -657,12 +657,12 @@ replicate_inline (vlib_main_t * vm, rep0 = replicate_get(repi0); vlib_increment_combined_counter( - cm, cpu_index, repi0, 1, + cm, thread_index, repi0, 1, vlib_buffer_length_in_chain(vm, b0)); - vec_validate (rm->clones[cpu_index], rep0->rep_n_buckets - 1); + vec_validate (rm->clones[thread_index], rep0->rep_n_buckets - 1); - num_cloned = vlib_buffer_clone (vm, bi0, rm->clones[cpu_index], rep0->rep_n_buckets, 128); + num_cloned = vlib_buffer_clone (vm, bi0, rm->clones[thread_index], rep0->rep_n_buckets, 128); if (num_cloned != rep0->rep_n_buckets) { @@ -673,7 +673,7 @@ replicate_inline (vlib_main_t * vm, for (bucket = 0; bucket < num_cloned; bucket++) { - ci0 = rm->clones[cpu_index][bucket]; + ci0 = rm->clones[thread_index][bucket]; c0 = vlib_get_buffer(vm, ci0); to_next[0] = ci0; @@ -700,7 +700,7 @@ replicate_inline (vlib_main_t * vm, vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); } } - vec_reset_length (rm->clones[cpu_index]); + vec_reset_length (rm->clones[thread_index]); } vlib_put_next_frame (vm, node, next_index, n_left_to_next); diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c index ee757505..c74a097e 100644 --- a/src/vnet/ethernet/arp.c +++ b/src/vnet/ethernet/arp.c @@ -1771,7 +1771,7 @@ set_ip4_over_ethernet_rpc_callback (vnet_arp_set_ip4_over_ethernet_rpc_args_t * a) { vnet_main_t *vm = vnet_get_main (); - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); if (a->flags & ETHERNET_ARP_ARGS_REMOVE) vnet_arp_unset_ip4_over_ethernet_internal (vm, a); diff --git a/src/vnet/ethernet/interface.c b/src/vnet/ethernet/interface.c index 9894e3c8..335e3f9f 100644 --- a/src/vnet/ethernet/interface.c +++ b/src/vnet/ethernet/interface.c @@ -362,7 +362,7 @@ simulated_ethernet_interface_tx (vlib_main_t * vm, u32 next_index = VNET_SIMULATED_ETHERNET_TX_NEXT_ETHERNET_INPUT; u32 i, next_node_index, bvi_flag, sw_if_index; u32 n_pkts = 0, n_bytes = 0; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; vnet_main_t *vnm = vnet_get_main (); vnet_interface_main_t *im = &vnm->interface_main; vlib_node_main_t *nm = &vm->node_main; @@ -420,8 +420,9 @@ simulated_ethernet_interface_tx (vlib_main_t * vm, /* increment TX interface stat */ vlib_increment_combined_counter (im->combined_sw_if_counters + - VNET_INTERFACE_COUNTER_TX, cpu_index, - sw_if_index, n_pkts, n_bytes); + VNET_INTERFACE_COUNTER_TX, + thread_index, sw_if_index, n_pkts, + n_bytes); } return n_left_from; diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index b699e381..f7787ed2 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -291,7 +291,7 @@ ethernet_input_inline (vlib_main_t * vm, vlib_node_runtime_t *error_node; u32 n_left_from, next_index, *from, *to_next; u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 cached_sw_if_index = ~0; u32 cached_is_l2 = 0; /* shut up gcc */ vnet_hw_interface_t *hi = NULL; /* used for main interface only */ @@ -510,7 +510,7 @@ ethernet_input_inline (vlib_main_t * vm, interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, new_sw_if_index0, 1, len0); if (new_sw_if_index1 != old_sw_if_index1 @@ -519,7 +519,7 @@ ethernet_input_inline (vlib_main_t * vm, interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, new_sw_if_index1, 1, len1); @@ -530,7 +530,7 @@ ethernet_input_inline (vlib_main_t * vm, vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = stats_n_bytes = 0; @@ -696,13 +696,13 @@ ethernet_input_inline (vlib_main_t * vm, vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, new_sw_if_index0, 1, len0); + thread_index, new_sw_if_index0, 1, len0); if (stats_n_packets > 0) { vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = stats_n_bytes = 0; } @@ -734,7 +734,7 @@ ethernet_input_inline (vlib_main_t * vm, vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } diff --git a/src/vnet/gre/node.c b/src/vnet/gre/node.c index 2683586e..acf15f24 100644 --- a/src/vnet/gre/node.c +++ b/src/vnet/gre/node.c @@ -75,7 +75,7 @@ gre_input (vlib_main_t * vm, u64 cached_tunnel_key6[4]; u32 cached_tunnel_sw_if_index = 0, tunnel_sw_if_index = 0; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); u32 len; vnet_interface_main_t *im = &gm->vnet_main->interface_main; @@ -257,7 +257,7 @@ gre_input (vlib_main_t * vm, len = vlib_buffer_length_in_chain (vm, b0); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, tunnel_sw_if_index, 1 /* packets */, len /* bytes */); @@ -324,7 +324,7 @@ drop0: len = vlib_buffer_length_in_chain (vm, b1); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, tunnel_sw_if_index, 1 /* packets */, len /* bytes */); @@ -502,7 +502,7 @@ drop1: len = vlib_buffer_length_in_chain (vm, b0); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, tunnel_sw_if_index, 1 /* packets */, len /* bytes */); diff --git a/src/vnet/interface.h b/src/vnet/interface.h index a1ea2d61..08f08b10 100644 --- a/src/vnet/interface.h +++ b/src/vnet/interface.h @@ -468,7 +468,7 @@ typedef struct vnet_hw_interface_t u32 input_node_index; /* input node cpu index by queue */ - u32 *input_node_cpu_index_by_queue; + u32 *input_node_thread_index_by_queue; } vnet_hw_interface_t; diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c index 03f2cdca..663dc309 100644 --- a/src/vnet/interface_output.c +++ b/src/vnet/interface_output.c @@ -196,7 +196,7 @@ slow_path (vlib_main_t * vm, */ static_always_inline void incr_output_stats (vnet_main_t * vnm, - u32 cpu_index, + u32 thread_index, u32 length, u32 sw_if_index, u32 * last_sw_if_index, u32 * n_packets, u32 * n_bytes) @@ -216,7 +216,7 @@ incr_output_stats (vnet_main_t * vnm, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, + thread_index, *last_sw_if_index, *n_packets, *n_bytes); } @@ -240,7 +240,7 @@ vnet_interface_output_node_flatten (vlib_main_t * vm, u32 n_left_to_tx, *from, *from_end, *to_tx; u32 n_bytes, n_buffers, n_packets; u32 last_sw_if_index; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; n_buffers = frame->n_vectors; @@ -266,7 +266,7 @@ vnet_interface_output_node_flatten (vlib_main_t * vm, cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, VNET_INTERFACE_COUNTER_TX_ERROR); - vlib_increment_simple_counter (cm, cpu_index, + vlib_increment_simple_counter (cm, thread_index, rt->sw_if_index, n_buffers); return vlib_error_drop_buffers (vm, node, from, /* buffer stride */ 1, @@ -341,18 +341,18 @@ vnet_interface_output_node_flatten (vlib_main_t * vm, from += 1; to_tx += n_buffers; n_left_to_tx -= n_buffers; - incr_output_stats (vnm, cpu_index, n_slow_bytes, + incr_output_stats (vnm, thread_index, n_slow_bytes, vnet_buffer (b)->sw_if_index[VLIB_TX], &last_sw_if_index, &n_packets, &n_bytes); } } else { - incr_output_stats (vnm, cpu_index, + incr_output_stats (vnm, thread_index, vlib_buffer_length_in_chain (vm, b0), vnet_buffer (b0)->sw_if_index[VLIB_TX], &last_sw_if_index, &n_packets, &n_bytes); - incr_output_stats (vnm, cpu_index, + incr_output_stats (vnm, thread_index, vlib_buffer_length_in_chain (vm, b0), vnet_buffer (b1)->sw_if_index[VLIB_TX], &last_sw_if_index, &n_packets, &n_bytes); @@ -396,7 +396,7 @@ vnet_interface_output_node_flatten (vlib_main_t * vm, to_tx += n_buffers; n_left_to_tx -= n_buffers; } - incr_output_stats (vnm, cpu_index, + incr_output_stats (vnm, thread_index, vlib_buffer_length_in_chain (vm, b0), vnet_buffer (b0)->sw_if_index[VLIB_TX], &last_sw_if_index, &n_packets, &n_bytes); @@ -408,7 +408,7 @@ vnet_interface_output_node_flatten (vlib_main_t * vm, } /* Final update of interface stats. */ - incr_output_stats (vnm, cpu_index, 0, ~0, /* ~0 will flush stats */ + incr_output_stats (vnm, thread_index, 0, ~0, /* ~0 will flush stats */ &last_sw_if_index, &n_packets, &n_bytes); return n_buffers; @@ -428,7 +428,7 @@ vnet_interface_output_node (vlib_main_t * vm, u32 n_left_to_tx, *from, *from_end, *to_tx; u32 n_bytes, n_buffers, n_packets; u32 n_bytes_b0, n_bytes_b1, n_bytes_b2, n_bytes_b3; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; vnet_interface_main_t *im = &vnm->interface_main; u32 next_index = VNET_INTERFACE_OUTPUT_NEXT_TX; u32 current_config_index = ~0; @@ -458,7 +458,7 @@ vnet_interface_output_node (vlib_main_t * vm, cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, VNET_INTERFACE_COUNTER_TX_ERROR); - vlib_increment_simple_counter (cm, cpu_index, + vlib_increment_simple_counter (cm, thread_index, rt->sw_if_index, n_buffers); return vlib_error_drop_buffers (vm, node, from, @@ -558,7 +558,7 @@ vnet_interface_output_node (vlib_main_t * vm, { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, tx_swif0, 1, + thread_index, tx_swif0, 1, n_bytes_b0); } @@ -567,7 +567,7 @@ vnet_interface_output_node (vlib_main_t * vm, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, tx_swif1, 1, + thread_index, tx_swif1, 1, n_bytes_b1); } @@ -576,7 +576,7 @@ vnet_interface_output_node (vlib_main_t * vm, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, tx_swif2, 1, + thread_index, tx_swif2, 1, n_bytes_b2); } if (PREDICT_FALSE (tx_swif3 != rt->sw_if_index)) @@ -584,7 +584,7 @@ vnet_interface_output_node (vlib_main_t * vm, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, tx_swif3, 1, + thread_index, tx_swif3, 1, n_bytes_b3); } } @@ -623,7 +623,7 @@ vnet_interface_output_node (vlib_main_t * vm, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, tx_swif0, 1, + thread_index, tx_swif0, 1, n_bytes_b0); } } @@ -634,7 +634,7 @@ vnet_interface_output_node (vlib_main_t * vm, /* Update main interface stats. */ vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, + thread_index, rt->sw_if_index, n_packets, n_bytes); return n_buffers; } @@ -893,7 +893,7 @@ process_drop_punt (vlib_main_t * vm, u32 current_sw_if_index, n_errors_current_sw_if_index; u64 current_counter; vlib_simple_counter_main_t *cm; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; static vlib_error_t memory[VNET_ERROR_N_DISPOSITION]; static char memory_init[VNET_ERROR_N_DISPOSITION]; @@ -965,19 +965,19 @@ process_drop_punt (vlib_main_t * vm, current_counter -= 2; n_errors_current_sw_if_index -= 2; - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index1, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index1, 1); /* Increment super-interface drop/punt counters for sub-interfaces. */ sw_if0 = vnet_get_sw_interface (vnm, sw_if_index0); vlib_increment_simple_counter - (cm, cpu_index, sw_if0->sup_sw_if_index, + (cm, thread_index, sw_if0->sup_sw_if_index, sw_if0->sup_sw_if_index != sw_if_index0); sw_if1 = vnet_get_sw_interface (vnm, sw_if_index1); vlib_increment_simple_counter - (cm, cpu_index, sw_if1->sup_sw_if_index, + (cm, thread_index, sw_if1->sup_sw_if_index, sw_if1->sup_sw_if_index != sw_if_index1); em->counters[current_counter_index] = current_counter; @@ -1013,11 +1013,12 @@ process_drop_punt (vlib_main_t * vm, sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; /* Increment drop/punt counters. */ - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); /* Increment super-interface drop/punt counters for sub-interfaces. */ sw_if0 = vnet_get_sw_interface (vnm, sw_if_index0); - vlib_increment_simple_counter (cm, cpu_index, sw_if0->sup_sw_if_index, + vlib_increment_simple_counter (cm, thread_index, + sw_if0->sup_sw_if_index, sw_if0->sup_sw_if_index != sw_if_index0); if (PREDICT_FALSE (e0 != current_error)) @@ -1041,12 +1042,12 @@ process_drop_punt (vlib_main_t * vm, { vnet_sw_interface_t *si; - vlib_increment_simple_counter (cm, cpu_index, current_sw_if_index, + vlib_increment_simple_counter (cm, thread_index, current_sw_if_index, n_errors_current_sw_if_index); si = vnet_get_sw_interface (vnm, current_sw_if_index); if (si->sup_sw_if_index != current_sw_if_index) - vlib_increment_simple_counter (cm, cpu_index, si->sup_sw_if_index, + vlib_increment_simple_counter (cm, thread_index, si->sup_sw_if_index, n_errors_current_sw_if_index); } diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c index ee1703e7..fdfe7f63 100644 --- a/src/vnet/ip/ip4_forward.c +++ b/src/vnet/ip/ip4_forward.c @@ -75,7 +75,7 @@ ip4_lookup_inline (vlib_main_t * vm, vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters; u32 n_left_from, n_left_to_next, *from, *to_next; ip_lookup_next_t next; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -292,19 +292,19 @@ ip4_lookup_inline (vlib_main_t * vm, vnet_buffer (p3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lb_index0, 1, + (cm, thread_index, lb_index0, 1, vlib_buffer_length_in_chain (vm, p0) + sizeof (ethernet_header_t)); vlib_increment_combined_counter - (cm, cpu_index, lb_index1, 1, + (cm, thread_index, lb_index1, 1, vlib_buffer_length_in_chain (vm, p1) + sizeof (ethernet_header_t)); vlib_increment_combined_counter - (cm, cpu_index, lb_index2, 1, + (cm, thread_index, lb_index2, 1, vlib_buffer_length_in_chain (vm, p2) + sizeof (ethernet_header_t)); vlib_increment_combined_counter - (cm, cpu_index, lb_index3, 1, + (cm, thread_index, lb_index3, 1, vlib_buffer_length_in_chain (vm, p3) + sizeof (ethernet_header_t)); @@ -392,7 +392,7 @@ ip4_lookup_inline (vlib_main_t * vm, vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); from += 1; to_next += 1; @@ -479,7 +479,7 @@ ip4_load_balance (vlib_main_t * vm, vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters; u32 n_left_from, n_left_to_next, *from, *to_next; ip_lookup_next_t next; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -584,9 +584,9 @@ ip4_load_balance (vlib_main_t * vm, vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); vlib_validate_buffer_enqueue_x2 (vm, node, next, to_next, n_left_to_next, @@ -639,7 +639,7 @@ ip4_load_balance (vlib_main_t * vm, vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_validate_buffer_enqueue_x1 (vm, node, next, to_next, n_left_to_next, @@ -2330,7 +2330,7 @@ ip4_rewrite_inline (vlib_main_t * vm, n_left_from = frame->n_vectors; next_index = node->cached_next_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -2379,9 +2379,9 @@ ip4_rewrite_inline (vlib_main_t * vm, if (do_counters) { vlib_prefetch_combined_counter (&adjacency_counters, - cpu_index, adj_index0); + thread_index, adj_index0); vlib_prefetch_combined_counter (&adjacency_counters, - cpu_index, adj_index1); + thread_index, adj_index1); } ip0 = vlib_buffer_get_current (p0); @@ -2527,13 +2527,13 @@ ip4_rewrite_inline (vlib_main_t * vm, { vlib_increment_combined_counter (&adjacency_counters, - cpu_index, + thread_index, adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); vlib_increment_combined_counter (&adjacency_counters, - cpu_index, + thread_index, adj_index1, 1, vlib_buffer_length_in_chain (vm, p1) + rw_len1); } @@ -2618,7 +2618,7 @@ ip4_rewrite_inline (vlib_main_t * vm, if (do_counters) vlib_prefetch_combined_counter (&adjacency_counters, - cpu_index, adj_index0); + thread_index, adj_index0); /* Guess we are only writing on simple Ethernet header. */ vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t)); @@ -2637,7 +2637,7 @@ ip4_rewrite_inline (vlib_main_t * vm, if (do_counters) vlib_increment_combined_counter (&adjacency_counters, - cpu_index, adj_index0, 1, + thread_index, adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); /* Check MTU of outgoing interface. */ diff --git a/src/vnet/ip/ip4_input.c b/src/vnet/ip/ip4_input.c index ba200a9f..3b08f4b0 100644 --- a/src/vnet/ip/ip4_input.c +++ b/src/vnet/ip/ip4_input.c @@ -85,7 +85,7 @@ ip4_input_inline (vlib_main_t * vm, vlib_node_runtime_t *error_node = vlib_node_get_runtime (vm, ip4_input_node.index); vlib_simple_counter_main_t *cm; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -178,8 +178,8 @@ ip4_input_inline (vlib_main_t * vm, vnet_feature_arc_start (arc0, sw_if_index0, &next0, p0); vnet_feature_arc_start (arc1, sw_if_index1, &next1, p1); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index1, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index1, 1); /* Punt packets with options or wrong version. */ if (PREDICT_FALSE (ip0->ip_version_and_header_length != 0x45)) @@ -299,7 +299,7 @@ ip4_input_inline (vlib_main_t * vm, vnet_buffer (p0)->ip.adj_index[VLIB_RX] = ~0; vnet_feature_arc_start (arc0, sw_if_index0, &next0, p0); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); /* Punt packets with options or wrong version. */ if (PREDICT_FALSE (ip0->ip_version_and_header_length != 0x45)) diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index c120f12c..c2fc4f87 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -74,7 +74,7 @@ ip6_lookup_inline (vlib_main_t * vm, vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters; u32 n_left_from, n_left_to_next, *from, *to_next; ip_lookup_next_t next; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -185,9 +185,9 @@ ip6_lookup_inline (vlib_main_t * vm, vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); from += 2; to_next += 2; @@ -291,7 +291,7 @@ ip6_lookup_inline (vlib_main_t * vm, vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); from += 1; to_next += 1; @@ -703,7 +703,7 @@ ip6_load_balance (vlib_main_t * vm, vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters; u32 n_left_from, n_left_to_next, *from, *to_next; ip_lookup_next_t next; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); ip6_main_t *im = &ip6_main; from = vlib_frame_vector_args (frame); @@ -824,9 +824,9 @@ ip6_load_balance (vlib_main_t * vm, vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); vlib_validate_buffer_enqueue_x2 (vm, node, next, to_next, n_left_to_next, @@ -886,7 +886,7 @@ ip6_load_balance (vlib_main_t * vm, } vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_validate_buffer_enqueue_x1 (vm, node, next, to_next, n_left_to_next, @@ -1897,7 +1897,7 @@ ip6_rewrite_inline (vlib_main_t * vm, n_left_from = frame->n_vectors; next_index = node->cached_next_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -2019,11 +2019,11 @@ ip6_rewrite_inline (vlib_main_t * vm, { vlib_increment_combined_counter (&adjacency_counters, - cpu_index, adj_index0, 1, + thread_index, adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); vlib_increment_combined_counter (&adjacency_counters, - cpu_index, adj_index1, 1, + thread_index, adj_index1, 1, vlib_buffer_length_in_chain (vm, p1) + rw_len1); } @@ -2156,7 +2156,7 @@ ip6_rewrite_inline (vlib_main_t * vm, { vlib_increment_combined_counter (&adjacency_counters, - cpu_index, adj_index0, 1, + thread_index, adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); } diff --git a/src/vnet/ip/ip6_input.c b/src/vnet/ip/ip6_input.c index 20306088..ffdc4727 100644 --- a/src/vnet/ip/ip6_input.c +++ b/src/vnet/ip/ip6_input.c @@ -82,7 +82,7 @@ ip6_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) vlib_node_runtime_t *error_node = vlib_node_get_runtime (vm, ip6_input_node.index); vlib_simple_counter_main_t *cm; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -171,8 +171,8 @@ ip6_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) vnet_feature_arc_start (arc0, sw_if_index0, &next0, p0); vnet_feature_arc_start (arc1, sw_if_index1, &next1, p1); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index1, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index1, 1); error0 = error1 = IP6_ERROR_NONE; @@ -270,7 +270,7 @@ ip6_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) vnet_buffer (p0)->ip.adj_index[VLIB_RX] = ~0; vnet_feature_arc_start (arc0, sw_if_index0, &next0, p0); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); error0 = IP6_ERROR_NONE; /* Version != 6? Drop it. */ diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c index 5d1fb6f8..2af546df 100644 --- a/src/vnet/ip/ip6_neighbor.c +++ b/src/vnet/ip/ip6_neighbor.c @@ -581,7 +581,7 @@ vnet_set_ip6_ethernet_neighbor (vlib_main_t * vm, u32 next_index; pending_resolution_t *pr, *mc; - if (os_get_cpu_number ()) + if (vlib_get_thread_index ()) { set_unset_ip6_neighbor_rpc (vm, sw_if_index, a, link_layer_address, 1 /* set new neighbor */ , is_static, @@ -722,7 +722,7 @@ vnet_unset_ip6_ethernet_neighbor (vlib_main_t * vm, uword *p; int rv = 0; - if (os_get_cpu_number ()) + if (vlib_get_thread_index ()) { set_unset_ip6_neighbor_rpc (vm, sw_if_index, a, link_layer_address, 0 /* unset */ , 0, 0); diff --git a/src/vnet/ipsec/esp.h b/src/vnet/ipsec/esp.h index 50cac806..799003b9 100644 --- a/src/vnet/ipsec/esp.h +++ b/src/vnet/ipsec/esp.h @@ -282,8 +282,8 @@ hmac_calc (ipsec_integ_alg_t alg, u8 * data, int data_len, u8 * signature, u8 use_esn, u32 seq_hi) { esp_main_t *em = &esp_main; - u32 cpu_index = os_get_cpu_number (); - HMAC_CTX *ctx = &(em->per_thread_data[cpu_index].hmac_ctx); + u32 thread_index = vlib_get_thread_index (); + HMAC_CTX *ctx = &(em->per_thread_data[thread_index].hmac_ctx); const EVP_MD *md = NULL; unsigned int len; @@ -292,10 +292,10 @@ hmac_calc (ipsec_integ_alg_t alg, if (PREDICT_FALSE (em->esp_integ_algs[alg].md == 0)) return 0; - if (PREDICT_FALSE (alg != em->per_thread_data[cpu_index].last_integ_alg)) + if (PREDICT_FALSE (alg != em->per_thread_data[thread_index].last_integ_alg)) { md = em->esp_integ_algs[alg].md; - em->per_thread_data[cpu_index].last_integ_alg = alg; + em->per_thread_data[thread_index].last_integ_alg = alg; } HMAC_Init (ctx, key, key_len, md); diff --git a/src/vnet/ipsec/esp_decrypt.c b/src/vnet/ipsec/esp_decrypt.c index 7289b260..925d2b45 100644 --- a/src/vnet/ipsec/esp_decrypt.c +++ b/src/vnet/ipsec/esp_decrypt.c @@ -85,8 +85,8 @@ esp_decrypt_aes_cbc (ipsec_crypto_alg_t alg, u8 * in, u8 * out, size_t in_len, u8 * key, u8 * iv) { esp_main_t *em = &esp_main; - u32 cpu_index = os_get_cpu_number (); - EVP_CIPHER_CTX *ctx = &(em->per_thread_data[cpu_index].decrypt_ctx); + u32 thread_index = vlib_get_thread_index (); + EVP_CIPHER_CTX *ctx = &(em->per_thread_data[thread_index].decrypt_ctx); const EVP_CIPHER *cipher = NULL; int out_len; @@ -95,10 +95,11 @@ esp_decrypt_aes_cbc (ipsec_crypto_alg_t alg, if (PREDICT_FALSE (em->esp_crypto_algs[alg].type == 0)) return; - if (PREDICT_FALSE (alg != em->per_thread_data[cpu_index].last_decrypt_alg)) + if (PREDICT_FALSE + (alg != em->per_thread_data[thread_index].last_decrypt_alg)) { cipher = em->esp_crypto_algs[alg].type; - em->per_thread_data[cpu_index].last_decrypt_alg = alg; + em->per_thread_data[thread_index].last_decrypt_alg = alg; } EVP_DecryptInit_ex (ctx, cipher, NULL, key, iv); @@ -117,11 +118,11 @@ esp_decrypt_node_fn (vlib_main_t * vm, u32 *recycle = 0; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); ipsec_alloc_empty_buffers (vm, im); - u32 *empty_buffers = im->empty_buffers[cpu_index]; + u32 *empty_buffers = im->empty_buffers[thread_index]; if (PREDICT_FALSE (vec_len (empty_buffers) < n_left_from)) { diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c index 44ae2297..b2bc4e0b 100644 --- a/src/vnet/ipsec/esp_encrypt.c +++ b/src/vnet/ipsec/esp_encrypt.c @@ -88,8 +88,8 @@ esp_encrypt_aes_cbc (ipsec_crypto_alg_t alg, u8 * in, u8 * out, size_t in_len, u8 * key, u8 * iv) { esp_main_t *em = &esp_main; - u32 cpu_index = os_get_cpu_number (); - EVP_CIPHER_CTX *ctx = &(em->per_thread_data[cpu_index].encrypt_ctx); + u32 thread_index = vlib_get_thread_index (); + EVP_CIPHER_CTX *ctx = &(em->per_thread_data[thread_index].encrypt_ctx); const EVP_CIPHER *cipher = NULL; int out_len; @@ -98,10 +98,11 @@ esp_encrypt_aes_cbc (ipsec_crypto_alg_t alg, if (PREDICT_FALSE (em->esp_crypto_algs[alg].type == IPSEC_CRYPTO_ALG_NONE)) return; - if (PREDICT_FALSE (alg != em->per_thread_data[cpu_index].last_encrypt_alg)) + if (PREDICT_FALSE + (alg != em->per_thread_data[thread_index].last_encrypt_alg)) { cipher = em->esp_crypto_algs[alg].type; - em->per_thread_data[cpu_index].last_encrypt_alg = alg; + em->per_thread_data[thread_index].last_encrypt_alg = alg; } EVP_EncryptInit_ex (ctx, cipher, NULL, key, iv); @@ -119,11 +120,11 @@ esp_encrypt_node_fn (vlib_main_t * vm, n_left_from = from_frame->n_vectors; ipsec_main_t *im = &ipsec_main; u32 *recycle = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); ipsec_alloc_empty_buffers (vm, im); - u32 *empty_buffers = im->empty_buffers[cpu_index]; + u32 *empty_buffers = im->empty_buffers[thread_index]; if (PREDICT_FALSE (vec_len (empty_buffers) < n_left_from)) { diff --git a/src/vnet/ipsec/ikev2.c b/src/vnet/ipsec/ikev2.c index 2c1074d8..3f9978a7 100644 --- a/src/vnet/ipsec/ikev2.c +++ b/src/vnet/ipsec/ikev2.c @@ -303,16 +303,16 @@ static void ikev2_delete_sa (ikev2_sa_t * sa) { ikev2_main_t *km = &ikev2_main; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); uword *p; ikev2_sa_free_all_vec (sa); - p = hash_get (km->per_thread_data[cpu_index].sa_by_rspi, sa->rspi); + p = hash_get (km->per_thread_data[thread_index].sa_by_rspi, sa->rspi); if (p) { - hash_unset (km->per_thread_data[cpu_index].sa_by_rspi, sa->rspi); - pool_put (km->per_thread_data[cpu_index].sas, sa); + hash_unset (km->per_thread_data[thread_index].sa_by_rspi, sa->rspi); + pool_put (km->per_thread_data[thread_index].sas, sa); } } @@ -776,29 +776,31 @@ ikev2_initial_contact_cleanup (ikev2_sa_t * sa) ikev2_sa_t *tmp; u32 i, *delete = 0; ikev2_child_sa_t *c; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); if (!sa->initial_contact) return; /* find old IKE SAs with the same authenticated identity */ /* *INDENT-OFF* */ - pool_foreach (tmp, km->per_thread_data[cpu_index].sas, ({ + pool_foreach (tmp, km->per_thread_data[thread_index].sas, ({ if (tmp->i_id.type != sa->i_id.type || vec_len(tmp->i_id.data) != vec_len(sa->i_id.data) || memcmp(sa->i_id.data, tmp->i_id.data, vec_len(sa->i_id.data))) continue; if (sa->rspi != tmp->rspi) - vec_add1(delete, tmp - km->per_thread_data[cpu_index].sas); + vec_add1(delete, tmp - km->per_thread_data[thread_index].sas); })); /* *INDENT-ON* */ for (i = 0; i < vec_len (delete); i++) { - tmp = pool_elt_at_index (km->per_thread_data[cpu_index].sas, delete[i]); - vec_foreach (c, tmp->childs) - ikev2_delete_tunnel_interface (km->vnet_main, tmp, c); + tmp = + pool_elt_at_index (km->per_thread_data[thread_index].sas, delete[i]); + vec_foreach (c, + tmp->childs) ikev2_delete_tunnel_interface (km->vnet_main, + tmp, c); ikev2_delete_sa (tmp); } @@ -1922,10 +1924,10 @@ ikev2_retransmit_sa_init (ike_header_t * ike, { ikev2_main_t *km = &ikev2_main; ikev2_sa_t *sa; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); /* *INDENT-OFF* */ - pool_foreach (sa, km->per_thread_data[cpu_index].sas, ({ + pool_foreach (sa, km->per_thread_data[thread_index].sas, ({ if (sa->ispi == clib_net_to_host_u64(ike->ispi) && sa->iaddr.as_u32 == iaddr.as_u32 && sa->raddr.as_u32 == raddr.as_u32) @@ -2036,7 +2038,7 @@ ikev2_node_fn (vlib_main_t * vm, u32 n_left_from, *from, *to_next; ikev2_next_t next_index; ikev2_main_t *km = &ikev2_main; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -2134,11 +2136,14 @@ ikev2_node_fn (vlib_main_t * vm, if (sa0->state == IKEV2_STATE_SA_INIT) { /* add SA to the pool */ - pool_get (km->per_thread_data[cpu_index].sas, sa0); + pool_get (km->per_thread_data[thread_index].sas, + sa0); clib_memcpy (sa0, &sa, sizeof (*sa0)); - hash_set (km->per_thread_data[cpu_index].sa_by_rspi, + hash_set (km-> + per_thread_data[thread_index].sa_by_rspi, sa0->rspi, - sa0 - km->per_thread_data[cpu_index].sas); + sa0 - + km->per_thread_data[thread_index].sas); } else { @@ -2169,11 +2174,11 @@ ikev2_node_fn (vlib_main_t * vm, if (sa0->state == IKEV2_STATE_SA_INIT) { /* add SA to the pool */ - pool_get (km->per_thread_data[cpu_index].sas, sa0); + pool_get (km->per_thread_data[thread_index].sas, sa0); clib_memcpy (sa0, &sa, sizeof (*sa0)); - hash_set (km->per_thread_data[cpu_index].sa_by_rspi, + hash_set (km->per_thread_data[thread_index].sa_by_rspi, sa0->rspi, - sa0 - km->per_thread_data[cpu_index].sas); + sa0 - km->per_thread_data[thread_index].sas); } else { @@ -2184,12 +2189,13 @@ ikev2_node_fn (vlib_main_t * vm, else if (ike0->exchange == IKEV2_EXCHANGE_IKE_AUTH) { uword *p; - p = hash_get (km->per_thread_data[cpu_index].sa_by_rspi, + p = hash_get (km->per_thread_data[thread_index].sa_by_rspi, clib_net_to_host_u64 (ike0->rspi)); if (p) { - sa0 = pool_elt_at_index (km->per_thread_data[cpu_index].sas, - p[0]); + sa0 = + pool_elt_at_index (km->per_thread_data[thread_index].sas, + p[0]); r = ikev2_retransmit_resp (sa0, ike0); if (r == 1) @@ -2240,12 +2246,13 @@ ikev2_node_fn (vlib_main_t * vm, else if (ike0->exchange == IKEV2_EXCHANGE_INFORMATIONAL) { uword *p; - p = hash_get (km->per_thread_data[cpu_index].sa_by_rspi, + p = hash_get (km->per_thread_data[thread_index].sa_by_rspi, clib_net_to_host_u64 (ike0->rspi)); if (p) { - sa0 = pool_elt_at_index (km->per_thread_data[cpu_index].sas, - p[0]); + sa0 = + pool_elt_at_index (km->per_thread_data[thread_index].sas, + p[0]); r = ikev2_retransmit_resp (sa0, ike0); if (r == 1) @@ -2305,12 +2312,13 @@ ikev2_node_fn (vlib_main_t * vm, else if (ike0->exchange == IKEV2_EXCHANGE_CREATE_CHILD_SA) { uword *p; - p = hash_get (km->per_thread_data[cpu_index].sa_by_rspi, + p = hash_get (km->per_thread_data[thread_index].sa_by_rspi, clib_net_to_host_u64 (ike0->rspi)); if (p) { - sa0 = pool_elt_at_index (km->per_thread_data[cpu_index].sas, - p[0]); + sa0 = + pool_elt_at_index (km->per_thread_data[thread_index].sas, + p[0]); r = ikev2_retransmit_resp (sa0, ike0); if (r == 1) diff --git a/src/vnet/ipsec/ipsec.h b/src/vnet/ipsec/ipsec.h index 58f0f145..c884e360 100644 --- a/src/vnet/ipsec/ipsec.h +++ b/src/vnet/ipsec/ipsec.h @@ -324,21 +324,21 @@ int ipsec_set_interface_key (vnet_main_t * vnm, u32 hw_if_index, always_inline void ipsec_alloc_empty_buffers (vlib_main_t * vm, ipsec_main_t * im) { - u32 cpu_index = os_get_cpu_number (); - uword l = vec_len (im->empty_buffers[cpu_index]); + u32 thread_index = vlib_get_thread_index (); + uword l = vec_len (im->empty_buffers[thread_index]); uword n_alloc = 0; if (PREDICT_FALSE (l < VLIB_FRAME_SIZE)) { - if (!im->empty_buffers[cpu_index]) + if (!im->empty_buffers[thread_index]) { - vec_alloc (im->empty_buffers[cpu_index], 2 * VLIB_FRAME_SIZE); + vec_alloc (im->empty_buffers[thread_index], 2 * VLIB_FRAME_SIZE); } - n_alloc = vlib_buffer_alloc (vm, im->empty_buffers[cpu_index] + l, + n_alloc = vlib_buffer_alloc (vm, im->empty_buffers[thread_index] + l, 2 * VLIB_FRAME_SIZE - l); - _vec_len (im->empty_buffers[cpu_index]) = l + n_alloc; + _vec_len (im->empty_buffers[thread_index]) = l + n_alloc; } } diff --git a/src/vnet/ipsec/ipsec_if.c b/src/vnet/ipsec/ipsec_if.c index dc882004..ed124894 100644 --- a/src/vnet/ipsec/ipsec_if.c +++ b/src/vnet/ipsec/ipsec_if.c @@ -99,7 +99,7 @@ static int ipsec_add_del_tunnel_if_rpc_callback (ipsec_add_del_tunnel_args_t * a) { vnet_main_t *vnm = vnet_get_main (); - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); return ipsec_add_del_tunnel_if_internal (vnm, a); } diff --git a/src/vnet/l2/l2_bvi.h b/src/vnet/l2/l2_bvi.h index dd1130a6..e21a1616 100644 --- a/src/vnet/l2/l2_bvi.h +++ b/src/vnet/l2/l2_bvi.h @@ -97,7 +97,7 @@ l2_to_bvi (vlib_main_t * vlib_main, vlib_increment_combined_counter (vnet_main->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - vlib_main->cpu_index, + vlib_main->thread_index, vnet_buffer (b0)->sw_if_index[VLIB_RX], 1, vlib_buffer_length_in_chain (vlib_main, b0)); return TO_BVI_ERR_OK; diff --git a/src/vnet/l2/l2_input.c b/src/vnet/l2/l2_input.c index 041ff38d..e5d6878a 100644 --- a/src/vnet/l2/l2_input.c +++ b/src/vnet/l2/l2_input.c @@ -117,7 +117,7 @@ typedef enum static_always_inline void classify_and_dispatch (vlib_main_t * vm, vlib_node_runtime_t * node, - u32 cpu_index, + u32 thread_index, l2input_main_t * msm, vlib_buffer_t * b0, u32 * next0) { /* @@ -237,7 +237,7 @@ l2input_node_inline (vlib_main_t * vm, u32 n_left_from, *from, *to_next; l2input_next_t next_index; l2input_main_t *msm = &l2input_main; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; /* number of packets to process */ @@ -350,10 +350,10 @@ l2input_node_inline (vlib_main_t * vm, vlib_node_increment_counter (vm, l2input_node.index, L2INPUT_ERROR_L2INPUT, 4); - classify_and_dispatch (vm, node, cpu_index, msm, b0, &next0); - classify_and_dispatch (vm, node, cpu_index, msm, b1, &next1); - classify_and_dispatch (vm, node, cpu_index, msm, b2, &next2); - classify_and_dispatch (vm, node, cpu_index, msm, b3, &next3); + classify_and_dispatch (vm, node, thread_index, msm, b0, &next0); + classify_and_dispatch (vm, node, thread_index, msm, b1, &next1); + classify_and_dispatch (vm, node, thread_index, msm, b2, &next2); + classify_and_dispatch (vm, node, thread_index, msm, b3, &next3); /* verify speculative enqueues, maybe switch current next frame */ /* if next0==next1==next_index then nothing special needs to be done */ @@ -393,7 +393,7 @@ l2input_node_inline (vlib_main_t * vm, vlib_node_increment_counter (vm, l2input_node.index, L2INPUT_ERROR_L2INPUT, 1); - classify_and_dispatch (vm, node, cpu_index, msm, b0, &next0); + classify_and_dispatch (vm, node, thread_index, msm, b0, &next0); /* verify speculative enqueue, maybe switch current next frame */ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, diff --git a/src/vnet/l2/l2_output.c b/src/vnet/l2/l2_output.c index 00f22571..e17b2a16 100644 --- a/src/vnet/l2/l2_output.c +++ b/src/vnet/l2/l2_output.c @@ -643,11 +643,11 @@ l2output_create_output_node_mapping (vlib_main_t * vlib_main, vnet_main_t * vnet hw0 = vnet_get_sup_hw_interface (vnet_main, sw_if_index); - uword cpu_number; + uword thread_index; - cpu_number = os_get_cpu_number (); + thread_index = vlib_get_thread_index (); - if (cpu_number) + if (thread_index) { u32 oldflags; diff --git a/src/vnet/l2tp/decap.c b/src/vnet/l2tp/decap.c index e8986935..46104129 100644 --- a/src/vnet/l2tp/decap.c +++ b/src/vnet/l2tp/decap.c @@ -149,7 +149,7 @@ last_stage (vlib_main_t * vm, vlib_node_runtime_t * node, u32 bi) /* per-mapping byte stats include the ethernet header */ vlib_increment_combined_counter (&lm->counter_main, - os_get_cpu_number (), + vlib_get_thread_index (), counter_index, 1 /* packet_increment */ , vlib_buffer_length_in_chain (vm, b) + sizeof (ethernet_header_t)); diff --git a/src/vnet/l2tp/encap.c b/src/vnet/l2tp/encap.c index ed7a9580..dcdfde4b 100644 --- a/src/vnet/l2tp/encap.c +++ b/src/vnet/l2tp/encap.c @@ -124,7 +124,7 @@ last_stage (vlib_main_t * vm, vlib_node_runtime_t * node, u32 bi) /* per-mapping byte stats include the ethernet header */ vlib_increment_combined_counter (&lm->counter_main, - os_get_cpu_number (), + vlib_get_thread_index (), counter_index, 1 /* packet_increment */ , vlib_buffer_length_in_chain (vm, b)); diff --git a/src/vnet/l2tp/l2tp.c b/src/vnet/l2tp/l2tp.c index cb94d7e7..3dedc447 100644 --- a/src/vnet/l2tp/l2tp.c +++ b/src/vnet/l2tp/l2tp.c @@ -157,7 +157,7 @@ test_counters_command_fn (vlib_main_t * vm, u32 session_index; u32 counter_index; u32 nincr = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); /* *INDENT-OFF* */ pool_foreach (session, lm->sessions, @@ -167,11 +167,11 @@ test_counters_command_fn (vlib_main_t * vm, session_index_to_counter_index (session_index, SESSION_COUNTER_USER_TO_NETWORK); vlib_increment_combined_counter (&lm->counter_main, - cpu_index, + thread_index, counter_index, 1/*pkt*/, 1111 /*bytes*/); vlib_increment_combined_counter (&lm->counter_main, - cpu_index, + thread_index, counter_index+1, 1/*pkt*/, 2222 /*bytes*/); nincr++; diff --git a/src/vnet/lisp-gpe/decap.c b/src/vnet/lisp-gpe/decap.c index d887a95f..68769710 100644 --- a/src/vnet/lisp-gpe/decap.c +++ b/src/vnet/lisp-gpe/decap.c @@ -103,7 +103,7 @@ next_index_to_iface (lisp_gpe_main_t * lgm, u32 next_index) } static_always_inline void -incr_decap_stats (vnet_main_t * vnm, u32 cpu_index, u32 length, +incr_decap_stats (vnet_main_t * vnm, u32 thread_index, u32 length, u32 sw_if_index, u32 * last_sw_if_index, u32 * n_packets, u32 * n_bytes) { @@ -122,7 +122,7 @@ incr_decap_stats (vnet_main_t * vnm, u32 cpu_index, u32 length, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, *last_sw_if_index, + thread_index, *last_sw_if_index, *n_packets, *n_bytes); } *last_sw_if_index = sw_if_index; @@ -150,11 +150,11 @@ static uword lisp_gpe_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame, u8 is_v4) { - u32 n_left_from, next_index, *from, *to_next, cpu_index; + u32 n_left_from, next_index, *from, *to_next, thread_index; u32 n_bytes = 0, n_packets = 0, last_sw_if_index = ~0, drops = 0; lisp_gpe_main_t *lgm = vnet_lisp_gpe_get_main (); - cpu_index = os_get_cpu_number (); + thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -267,7 +267,7 @@ lisp_gpe_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (si0) { - incr_decap_stats (lgm->vnet_main, cpu_index, + incr_decap_stats (lgm->vnet_main, thread_index, vlib_buffer_length_in_chain (vm, b0), si0[0], &last_sw_if_index, &n_packets, &n_bytes); vnet_buffer (b0)->sw_if_index[VLIB_RX] = si0[0]; @@ -282,7 +282,7 @@ lisp_gpe_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (si1) { - incr_decap_stats (lgm->vnet_main, cpu_index, + incr_decap_stats (lgm->vnet_main, thread_index, vlib_buffer_length_in_chain (vm, b1), si1[0], &last_sw_if_index, &n_packets, &n_bytes); vnet_buffer (b1)->sw_if_index[VLIB_RX] = si1[0]; @@ -397,7 +397,7 @@ lisp_gpe_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (si0) { - incr_decap_stats (lgm->vnet_main, cpu_index, + incr_decap_stats (lgm->vnet_main, thread_index, vlib_buffer_length_in_chain (vm, b0), si0[0], &last_sw_if_index, &n_packets, &n_bytes); vnet_buffer (b0)->sw_if_index[VLIB_RX] = si0[0]; @@ -430,7 +430,7 @@ lisp_gpe_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, } /* flush iface stats */ - incr_decap_stats (lgm->vnet_main, cpu_index, 0, ~0, &last_sw_if_index, + incr_decap_stats (lgm->vnet_main, thread_index, 0, ~0, &last_sw_if_index, &n_packets, &n_bytes); vlib_node_increment_counter (vm, lisp_gpe_ip4_input_node.index, LISP_GPE_ERROR_NO_TUNNEL, drops); diff --git a/src/vnet/lldp/lldp_input.c b/src/vnet/lldp/lldp_input.c index 762743d0..e88f6fdb 100644 --- a/src/vnet/lldp/lldp_input.c +++ b/src/vnet/lldp/lldp_input.c @@ -35,7 +35,7 @@ typedef struct static void lldp_rpc_update_peer_cb (const lldp_intf_update_t * a) { - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); lldp_intf_t *n = lldp_get_intf (&lldp_main, a->hw_if_index); if (!n) diff --git a/src/vnet/map/ip4_map.c b/src/vnet/map/ip4_map.c index 1a20d704..e39b6f14 100644 --- a/src/vnet/map/ip4_map.c +++ b/src/vnet/map/ip4_map.c @@ -248,7 +248,7 @@ ip4_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) next_index = node->cached_next_index; map_main_t *mm = &map_main; vlib_combined_counter_main_t *cm = mm->domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -377,7 +377,7 @@ ip4_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) ip40) ? IP4_MAP_NEXT_IP6_REWRITE : next0; vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, map_domain_index0, 1, clib_net_to_host_u16 (ip6h0->payload_length) + @@ -409,7 +409,7 @@ ip4_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) ip41) ? IP4_MAP_NEXT_IP6_REWRITE : next1; vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, map_domain_index1, 1, clib_net_to_host_u16 (ip6h1->payload_length) + @@ -520,7 +520,7 @@ ip4_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) ip40) ? IP4_MAP_NEXT_IP6_REWRITE : next0; vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, map_domain_index0, 1, clib_net_to_host_u16 (ip6h0->payload_length) + @@ -564,7 +564,7 @@ ip4_map_reass (vlib_main_t * vm, next_index = node->cached_next_index; map_main_t *mm = &map_main; vlib_combined_counter_main_t *cm = mm->domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 *fragments_to_drop = NULL; u32 *fragments_to_loopback = NULL; @@ -694,8 +694,8 @@ ip4_map_reass (vlib_main_t * vm, { if (error0 == MAP_ERROR_NONE) vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, map_domain_index0, - 1, + thread_index, + map_domain_index0, 1, clib_net_to_host_u16 (ip60->payload_length) + 40); next0 = diff --git a/src/vnet/map/ip4_map_t.c b/src/vnet/map/ip4_map_t.c index b63d76bf..5f2bcbf9 100644 --- a/src/vnet/map/ip4_map_t.c +++ b/src/vnet/map/ip4_map_t.c @@ -477,7 +477,7 @@ ip4_map_t_icmp (vlib_main_t * vm, n_left_from = frame->n_vectors; next_index = node->cached_next_index; vlib_combined_counter_main_t *cm = map_main.domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -520,7 +520,7 @@ ip4_map_t_icmp (vlib_main_t * vm, if (PREDICT_TRUE (error0 == MAP_ERROR_NONE)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, vnet_buffer (p0)->map_t. map_domain_index, 1, len0); } @@ -1051,7 +1051,7 @@ ip4_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) n_left_from = frame->n_vectors; next_index = node->cached_next_index; vlib_combined_counter_main_t *cm = map_main.domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -1158,7 +1158,7 @@ ip4_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) (error0 == MAP_ERROR_NONE && next0 != IP4_MAPT_NEXT_MAPT_ICMP)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, vnet_buffer (p0)->map_t. map_domain_index, 1, clib_net_to_host_u16 (ip40-> @@ -1169,7 +1169,7 @@ ip4_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) (error1 == MAP_ERROR_NONE && next1 != IP4_MAPT_NEXT_MAPT_ICMP)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, vnet_buffer (p1)->map_t. map_domain_index, 1, clib_net_to_host_u16 (ip41-> @@ -1252,7 +1252,7 @@ ip4_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) (error0 == MAP_ERROR_NONE && next0 != IP4_MAPT_NEXT_MAPT_ICMP)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, vnet_buffer (p0)->map_t. map_domain_index, 1, clib_net_to_host_u16 (ip40-> diff --git a/src/vnet/map/ip6_map.c b/src/vnet/map/ip6_map.c index f7eb768f..63ada962 100644 --- a/src/vnet/map/ip6_map.c +++ b/src/vnet/map/ip6_map.c @@ -172,7 +172,7 @@ ip6_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) vlib_node_get_runtime (vm, ip6_map_node.index); map_main_t *mm = &map_main; vlib_combined_counter_main_t *cm = mm->domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -319,7 +319,7 @@ ip6_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) IP6_MAP_NEXT_IP4_REWRITE : next0; } vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, map_domain_index0, 1, clib_net_to_host_u16 (ip40->length)); @@ -352,7 +352,7 @@ ip6_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) IP6_MAP_NEXT_IP4_REWRITE : next1; } vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, map_domain_index1, 1, clib_net_to_host_u16 (ip41->length)); @@ -505,7 +505,7 @@ ip6_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) IP6_MAP_NEXT_IP4_REWRITE : next0; } vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, map_domain_index0, 1, clib_net_to_host_u16 (ip40->length)); @@ -820,7 +820,7 @@ ip6_map_ip4_reass (vlib_main_t * vm, vlib_node_get_runtime (vm, ip6_map_ip4_reass_node.index); map_main_t *mm = &map_main; vlib_combined_counter_main_t *cm = mm->domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 *fragments_to_drop = NULL; u32 *fragments_to_loopback = NULL; @@ -958,8 +958,8 @@ ip6_map_ip4_reass (vlib_main_t * vm, { if (error0 == MAP_ERROR_NONE) vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, map_domain_index0, - 1, + thread_index, + map_domain_index0, 1, clib_net_to_host_u16 (ip40->length)); next0 = @@ -1015,7 +1015,7 @@ ip6_map_icmp_relay (vlib_main_t * vm, vlib_node_runtime_t *error_node = vlib_node_get_runtime (vm, ip6_map_icmp_relay_node.index); map_main_t *mm = &map_main; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u16 *fragment_ids, *fid; from = vlib_frame_vector_args (frame); @@ -1143,7 +1143,8 @@ ip6_map_icmp_relay (vlib_main_t * vm, ip_csum_t sum = ip_incremental_checksum (0, new_icmp40, nlen - 20); new_icmp40->checksum = ~ip_csum_fold (sum); - vlib_increment_simple_counter (&mm->icmp_relayed, cpu_index, 0, 1); + vlib_increment_simple_counter (&mm->icmp_relayed, thread_index, 0, + 1); error: if (PREDICT_FALSE (p0->flags & VLIB_BUFFER_IS_TRACED)) diff --git a/src/vnet/map/ip6_map_t.c b/src/vnet/map/ip6_map_t.c index eb3996c2..99151678 100644 --- a/src/vnet/map/ip6_map_t.c +++ b/src/vnet/map/ip6_map_t.c @@ -448,7 +448,7 @@ ip6_map_t_icmp (vlib_main_t * vm, n_left_from = frame->n_vectors; next_index = node->cached_next_index; vlib_combined_counter_main_t *cm = map_main.domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -493,7 +493,7 @@ ip6_map_t_icmp (vlib_main_t * vm, if (PREDICT_TRUE (error0 == MAP_ERROR_NONE)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, vnet_buffer (p0)-> map_t.map_domain_index, 1, len0); @@ -1051,7 +1051,7 @@ ip6_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) vlib_node_runtime_t *error_node = vlib_node_get_runtime (vm, ip6_map_t_node.index); vlib_combined_counter_main_t *cm = map_main.domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -1218,7 +1218,7 @@ ip6_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) (error0 == MAP_ERROR_NONE && next0 != IP6_MAPT_NEXT_MAPT_ICMP)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, vnet_buffer (p0)-> map_t.map_domain_index, 1, clib_net_to_host_u16 @@ -1229,7 +1229,7 @@ ip6_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) (error1 == MAP_ERROR_NONE && next1 != IP6_MAPT_NEXT_MAPT_ICMP)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, vnet_buffer (p1)-> map_t.map_domain_index, 1, clib_net_to_host_u16 @@ -1403,7 +1403,7 @@ ip6_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) (error0 == MAP_ERROR_NONE && next0 != IP6_MAPT_NEXT_MAPT_ICMP)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, vnet_buffer (p0)-> map_t.map_domain_index, 1, clib_net_to_host_u16 diff --git a/src/vnet/mpls/mpls_input.c b/src/vnet/mpls/mpls_input.c index 893c4511..1b9bdd05 100644 --- a/src/vnet/mpls/mpls_input.c +++ b/src/vnet/mpls/mpls_input.c @@ -76,7 +76,7 @@ mpls_input_inline (vlib_main_t * vm, u32 n_left_from, next_index, * from, * to_next; mpls_input_runtime_t * rt; mpls_main_t * mm; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); vlib_simple_counter_main_t * cm; vnet_main_t * vnm = vnet_get_main(); @@ -151,7 +151,7 @@ mpls_input_inline (vlib_main_t * vm, next0 = MPLS_INPUT_NEXT_LOOKUP; vnet_feature_arc_start(mm->input_feature_arc_index, sw_if_index0, &next0, b0); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); } if (PREDICT_FALSE(h1[3] == 0)) @@ -164,7 +164,7 @@ mpls_input_inline (vlib_main_t * vm, next1 = MPLS_INPUT_NEXT_LOOKUP; vnet_feature_arc_start(mm->input_feature_arc_index, sw_if_index1, &next1, b1); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index1, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index1, 1); } if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) @@ -215,7 +215,7 @@ mpls_input_inline (vlib_main_t * vm, { next0 = MPLS_INPUT_NEXT_LOOKUP; vnet_feature_arc_start(mm->input_feature_arc_index, sw_if_index0, &next0, b0); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); } if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) diff --git a/src/vnet/mpls/mpls_lookup.c b/src/vnet/mpls/mpls_lookup.c index 475bb204..ace6a70f 100644 --- a/src/vnet/mpls/mpls_lookup.c +++ b/src/vnet/mpls/mpls_lookup.c @@ -67,7 +67,7 @@ mpls_lookup (vlib_main_t * vm, vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters; u32 n_left_from, next_index, * from, * to_next; mpls_main_t * mm = &mpls_main; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -220,16 +220,16 @@ mpls_lookup (vlib_main_t * vm, vnet_buffer (b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, b1)); vlib_increment_combined_counter - (cm, cpu_index, lbi2, 1, + (cm, thread_index, lbi2, 1, vlib_buffer_length_in_chain (vm, b2)); vlib_increment_combined_counter - (cm, cpu_index, lbi3, 1, + (cm, thread_index, lbi3, 1, vlib_buffer_length_in_chain (vm, b3)); /* @@ -351,7 +351,7 @@ mpls_lookup (vlib_main_t * vm, vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); /* @@ -440,7 +440,7 @@ mpls_load_balance (vlib_main_t * vm, { vlib_combined_counter_main_t * cm = &load_balance_main.lbm_via_counters; u32 n_left_from, n_left_to_next, * from, * to_next; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); u32 next; from = vlib_frame_vector_args (frame); @@ -536,10 +536,10 @@ mpls_load_balance (vlib_main_t * vm, vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); if (PREDICT_FALSE(p0->flags & VLIB_BUFFER_IS_TRACED)) @@ -597,7 +597,7 @@ mpls_load_balance (vlib_main_t * vm, vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_validate_buffer_enqueue_x1 (vm, node, next, diff --git a/src/vnet/mpls/mpls_output.c b/src/vnet/mpls/mpls_output.c index 08018fd1..d90dec21 100644 --- a/src/vnet/mpls/mpls_output.c +++ b/src/vnet/mpls/mpls_output.c @@ -64,12 +64,12 @@ mpls_output_inline (vlib_main_t * vm, vlib_frame_t * from_frame, int is_midchain) { - u32 n_left_from, next_index, * from, * to_next, cpu_index; + u32 n_left_from, next_index, * from, * to_next, thread_index; vlib_node_runtime_t * error_node; u32 n_left_to_next; mpls_main_t *mm; - cpu_index = os_get_cpu_number(); + thread_index = vlib_get_thread_index(); error_node = vlib_node_get_runtime (vm, mpls_output_node.index); from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -137,13 +137,13 @@ mpls_output_inline (vlib_main_t * vm, /* Bump the adj counters for packet and bytes */ vlib_increment_combined_counter (&adjacency_counters, - cpu_index, + thread_index, adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); vlib_increment_combined_counter (&adjacency_counters, - cpu_index, + thread_index, adj_index1, 1, vlib_buffer_length_in_chain (vm, p1) + rw_len1); @@ -245,7 +245,7 @@ mpls_output_inline (vlib_main_t * vm, vlib_increment_combined_counter (&adjacency_counters, - cpu_index, + thread_index, adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c index 2649798b..597ae060 100644 --- a/src/vnet/pg/input.c +++ b/src/vnet/pg/input.c @@ -893,7 +893,7 @@ pg_generate_set_lengths (pg_main_t * pg, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number (), + vlib_get_thread_index (), si->sw_if_index, n_buffers, length_sum); } @@ -1266,7 +1266,7 @@ pg_stream_fill_helper (pg_main_t * pg, l += vlib_buffer_index_length_in_chain (vm, buffers[i]); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number (), + vlib_get_thread_index (), si->sw_if_index, n_alloc, l); s->current_replay_packet_index += n_alloc; s->current_replay_packet_index %= diff --git a/src/vnet/replication.c b/src/vnet/replication.c index 86d922b5..233a8c2f 100644 --- a/src/vnet/replication.c +++ b/src/vnet/replication.c @@ -31,16 +31,16 @@ replication_prep (vlib_main_t * vm, { replication_main_t *rm = &replication_main; replication_context_t *ctx; - uword cpu_number = vm->cpu_index; + uword thread_index = vm->thread_index; ip4_header_t *ip; u32 ctx_id; /* Allocate a context, reserve context 0 */ - if (PREDICT_FALSE (rm->contexts[cpu_number] == 0)) - pool_get_aligned (rm->contexts[cpu_number], ctx, CLIB_CACHE_LINE_BYTES); + if (PREDICT_FALSE (rm->contexts[thread_index] == 0)) + pool_get_aligned (rm->contexts[thread_index], ctx, CLIB_CACHE_LINE_BYTES); - pool_get_aligned (rm->contexts[cpu_number], ctx, CLIB_CACHE_LINE_BYTES); - ctx_id = ctx - rm->contexts[cpu_number]; + pool_get_aligned (rm->contexts[thread_index], ctx, CLIB_CACHE_LINE_BYTES); + ctx_id = ctx - rm->contexts[thread_index]; /* Save state from vlib buffer */ ctx->saved_free_list_index = b0->free_list_index; @@ -94,11 +94,11 @@ replication_recycle (vlib_main_t * vm, vlib_buffer_t * b0, u32 is_last) { replication_main_t *rm = &replication_main; replication_context_t *ctx; - uword cpu_number = vm->cpu_index; + uword thread_index = vm->thread_index; ip4_header_t *ip; /* Get access to the replication context */ - ctx = pool_elt_at_index (rm->contexts[cpu_number], b0->recycle_count); + ctx = pool_elt_at_index (rm->contexts[thread_index], b0->recycle_count); /* Restore vnet buffer state */ clib_memcpy (vnet_buffer (b0), ctx->vnet_buffer, @@ -133,7 +133,7 @@ replication_recycle (vlib_main_t * vm, vlib_buffer_t * b0, u32 is_last) b0->flags &= ~VLIB_BUFFER_RECYCLE; /* Free context back to its pool */ - pool_put (rm->contexts[cpu_number], ctx); + pool_put (rm->contexts[thread_index], ctx); } return ctx; @@ -160,7 +160,7 @@ replication_recycle_callback (vlib_main_t * vm, vlib_buffer_free_list_t * fl) replication_main_t *rm = &replication_main; replication_context_t *ctx; u32 feature_node_index = 0; - uword cpu_number = vm->cpu_index; + uword thread_index = vm->thread_index; /* * All buffers in the list are destined to the same recycle node. @@ -172,7 +172,7 @@ replication_recycle_callback (vlib_main_t * vm, vlib_buffer_free_list_t * fl) { bi0 = fl->buffers[0]; b0 = vlib_get_buffer (vm, bi0); - ctx = pool_elt_at_index (rm->contexts[cpu_number], b0->recycle_count); + ctx = pool_elt_at_index (rm->contexts[thread_index], b0->recycle_count); feature_node_index = ctx->recycle_node_index; } diff --git a/src/vnet/replication.h b/src/vnet/replication.h index 5dc554c9..ce4b3ff1 100644 --- a/src/vnet/replication.h +++ b/src/vnet/replication.h @@ -100,7 +100,7 @@ replication_get_ctx (vlib_buffer_t * b0) replication_main_t *rm = &replication_main; return replication_is_recycled (b0) ? - pool_elt_at_index (rm->contexts[os_get_cpu_number ()], + pool_elt_at_index (rm->contexts[vlib_get_thread_index ()], b0->recycle_count) : 0; } diff --git a/src/vnet/session/node.c b/src/vnet/session/node.c index b86e87d9..dd211c51 100644 --- a/src/vnet/session/node.c +++ b/src/vnet/session/node.c @@ -311,7 +311,7 @@ session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, unix_shared_memory_queue_t *q; application_t *app; int n_tx_packets = 0; - u32 my_thread_index = vm->cpu_index; + u32 my_thread_index = vm->thread_index; int i, rv; f64 now = vlib_time_now (vm); diff --git a/src/vnet/sr/sr_localsid.c b/src/vnet/sr/sr_localsid.c index 2e3d56de..6d72a506 100755 --- a/src/vnet/sr/sr_localsid.c +++ b/src/vnet/sr/sr_localsid.c @@ -887,7 +887,7 @@ sr_localsid_d_fn (vlib_main_t * vm, vlib_node_runtime_t * node, from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; next_index = node->cached_next_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -974,26 +974,26 @@ sr_localsid_d_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_increment_combined_counter (((next0 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls0 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b0)); + &(sm->sr_ls_valid_counters)), thread_index, ls0 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter (((next1 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls1 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b1)); + &(sm->sr_ls_valid_counters)), thread_index, ls1 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b1)); vlib_increment_combined_counter (((next2 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls2 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b2)); + &(sm->sr_ls_valid_counters)), thread_index, ls2 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b2)); vlib_increment_combined_counter (((next3 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls3 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b3)); + &(sm->sr_ls_valid_counters)), thread_index, ls3 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b3)); vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, n_left_to_next, bi0, bi1, bi2, bi3, @@ -1062,8 +1062,8 @@ sr_localsid_d_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_increment_combined_counter (((next0 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls0 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b0)); + &(sm->sr_ls_valid_counters)), thread_index, ls0 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b0)); vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0); @@ -1103,7 +1103,7 @@ sr_localsid_fn (vlib_main_t * vm, vlib_node_runtime_t * node, from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; next_index = node->cached_next_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -1205,26 +1205,26 @@ sr_localsid_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_increment_combined_counter (((next0 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls0 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b0)); + &(sm->sr_ls_valid_counters)), thread_index, ls0 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter (((next1 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls1 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b1)); + &(sm->sr_ls_valid_counters)), thread_index, ls1 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b1)); vlib_increment_combined_counter (((next2 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls2 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b2)); + &(sm->sr_ls_valid_counters)), thread_index, ls2 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b2)); vlib_increment_combined_counter (((next3 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls3 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b3)); + &(sm->sr_ls_valid_counters)), thread_index, ls3 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b3)); vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, n_left_to_next, bi0, bi1, bi2, bi3, @@ -1295,8 +1295,8 @@ sr_localsid_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_increment_combined_counter (((next0 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls0 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b0)); + &(sm->sr_ls_valid_counters)), thread_index, ls0 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b0)); vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0); diff --git a/src/vnet/tcp/builtin_client.c b/src/vnet/tcp/builtin_client.c index e3705060..c1567aa0 100644 --- a/src/vnet/tcp/builtin_client.c +++ b/src/vnet/tcp/builtin_client.c @@ -174,7 +174,7 @@ tclient_thread_fn (void *arg) pthread_sigmask (SIG_SETMASK, &s, 0); } - clib_per_cpu_mheaps[os_get_cpu_number ()] = clib_per_cpu_mheaps[0]; + clib_per_cpu_mheaps[vlib_get_thread_index ()] = clib_per_cpu_mheaps[0]; while (1) { diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c index b2a371e2..b6c34828 100644 --- a/src/vnet/tcp/tcp.c +++ b/src/vnet/tcp/tcp.c @@ -646,10 +646,10 @@ const static transport_proto_vft_t tcp6_proto = { void tcp_timer_keep_handler (u32 conn_index) { - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); tcp_connection_t *tc; - tc = tcp_connection_get (conn_index, cpu_index); + tc = tcp_connection_get (conn_index, thread_index); tc->timers[TCP_TIMER_KEEP] = TCP_TIMER_HANDLE_INVALID; tcp_connection_close (tc); @@ -675,10 +675,10 @@ tcp_timer_establish_handler (u32 conn_index) void tcp_timer_waitclose_handler (u32 conn_index) { - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); tcp_connection_t *tc; - tc = tcp_connection_get (conn_index, cpu_index); + tc = tcp_connection_get (conn_index, thread_index); tc->timers[TCP_TIMER_WAITCLOSE] = TCP_TIMER_HANDLE_INVALID; /* Session didn't come back with a close(). Send FIN either way diff --git a/src/vnet/tcp/tcp_debug.h b/src/vnet/tcp/tcp_debug.h index 0090e15e..eaca672c 100644 --- a/src/vnet/tcp/tcp_debug.h +++ b/src/vnet/tcp/tcp_debug.h @@ -343,7 +343,7 @@ typedef enum _tcp_dbg_evt } \ else \ { \ - u32 _thread_index = os_get_cpu_number (); \ + u32 _thread_index = vlib_get_thread_index (); \ _tc = tcp_connection_get (_tc_index, _thread_index); \ } \ ELOG_TYPE_DECLARE (_e) = \ diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c index a8224dc2..7e9fa47b 100644 --- a/src/vnet/tcp/tcp_input.c +++ b/src/vnet/tcp/tcp_input.c @@ -1142,7 +1142,7 @@ tcp46_established_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame, int is_ip4) { u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index, errors = 0; + u32 my_thread_index = vm->thread_index, errors = 0; tcp_main_t *tm = vnet_get_tcp_main (); from = vlib_frame_vector_args (from_frame); @@ -1332,7 +1332,7 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { tcp_main_t *tm = vnet_get_tcp_main (); u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index, errors = 0; + u32 my_thread_index = vm->thread_index, errors = 0; u8 sst = is_ip4 ? SESSION_TYPE_IP4_TCP : SESSION_TYPE_IP6_TCP; from = vlib_frame_vector_args (from_frame); @@ -1634,7 +1634,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { tcp_main_t *tm = vnet_get_tcp_main (); u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index, errors = 0; + u32 my_thread_index = vm->thread_index, errors = 0; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -1989,7 +1989,7 @@ tcp46_listen_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame, int is_ip4) { u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index; + u32 my_thread_index = vm->thread_index; tcp_main_t *tm = vnet_get_tcp_main (); u8 sst = is_ip4 ? SESSION_TYPE_IP4_TCP : SESSION_TYPE_IP6_TCP; @@ -2243,7 +2243,7 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame, int is_ip4) { u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index; + u32 my_thread_index = vm->thread_index; tcp_main_t *tm = vnet_get_tcp_main (); from = vlib_frame_vector_args (from_frame); diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index ea157bd7..e18bfad7 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -387,8 +387,8 @@ tcp_make_options (tcp_connection_t * tc, tcp_options_t * opts, #define tcp_get_free_buffer_index(tm, bidx) \ do { \ u32 *my_tx_buffers, n_free_buffers; \ - u32 cpu_index = os_get_cpu_number(); \ - my_tx_buffers = tm->tx_buffers[cpu_index]; \ + u32 thread_index = vlib_get_thread_index(); \ + my_tx_buffers = tm->tx_buffers[thread_index]; \ if (PREDICT_FALSE(vec_len (my_tx_buffers) == 0)) \ { \ n_free_buffers = 32; /* TODO config or macro */ \ @@ -396,7 +396,7 @@ do { \ _vec_len(my_tx_buffers) = vlib_buffer_alloc_from_free_list ( \ tm->vlib_main, my_tx_buffers, n_free_buffers, \ VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); \ - tm->tx_buffers[cpu_index] = my_tx_buffers; \ + tm->tx_buffers[thread_index] = my_tx_buffers; \ } \ /* buffer shortage */ \ if (PREDICT_FALSE (vec_len (my_tx_buffers) == 0)) \ @@ -408,8 +408,8 @@ do { \ #define tcp_return_buffer(tm) \ do { \ u32 *my_tx_buffers; \ - u32 cpu_index = os_get_cpu_number(); \ - my_tx_buffers = tm->tx_buffers[cpu_index]; \ + u32 thread_index = vlib_get_thread_index(); \ + my_tx_buffers = tm->tx_buffers[thread_index]; \ _vec_len (my_tx_buffers) +=1; \ } while (0) @@ -942,7 +942,7 @@ tcp_send_ack (tcp_connection_t * tc) void tcp_timer_delack_handler (u32 index) { - u32 thread_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); tcp_connection_t *tc; tc = tcp_connection_get (index, thread_index); @@ -1022,7 +1022,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) { tcp_main_t *tm = vnet_get_tcp_main (); vlib_main_t *vm = vlib_get_main (); - u32 thread_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); tcp_connection_t *tc; vlib_buffer_t *b; u32 bi, snd_space, n_bytes; @@ -1152,7 +1152,7 @@ tcp_timer_persist_handler (u32 index) { tcp_main_t *tm = vnet_get_tcp_main (); vlib_main_t *vm = vlib_get_main (); - u32 thread_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); tcp_connection_t *tc; vlib_buffer_t *b; u32 bi, n_bytes; @@ -1313,7 +1313,7 @@ tcp46_output_inline (vlib_main_t * vm, vlib_frame_t * from_frame, int is_ip4) { u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index; + u32 my_thread_index = vm->thread_index; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -1524,7 +1524,7 @@ tcp46_send_reset_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame, u8 is_ip4) { u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index; + u32 my_thread_index = vm->thread_index; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; diff --git a/src/vnet/udp/udp_input.c b/src/vnet/udp/udp_input.c index 4b22109b..810278e6 100644 --- a/src/vnet/udp/udp_input.c +++ b/src/vnet/udp/udp_input.c @@ -70,7 +70,7 @@ udp4_uri_input_node_fn (vlib_main_t * vm, udp4_uri_input_next_t next_index; udp_uri_main_t *um = vnet_get_udp_main (); session_manager_main_t *smm = vnet_get_session_manager_main (); - u32 my_thread_index = vm->cpu_index; + u32 my_thread_index = vm->thread_index; u8 my_enqueue_epoch; u32 *session_indices_to_enqueue; static u32 serial_number; diff --git a/src/vnet/unix/tapcli.c b/src/vnet/unix/tapcli.c index fb1a8bac..0fc62f6c 100644 --- a/src/vnet/unix/tapcli.c +++ b/src/vnet/unix/tapcli.c @@ -366,7 +366,7 @@ static uword tapcli_rx_iface(vlib_main_t * vm, vlib_increment_combined_counter ( vnet_main.interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number(), ti->sw_if_index, + vlib_get_thread_index(), ti->sw_if_index, 1, n_bytes_in_packet); if (PREDICT_FALSE(n_trace > 0)) { diff --git a/src/vnet/unix/tuntap.c b/src/vnet/unix/tuntap.c index 2cfcc92f..ac674653 100644 --- a/src/vnet/unix/tuntap.c +++ b/src/vnet/unix/tuntap.c @@ -189,7 +189,7 @@ tuntap_tx (vlib_main_t * vm, /* Update tuntap interface output stats. */ vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - vm->cpu_index, + vm->thread_index, tm->sw_if_index, n_packets, n_bytes); @@ -297,7 +297,7 @@ tuntap_rx (vlib_main_t * vm, vlib_increment_combined_counter (vnet_main.interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number(), + vlib_get_thread_index(), tm->sw_if_index, 1, n_bytes_in_packet); diff --git a/src/vnet/vxlan-gpe/decap.c b/src/vnet/vxlan-gpe/decap.c index 22ab4b62..d4fe4231 100644 --- a/src/vnet/vxlan-gpe/decap.c +++ b/src/vnet/vxlan-gpe/decap.c @@ -115,7 +115,7 @@ vxlan_gpe_input (vlib_main_t * vm, vxlan4_gpe_tunnel_key_t last_key4; vxlan6_gpe_tunnel_key_t last_key6; u32 pkts_decapsulated = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; if (is_ip4) @@ -342,7 +342,7 @@ vxlan_gpe_input (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; stats_sw_if_index = sw_if_index0; @@ -427,7 +427,7 @@ vxlan_gpe_input (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len1; stats_sw_if_index = sw_if_index1; @@ -588,7 +588,7 @@ vxlan_gpe_input (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; stats_sw_if_index = sw_if_index0; @@ -615,7 +615,7 @@ vxlan_gpe_input (vlib_main_t * vm, if (stats_n_packets) { vlib_increment_combined_counter ( - im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, cpu_index, + im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } diff --git a/src/vnet/vxlan-gpe/encap.c b/src/vnet/vxlan-gpe/encap.c index 3a486e56..67ed94b4 100644 --- a/src/vnet/vxlan-gpe/encap.c +++ b/src/vnet/vxlan-gpe/encap.c @@ -151,7 +151,7 @@ vxlan_gpe_encap (vlib_main_t * vm, vnet_main_t * vnm = ngm->vnet_main; vnet_interface_main_t * im = &vnm->interface_main; u32 pkts_encapsulated = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; from = vlib_frame_vector_args (from_frame); @@ -253,7 +253,7 @@ vxlan_gpe_encap (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_sw_if_index = sw_if_index0; stats_n_packets = 2; stats_n_bytes = len0 + len1; @@ -262,10 +262,10 @@ vxlan_gpe_encap (vlib_main_t * vm, { vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, sw_if_index0, 1, len0); + thread_index, sw_if_index0, 1, len0); vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, sw_if_index1, 1, len1); + thread_index, sw_if_index1, 1, len1); } } @@ -335,7 +335,7 @@ vxlan_gpe_encap (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; stats_sw_if_index = sw_if_index0; @@ -359,7 +359,7 @@ vxlan_gpe_encap (vlib_main_t * vm, if (stats_n_packets) { vlib_increment_combined_counter ( - im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, cpu_index, + im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } diff --git a/src/vnet/vxlan/decap.c b/src/vnet/vxlan/decap.c index 514b2c99..2acb1f6f 100644 --- a/src/vnet/vxlan/decap.c +++ b/src/vnet/vxlan/decap.c @@ -81,7 +81,7 @@ vxlan_input (vlib_main_t * vm, vxlan4_tunnel_key_t last_key4; vxlan6_tunnel_key_t last_key6; u32 pkts_decapsulated = 0; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; if (is_ip4) @@ -314,7 +314,7 @@ vxlan_input (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; @@ -468,7 +468,7 @@ vxlan_input (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len1; @@ -674,7 +674,7 @@ vxlan_input (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; @@ -711,7 +711,7 @@ vxlan_input (vlib_main_t * vm, { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } diff --git a/src/vnet/vxlan/encap.c b/src/vnet/vxlan/encap.c index 5b63064a..4cfbbc23 100644 --- a/src/vnet/vxlan/encap.c +++ b/src/vnet/vxlan/encap.c @@ -77,7 +77,7 @@ vxlan_encap_inline (vlib_main_t * vm, vnet_interface_main_t * im = &vnm->interface_main; u32 pkts_encapsulated = 0; u16 old_l0 = 0, old_l1 = 0; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; u32 sw_if_index0 = 0, sw_if_index1 = 0; u32 next0 = 0, next1 = 0; @@ -301,7 +301,7 @@ vxlan_encap_inline (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, stats_sw_if_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_sw_if_index = sw_if_index0; stats_n_packets = 2; @@ -311,10 +311,10 @@ vxlan_encap_inline (vlib_main_t * vm, { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, sw_if_index0, 1, len0); + thread_index, sw_if_index0, 1, len0); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, sw_if_index1, 1, len1); + thread_index, sw_if_index1, 1, len1); } } @@ -464,7 +464,7 @@ vxlan_encap_inline (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, stats_sw_if_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; @@ -496,7 +496,7 @@ vxlan_encap_inline (vlib_main_t * vm, { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } diff --git a/src/vpp/stats/stats.c b/src/vpp/stats/stats.c index 042d02e2..4309cd51 100644 --- a/src/vpp/stats/stats.c +++ b/src/vpp/stats/stats.c @@ -66,14 +66,14 @@ _(VNET_IP6_NBR_COUNTERS, vnet_ip6_nbr_counters) void dslock (stats_main_t * sm, int release_hint, int tag) { - u32 thread_id; + u32 thread_index; data_structure_lock_t *l = sm->data_structure_lock; if (PREDICT_FALSE (l == 0)) return; - thread_id = os_get_cpu_number (); - if (l->lock && l->thread_id == thread_id) + thread_index = vlib_get_thread_index (); + if (l->lock && l->thread_index == thread_index) { l->count++; return; @@ -85,7 +85,7 @@ dslock (stats_main_t * sm, int release_hint, int tag) while (__sync_lock_test_and_set (&l->lock, 1)) /* zzzz */ ; l->tag = tag; - l->thread_id = thread_id; + l->thread_index = thread_index; l->count = 1; } @@ -99,14 +99,14 @@ stats_dslock_with_hint (int hint, int tag) void dsunlock (stats_main_t * sm) { - u32 thread_id; + u32 thread_index; data_structure_lock_t *l = sm->data_structure_lock; if (PREDICT_FALSE (l == 0)) return; - thread_id = os_get_cpu_number (); - ASSERT (l->lock && l->thread_id == thread_id); + thread_index = vlib_get_thread_index (); + ASSERT (l->lock && l->thread_index == thread_index); l->count--; if (l->count == 0) { diff --git a/src/vpp/stats/stats.h b/src/vpp/stats/stats.h index 118115be..024dc78e 100644 --- a/src/vpp/stats/stats.h +++ b/src/vpp/stats/stats.h @@ -30,7 +30,7 @@ typedef struct { volatile u32 lock; volatile u32 release_hint; - u32 thread_id; + u32 thread_index; u32 count; int tag; } data_structure_lock_t; -- cgit 1.2.3-korg From 0f26c5a0138ac86d7ebd197c31a09d8d624c35fe Mon Sep 17 00:00:00 2001 From: Neale Ranns Date: Wed, 1 Mar 2017 15:12:11 -0800 Subject: MPLS Mcast 1 - interface-DPO Used in the Data-plane to change a packet's input interface 2 - MPLS multicast FIB entry Same as a unicast entry but it links to a replicate not a load-balance DPO 3 - Multicast MPLS tunnel Update MPLS tunnels to use a FIB path-list to describe the endpoint[s]. Use the path-list to generate the forwarding chain (DPOs) to link to . 4 - Resolve a path via a local label (of an mLDP LSP) For IP multicast entries to use an LSP in the replication list, we need to decribe the 'resolve-via-label' where the label is that of a multicast LSP. 5 - MPLS disposition path sets RPF-ID For a interface-less LSP (i.e. mLDP not RSVP-TE) at the tail of the LSP we still need to perform an RPF check. An MPLS disposition DPO performs the MPLS pop validation checks and sets the RPF-ID in the packet. 6 - RPF check with per-entry RPF-ID An RPF-ID is used instead of a real interface SW if index in the case the IP traffic arrives from an LSP that does not have an associated interface. Change-Id: Ib92e177be919147bafeb599729abf3d1abc2f4b3 Signed-off-by: Neale Ranns --- src/plugins/dpdk/device/node.c | 2 +- src/vat/api_format.c | 133 ++--- src/vnet.am | 2 + src/vnet/adj/adj.c | 13 +- src/vnet/adj/adj.h | 6 + src/vnet/adj/adj_internal.h | 14 +- src/vnet/adj/adj_mcast.c | 134 ++++- src/vnet/adj/adj_mcast.h | 27 + src/vnet/adj/adj_midchain.c | 62 ++- src/vnet/adj/adj_nbr.c | 2 - src/vnet/buffer.h | 3 + src/vnet/devices/ssvm/node.c | 2 +- src/vnet/dhcp/dhcp6_proxy_node.c | 1 + src/vnet/dpo/dpo.c | 10 + src/vnet/dpo/dpo.h | 8 +- src/vnet/dpo/interface_dpo.c | 416 ++++++++++++++++ src/vnet/dpo/interface_dpo.h | 67 +++ src/vnet/dpo/lookup_dpo.c | 211 +++++++- src/vnet/dpo/lookup_dpo.h | 20 + src/vnet/dpo/mpls_disposition.c | 364 ++++++++++++++ src/vnet/dpo/mpls_disposition.h | 85 ++++ src/vnet/dpo/mpls_label_dpo.c | 6 +- src/vnet/dpo/replicate_dpo.c | 48 +- src/vnet/dpo/replicate_dpo.h | 2 + src/vnet/ethernet/arp.c | 1 + src/vnet/ethernet/interface.c | 2 +- src/vnet/ethernet/node.c | 4 +- src/vnet/ethernet/types.def | 4 +- src/vnet/fib/fib_api.h | 4 + src/vnet/fib/fib_entry.c | 47 +- src/vnet/fib/fib_entry.h | 13 +- src/vnet/fib/fib_entry_src.c | 154 +++--- src/vnet/fib/fib_internal.h | 1 + src/vnet/fib/fib_path.c | 222 +++++++-- src/vnet/fib/fib_path.h | 17 +- src/vnet/fib/fib_path_ext.c | 4 +- src/vnet/fib/fib_path_ext.h | 3 +- src/vnet/fib/fib_path_list.c | 270 ++++++---- src/vnet/fib/fib_path_list.h | 22 +- src/vnet/fib/fib_table.c | 47 +- src/vnet/fib/fib_test.c | 345 +++++++++++-- src/vnet/fib/fib_test.h | 111 +++++ src/vnet/fib/fib_types.c | 15 +- src/vnet/fib/fib_types.h | 60 ++- src/vnet/fib/mpls_fib.c | 15 +- src/vnet/handoff.h | 10 +- src/vnet/interface.c | 2 +- src/vnet/ip/ip.api | 3 + src/vnet/ip/ip4_forward.c | 20 + src/vnet/ip/ip6_forward.c | 23 + src/vnet/ip/ip6_neighbor.c | 1 + src/vnet/ip/ip_api.c | 98 ++-- src/vnet/ip/lookup.c | 3 +- src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c | 1 + src/vnet/mfib/ip4_mfib.c | 1 + src/vnet/mfib/ip6_mfib.c | 1 + src/vnet/mfib/mfib_entry.c | 395 +++++++++------ src/vnet/mfib/mfib_entry.h | 20 +- src/vnet/mfib/mfib_forward.c | 29 +- src/vnet/mfib/mfib_table.c | 8 +- src/vnet/mfib/mfib_table.h | 1 + src/vnet/mfib/mfib_test.c | 127 ++++- src/vnet/mpls/mpls.api | 87 ++-- src/vnet/mpls/mpls.c | 17 +- src/vnet/mpls/mpls_api.c | 97 ++-- src/vnet/mpls/mpls_input.c | 2 +- src/vnet/mpls/mpls_lookup.c | 236 ++++++--- src/vnet/mpls/mpls_tunnel.c | 883 ++++++++++++++++++++++----------- src/vnet/mpls/mpls_tunnel.h | 57 ++- src/vnet/mpls/mpls_types.h | 20 + src/vnet/srp/interface.c | 2 +- test/test_ip_mcast.py | 1 + test/test_mpls.py | 277 ++++++++++- test/vpp_ip_route.py | 38 +- test/vpp_mpls_tunnel_interface.py | 46 ++ test/vpp_papi_provider.py | 16 +- 76 files changed, 4393 insertions(+), 1128 deletions(-) create mode 100644 src/vnet/dpo/interface_dpo.c create mode 100644 src/vnet/dpo/interface_dpo.h create mode 100644 src/vnet/dpo/mpls_disposition.c create mode 100644 src/vnet/dpo/mpls_disposition.h create mode 100644 src/vnet/fib/fib_test.h create mode 100644 test/vpp_mpls_tunnel_interface.py (limited to 'src/vnet/mpls/mpls_input.c') diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index b10e0fad..0549ba5d 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -52,7 +52,7 @@ always_inline int vlib_buffer_is_mpls (vlib_buffer_t * b) { ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b); - return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST)); + return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)); } always_inline u32 diff --git a/src/vat/api_format.c b/src/vat/api_format.c index 61b8e1d8..107aa012 100644 --- a/src/vat/api_format.c +++ b/src/vat/api_format.c @@ -16369,32 +16369,82 @@ api_netmap_delete (vat_main_t * vam) return ret; } -static void vl_api_mpls_tunnel_details_t_handler - (vl_api_mpls_tunnel_details_t * mp) +static void +vl_api_mpls_fib_path_print (vat_main_t * vam, vl_api_fib_path2_t * fp) +{ + if (fp->afi == IP46_TYPE_IP6) + print (vam->ofp, + " weight %d, sw_if_index %d, is_local %d, is_drop %d, " + "is_unreach %d, is_prohitbit %d, afi %d, next_hop %U", + ntohl (fp->weight), ntohl (fp->sw_if_index), fp->is_local, + fp->is_drop, fp->is_unreach, fp->is_prohibit, fp->afi, + format_ip6_address, fp->next_hop); + else if (fp->afi == IP46_TYPE_IP4) + print (vam->ofp, + " weight %d, sw_if_index %d, is_local %d, is_drop %d, " + "is_unreach %d, is_prohitbit %d, afi %d, next_hop %U", + ntohl (fp->weight), ntohl (fp->sw_if_index), fp->is_local, + fp->is_drop, fp->is_unreach, fp->is_prohibit, fp->afi, + format_ip4_address, fp->next_hop); +} + +static void +vl_api_mpls_fib_path_json_print (vat_json_node_t * node, + vl_api_fib_path2_t * fp) +{ + struct in_addr ip4; + struct in6_addr ip6; + + vat_json_object_add_uint (node, "weight", ntohl (fp->weight)); + vat_json_object_add_uint (node, "sw_if_index", ntohl (fp->sw_if_index)); + vat_json_object_add_uint (node, "is_local", fp->is_local); + vat_json_object_add_uint (node, "is_drop", fp->is_drop); + vat_json_object_add_uint (node, "is_unreach", fp->is_unreach); + vat_json_object_add_uint (node, "is_prohibit", fp->is_prohibit); + vat_json_object_add_uint (node, "next_hop_afi", fp->afi); + if (fp->afi == IP46_TYPE_IP4) + { + clib_memcpy (&ip4, &fp->next_hop, sizeof (ip4)); + vat_json_object_add_ip4 (node, "next_hop", ip4); + } + else if (fp->afi == IP46_TYPE_IP6) + { + clib_memcpy (&ip6, &fp->next_hop, sizeof (ip6)); + vat_json_object_add_ip6 (node, "next_hop", ip6); + } +} + +static void +vl_api_mpls_tunnel_details_t_handler (vl_api_mpls_tunnel_details_t * mp) { vat_main_t *vam = &vat_main; - i32 len = mp->mt_next_hop_n_labels; + int count = ntohl (mp->mt_count); + vl_api_fib_path2_t *fp; i32 i; - print (vam->ofp, "[%d]: via %U %d labels ", - mp->tunnel_index, - format_ip4_address, mp->mt_next_hop, - ntohl (mp->mt_next_hop_sw_if_index)); - for (i = 0; i < len; i++) + print (vam->ofp, "[%d]: sw_if_index %d via:", + ntohl (mp->mt_tunnel_index), ntohl (mp->mt_sw_if_index)); + fp = mp->mt_paths; + for (i = 0; i < count; i++) { - print (vam->ofp, "%u ", ntohl (mp->mt_next_hop_out_labels[i])); + vl_api_mpls_fib_path_print (vam, fp); + fp++; } + print (vam->ofp, ""); } -static void vl_api_mpls_tunnel_details_t_handler_json - (vl_api_mpls_tunnel_details_t * mp) +#define vl_api_mpls_tunnel_details_t_endian vl_noop_handler +#define vl_api_mpls_tunnel_details_t_print vl_noop_handler + +static void +vl_api_mpls_tunnel_details_t_handler_json (vl_api_mpls_tunnel_details_t * mp) { vat_main_t *vam = &vat_main; vat_json_node_t *node = NULL; - struct in_addr ip4; + int count = ntohl (mp->mt_count); + vl_api_fib_path2_t *fp; i32 i; - i32 len = mp->mt_next_hop_n_labels; if (VAT_JSON_ARRAY != vam->json_tree.type) { @@ -16404,17 +16454,17 @@ static void vl_api_mpls_tunnel_details_t_handler_json node = vat_json_array_add (&vam->json_tree); vat_json_init_object (node); - vat_json_object_add_uint (node, "tunnel_index", ntohl (mp->tunnel_index)); - clib_memcpy (&ip4, &(mp->mt_next_hop), sizeof (ip4)); - vat_json_object_add_ip4 (node, "next_hop", ip4); - vat_json_object_add_uint (node, "next_hop_sw_if_index", - ntohl (mp->mt_next_hop_sw_if_index)); - vat_json_object_add_uint (node, "l2_only", ntohl (mp->mt_l2_only)); - vat_json_object_add_uint (node, "label_count", len); - for (i = 0; i < len; i++) + vat_json_object_add_uint (node, "tunnel_index", + ntohl (mp->mt_tunnel_index)); + vat_json_object_add_uint (node, "sw_if_index", ntohl (mp->mt_sw_if_index)); + + vat_json_object_add_uint (node, "l2_only", mp->mt_l2_only); + + fp = mp->mt_paths; + for (i = 0; i < count; i++) { - vat_json_object_add_uint (node, "label", - ntohl (mp->mt_next_hop_out_labels[i])); + vl_api_mpls_fib_path_json_print (node, fp); + fp++; } } @@ -16453,6 +16503,7 @@ api_mpls_tunnel_dump (vat_main_t * vam) #define vl_api_mpls_fib_details_t_endian vl_noop_handler #define vl_api_mpls_fib_details_t_print vl_noop_handler + static void vl_api_mpls_fib_details_t_handler (vl_api_mpls_fib_details_t * mp) { @@ -16467,20 +16518,7 @@ vl_api_mpls_fib_details_t_handler (vl_api_mpls_fib_details_t * mp) fp = mp->path; for (i = 0; i < count; i++) { - if (fp->afi == IP46_TYPE_IP6) - print (vam->ofp, - " weight %d, sw_if_index %d, is_local %d, is_drop %d, " - "is_unreach %d, is_prohitbit %d, afi %d, next_hop %U", - ntohl (fp->weight), ntohl (fp->sw_if_index), fp->is_local, - fp->is_drop, fp->is_unreach, fp->is_prohibit, fp->afi, - format_ip6_address, fp->next_hop); - else if (fp->afi == IP46_TYPE_IP4) - print (vam->ofp, - " weight %d, sw_if_index %d, is_local %d, is_drop %d, " - "is_unreach %d, is_prohitbit %d, afi %d, next_hop %U", - ntohl (fp->weight), ntohl (fp->sw_if_index), fp->is_local, - fp->is_drop, fp->is_unreach, fp->is_prohibit, fp->afi, - format_ip4_address, fp->next_hop); + vl_api_mpls_fib_path_print (vam, fp); fp++; } } @@ -16491,8 +16529,6 @@ static void vl_api_mpls_fib_details_t_handler_json vat_main_t *vam = &vat_main; int count = ntohl (mp->count); vat_json_node_t *node = NULL; - struct in_addr ip4; - struct in6_addr ip6; vl_api_fib_path2_t *fp; int i; @@ -16511,23 +16547,8 @@ static void vl_api_mpls_fib_details_t_handler_json fp = mp->path; for (i = 0; i < count; i++) { - vat_json_object_add_uint (node, "weight", ntohl (fp->weight)); - vat_json_object_add_uint (node, "sw_if_index", ntohl (fp->sw_if_index)); - vat_json_object_add_uint (node, "is_local", fp->is_local); - vat_json_object_add_uint (node, "is_drop", fp->is_drop); - vat_json_object_add_uint (node, "is_unreach", fp->is_unreach); - vat_json_object_add_uint (node, "is_prohibit", fp->is_prohibit); - vat_json_object_add_uint (node, "next_hop_afi", fp->afi); - if (fp->afi == IP46_TYPE_IP4) - { - clib_memcpy (&ip4, &fp->next_hop, sizeof (ip4)); - vat_json_object_add_ip4 (node, "next_hop", ip4); - } - else if (fp->afi == IP46_TYPE_IP6) - { - clib_memcpy (&ip6, &fp->next_hop, sizeof (ip6)); - vat_json_object_add_ip6 (node, "next_hop", ip6); - } + vl_api_mpls_fib_path_json_print (node, fp); + fp++; } } diff --git a/src/vnet.am b/src/vnet.am index 643ae92e..bed4902b 100644 --- a/src/vnet.am +++ b/src/vnet.am @@ -990,6 +990,8 @@ libvnet_la_SOURCES += \ vnet/dpo/lookup_dpo.c \ vnet/dpo/classify_dpo.c \ vnet/dpo/replicate_dpo.c \ + vnet/dpo/interface_dpo.c \ + vnet/dpo/mpls_disposition.c \ vnet/dpo/mpls_label_dpo.c nobase_include_HEADERS += \ diff --git a/src/vnet/adj/adj.c b/src/vnet/adj/adj.c index 90182006..36dfe500 100644 --- a/src/vnet/adj/adj.c +++ b/src/vnet/adj/adj.c @@ -67,6 +67,10 @@ adj_alloc (fib_protocol_t proto) adj->lookup_next_index = 0; adj->ia_delegates = NULL; + /* lest it become a midchain in the future */ + memset(&adj->sub_type.midchain.next_dpo, 0, + sizeof(adj->sub_type.midchain.next_dpo)); + ip4_main.lookup_main.adjacency_heap = adj_pool; ip6_main.lookup_main.adjacency_heap = adj_pool; @@ -118,6 +122,9 @@ format_ip_adjacency (u8 * s, va_list * args) case IP_LOOKUP_NEXT_MCAST: s = format (s, "%U", format_adj_mcast, adj_index, 0); break; + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: + s = format (s, "%U", format_adj_mcast_midchain, adj_index, 0); + break; default: break; } @@ -180,6 +187,7 @@ adj_last_lock_gone (ip_adjacency_t *adj) adj->rewrite_header.sw_if_index); break; case IP_LOOKUP_NEXT_MCAST: + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: adj_mcast_remove(adj->ia_nh_proto, adj->rewrite_header.sw_if_index); break; @@ -338,6 +346,7 @@ adj_walk (u32 sw_if_index, FOR_EACH_FIB_IP_PROTOCOL(proto) { adj_nbr_walk(sw_if_index, proto, cb, ctx); + adj_mcast_walk(sw_if_index, proto, cb, ctx); } } @@ -544,9 +553,9 @@ adj_show (vlib_main_t * vm, * [@0] * [@1] glean: loop0 * [@2] ipv4 via 1.0.0.2 loop0: IP4: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc - * [@3] mpls via 1.0.0.2 loop0: MPLS_UNICAST: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc + * [@3] mpls via 1.0.0.2 loop0: MPLS: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc * [@4] ipv4 via 1.0.0.3 loop0: IP4: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc - * [@5] mpls via 1.0.0.3 loop0: MPLS_UNICAST: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc + * [@5] mpls via 1.0.0.3 loop0: MPLS: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc * @cliexend ?*/ VLIB_CLI_COMMAND (adj_show_command, static) = { diff --git a/src/vnet/adj/adj.h b/src/vnet/adj/adj.h index 32997c91..ed5eb1f1 100644 --- a/src/vnet/adj/adj.h +++ b/src/vnet/adj/adj.h @@ -81,6 +81,10 @@ typedef enum /** Multicast Adjacency. */ IP_LOOKUP_NEXT_MCAST, + /** Multicast Midchain Adjacency. An Adjacency for sending macst packets + * on a tunnel/virtual interface */ + IP_LOOKUP_NEXT_MCAST_MIDCHAIN, + IP_LOOKUP_N_NEXT, } __attribute__ ((packed)) ip_lookup_next_t; @@ -107,6 +111,7 @@ typedef enum [IP_LOOKUP_NEXT_REWRITE] = "ip4-rewrite", \ [IP_LOOKUP_NEXT_MCAST] = "ip4-rewrite-mcast", \ [IP_LOOKUP_NEXT_MIDCHAIN] = "ip4-midchain", \ + [IP_LOOKUP_NEXT_MCAST_MIDCHAIN] = "ip4-mcast-midchain", \ [IP_LOOKUP_NEXT_ICMP_ERROR] = "ip4-icmp-error", \ } @@ -119,6 +124,7 @@ typedef enum [IP_LOOKUP_NEXT_REWRITE] = "ip6-rewrite", \ [IP_LOOKUP_NEXT_MCAST] = "ip6-rewrite-mcast", \ [IP_LOOKUP_NEXT_MIDCHAIN] = "ip6-midchain", \ + [IP_LOOKUP_NEXT_MCAST_MIDCHAIN] = "ip6-mcast-midchain", \ [IP_LOOKUP_NEXT_ICMP_ERROR] = "ip6-icmp-error", \ [IP6_LOOKUP_NEXT_HOP_BY_HOP] = "ip6-hop-by-hop", \ [IP6_LOOKUP_NEXT_ADD_HOP_BY_HOP] = "ip6-add-hop-by-hop", \ diff --git a/src/vnet/adj/adj_internal.h b/src/vnet/adj/adj_internal.h index 30668625..2c123c54 100644 --- a/src/vnet/adj/adj_internal.h +++ b/src/vnet/adj/adj_internal.h @@ -17,6 +17,7 @@ #define __ADJ_INTERNAL_H__ #include +#include #include #include #include @@ -87,11 +88,14 @@ adj_get_index (ip_adjacency_t *adj) return (adj - adj_pool); } -extern void adj_nbr_update_rewrite_internal (ip_adjacency_t *adj, - ip_lookup_next_t adj_next_index, - u32 complete_next_index, - u32 next_index, - u8 *rewrite); +extern void adj_nbr_update_rewrite_internal(ip_adjacency_t *adj, + ip_lookup_next_t adj_next_index, + u32 complete_next_index, + u32 next_index, + u8 *rewrite); +extern void adj_midchain_setup(adj_index_t adj_index, + adj_midchain_fixup_t fixup, + adj_flags_t flags); extern ip_adjacency_t * adj_alloc(fib_protocol_t proto); diff --git a/src/vnet/adj/adj_mcast.c b/src/vnet/adj/adj_mcast.c index 4f678e43..755abfd4 100644 --- a/src/vnet/adj/adj_mcast.c +++ b/src/vnet/adj/adj_mcast.c @@ -13,7 +13,7 @@ * limitations under the License. */ -#include +#include #include #include #include @@ -129,6 +129,59 @@ adj_mcast_update_rewrite (adj_index_t adj_index, adj->rewrite_header.dst_mcast_mask = clib_host_to_net_u32(mask); } +/** + * adj_mcast_midchain_update_rewrite + * + * Update the adjacency's rewrite string. A NULL string implies the + * rewirte is reset (i.e. when ARP/ND etnry is gone). + * NB: the adj being updated may be handling traffic in the DP. + */ +void +adj_mcast_midchain_update_rewrite (adj_index_t adj_index, + adj_midchain_fixup_t fixup, + adj_flags_t flags, + u8 *rewrite, + u8 offset, + u32 mask) +{ + ip_adjacency_t *adj; + + ASSERT(ADJ_INDEX_INVALID != adj_index); + + adj = adj_get(adj_index); + + /* + * one time only update. since we don't support chainging the tunnel + * src,dst, this is all we need. + */ + ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_MCAST); + /* + * tunnels can always provide a rewrite. + */ + ASSERT(NULL != rewrite); + + adj_midchain_setup(adj_index, fixup, flags); + + /* + * update the adj's rewrite string and build the arc + * from the rewrite node to the interface's TX node + */ + adj_nbr_update_rewrite_internal(adj, IP_LOOKUP_NEXT_MCAST_MIDCHAIN, + adj_get_mcast_node(adj->ia_nh_proto), + vnet_tx_node_index_for_sw_interface( + vnet_get_main(), + adj->rewrite_header.sw_if_index), + rewrite); + + /* + * set the fields corresponding to the mcast IP address rewrite + * The mask must be stored in network byte order, since the packet's + * IP address will also be in network order. + */ + adj->rewrite_header.dst_mcast_offset = offset; + adj->rewrite_header.dst_mcast_mask = clib_host_to_net_u32(mask); +} + void adj_mcast_remove (fib_protocol_t proto, u32 sw_if_index) @@ -260,6 +313,24 @@ adj_mcast_interface_delete (vnet_main_t * vnm, VNET_SW_INTERFACE_ADD_DEL_FUNCTION(adj_mcast_interface_delete); +/** + * @brief Walk the multicast Adjacencies on a given interface + */ +void +adj_mcast_walk (u32 sw_if_index, + fib_protocol_t proto, + adj_walk_cb_t cb, + void *ctx) +{ + if (vec_len(adj_mcasts[proto]) > sw_if_index) + { + if (ADJ_INDEX_INVALID != adj_mcasts[proto][sw_if_index]) + { + cb(adj_mcasts[proto][sw_if_index], ctx); + } + } +} + u8* format_adj_mcast (u8* s, va_list *ap) { @@ -269,6 +340,8 @@ format_adj_mcast (u8* s, va_list *ap) s = format(s, "%U-mcast: ", format_fib_protocol, adj->ia_nh_proto); + if (adj->rewrite_header.flags & VNET_REWRITE_HAS_FEATURES) + s = format(s, "[features] "); s = format (s, "%U", format_vnet_rewrite, &adj->rewrite_header, sizeof (adj->rewrite_data), 0); @@ -276,6 +349,28 @@ format_adj_mcast (u8* s, va_list *ap) return (s); } +u8* +format_adj_mcast_midchain (u8* s, va_list *ap) +{ + index_t index = va_arg(*ap, index_t); + CLIB_UNUSED(u32 indent) = va_arg(*ap, u32); + vnet_main_t * vnm = vnet_get_main(); + ip_adjacency_t * adj = adj_get(index); + + s = format(s, "%U-mcast-midchain: ", + format_fib_protocol, adj->ia_nh_proto); + s = format (s, "%U", + format_vnet_rewrite, + vnm->vlib_main, &adj->rewrite_header, + sizeof (adj->rewrite_data), 0); + s = format (s, "\n%Ustacked-on:\n%U%U", + format_white_space, indent, + format_white_space, indent+2, + format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2); + + return (s); +} + static void adj_dpo_lock (dpo_id_t *dpo) @@ -293,6 +388,11 @@ const static dpo_vft_t adj_mcast_dpo_vft = { .dv_unlock = adj_dpo_unlock, .dv_format = format_adj_mcast, }; +const static dpo_vft_t adj_mcast_midchain_dpo_vft = { + .dv_lock = adj_dpo_lock, + .dv_unlock = adj_dpo_unlock, + .dv_format = format_adj_mcast_midchain, +}; /** * @brief The per-protocol VLIB graph nodes that are assigned to a mcast @@ -319,6 +419,31 @@ const static char* const * const adj_mcast_nodes[DPO_PROTO_NUM] = [DPO_PROTO_MPLS] = NULL, }; +/** + * @brief The per-protocol VLIB graph nodes that are assigned to a mcast + * object. + * + * this means that these graph nodes are ones from which a mcast is the + * parent object in the DPO-graph. + */ +const static char* const adj_mcast_midchain_ip4_nodes[] = +{ + "ip4-mcast-midchain", + NULL, +}; +const static char* const adj_mcast_midchain_ip6_nodes[] = +{ + "ip6-mcast-midchain", + NULL, +}; + +const static char* const * const adj_mcast_midchain_nodes[DPO_PROTO_NUM] = +{ + [DPO_PROTO_IP4] = adj_mcast_midchain_ip4_nodes, + [DPO_PROTO_IP6] = adj_mcast_midchain_ip6_nodes, + [DPO_PROTO_MPLS] = NULL, +}; + /** * @brief Return the size of the adj DB. * This is only for testing purposes so an efficient implementation is not needed @@ -349,5 +474,10 @@ adj_mcast_db_size (void) void adj_mcast_module_init (void) { - dpo_register(DPO_ADJACENCY_MCAST, &adj_mcast_dpo_vft, adj_mcast_nodes); + dpo_register(DPO_ADJACENCY_MCAST, + &adj_mcast_dpo_vft, + adj_mcast_nodes); + dpo_register(DPO_ADJACENCY_MCAST_MIDCHAIN, + &adj_mcast_midchain_dpo_vft, + adj_mcast_midchain_nodes); } diff --git a/src/vnet/adj/adj_mcast.h b/src/vnet/adj/adj_mcast.h index 40d44313..bfb0d6f6 100644 --- a/src/vnet/adj/adj_mcast.h +++ b/src/vnet/adj/adj_mcast.h @@ -26,6 +26,7 @@ #define __ADJ_MCAST_H__ #include +#include /** * @brief @@ -68,10 +69,36 @@ extern void adj_mcast_update_rewrite(adj_index_t adj_index, u8 offset, u32 mask); +/** + * @brief + * Update the rewrite string for an existing adjacecny and + * Convert the adjacency into a midchain + * + * @param + * The index of the adj to update + * + * @param + * The new rewrite + */ +extern void adj_mcast_midchain_update_rewrite(adj_index_t adj_index, + adj_midchain_fixup_t fixup, + adj_flags_t flags, + u8 *rewrite, + u8 offset, + u32 mask); +/** + * @brief Walk the multicast Adjacencies on a given interface + */ +extern void adj_mcast_walk (u32 sw_if_index, + fib_protocol_t adj_nh_proto, + adj_walk_cb_t cb, + void *ctx); + /** * @brief Format/display a mcast adjacency. */ extern u8* format_adj_mcast(u8* s, va_list *ap); +extern u8* format_adj_mcast_midchain(u8* s, va_list *ap); /** * @brief Get the sze of the mcast adj DB. Test purposes only. diff --git a/src/vnet/adj/adj_midchain.c b/src/vnet/adj/adj_midchain.c index 5756de43..a93a1c3e 100644 --- a/src/vnet/adj/adj_midchain.c +++ b/src/vnet/adj/adj_midchain.c @@ -346,7 +346,7 @@ adj_get_midchain_node (vnet_link_t link) static u8 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj) { - u8 arc = (u8) ~0; + u8 arc = (u8) ~0; switch (adj->ia_link) { case VNET_LINK_IP4: @@ -393,17 +393,14 @@ adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj) } /** - * adj_nbr_midchain_update_rewrite + * adj_midchain_setup * - * Update the adjacency's rewrite string. A NULL string implies the - * rewrite is reset (i.e. when ARP/ND etnry is gone). - * NB: the adj being updated may be handling traffic in the DP. + * Setup the adj as a mid-chain */ void -adj_nbr_midchain_update_rewrite (adj_index_t adj_index, - adj_midchain_fixup_t fixup, - adj_flags_t flags, - u8 *rewrite) +adj_midchain_setup (adj_index_t adj_index, + adj_midchain_fixup_t fixup, + adj_flags_t flags) { u32 feature_index, tx_node; ip_adjacency_t *adj; @@ -413,16 +410,6 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index, adj = adj_get(adj_index); - /* - * one time only update. since we don't support chainging the tunnel - * src,dst, this is all we need. - */ - ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP); - /* - * tunnels can always provide a rewrite. - */ - ASSERT(NULL != rewrite); - adj->sub_type.midchain.fixup_func = fixup; adj->ia_flags |= flags; @@ -447,6 +434,38 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index, dpo_stack_from_node(tx_node, &adj->sub_type.midchain.next_dpo, drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link))); +} + +/** + * adj_nbr_midchain_update_rewrite + * + * Update the adjacency's rewrite string. A NULL string implies the + * rewrite is reset (i.e. when ARP/ND etnry is gone). + * NB: the adj being updated may be handling traffic in the DP. + */ +void +adj_nbr_midchain_update_rewrite (adj_index_t adj_index, + adj_midchain_fixup_t fixup, + adj_flags_t flags, + u8 *rewrite) +{ + ip_adjacency_t *adj; + + ASSERT(ADJ_INDEX_INVALID != adj_index); + + adj = adj_get(adj_index); + + /* + * one time only update. since we don't support chainging the tunnel + * src,dst, this is all we need. + */ + ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP); + /* + * tunnels can always provide a rewrite. + */ + ASSERT(NULL != rewrite); + + adj_midchain_setup(adj_index, fixup, flags); /* * update the rewirte with the workers paused. @@ -454,7 +473,7 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index, adj_nbr_update_rewrite_internal(adj, IP_LOOKUP_NEXT_MIDCHAIN, adj_get_midchain_node(adj->ia_link), - tx_node, + adj_nbr_midchain_get_tx_node(adj), rewrite); } @@ -496,7 +515,8 @@ adj_nbr_midchain_stack (adj_index_t adj_index, adj = adj_get(adj_index); - ASSERT(IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index); + ASSERT((IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index) || + (IP_LOOKUP_NEXT_MCAST_MIDCHAIN == adj->lookup_next_index)); dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj), &adj->sub_type.midchain.next_dpo, diff --git a/src/vnet/adj/adj_nbr.c b/src/vnet/adj/adj_nbr.c index ddacb030..3d450d1f 100644 --- a/src/vnet/adj/adj_nbr.c +++ b/src/vnet/adj/adj_nbr.c @@ -195,8 +195,6 @@ adj_nbr_alloc (fib_protocol_t nh_proto, adj->ia_link = link_type; adj->ia_nh_proto = nh_proto; adj->rewrite_header.sw_if_index = sw_if_index; - memset(&adj->sub_type.midchain.next_dpo, 0, - sizeof(adj->sub_type.midchain.next_dpo)); adj_nbr_evaluate_feature (adj_get_index(adj)); return (adj); diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h index ea3ce093..ed869d1f 100644 --- a/src/vnet/buffer.h +++ b/src/vnet/buffer.h @@ -130,6 +130,9 @@ typedef struct /* Rewrite length */ u32 save_rewrite_length; + + /* MFIB RPF ID */ + u32 rpf_id; }; /* ICMP */ diff --git a/src/vnet/devices/ssvm/node.c b/src/vnet/devices/ssvm/node.c index 539b4161..b7a8db05 100644 --- a/src/vnet/devices/ssvm/node.c +++ b/src/vnet/devices/ssvm/node.c @@ -210,7 +210,7 @@ ssvm_eth_device_input (ssvm_eth_main_t * em, next0 = SSVM_ETH_INPUT_NEXT_IP4_INPUT; else if (type0 == ETHERNET_TYPE_IP6) next0 = SSVM_ETH_INPUT_NEXT_IP6_INPUT; - else if (type0 == ETHERNET_TYPE_MPLS_UNICAST) + else if (type0 == ETHERNET_TYPE_MPLS) next0 = SSVM_ETH_INPUT_NEXT_MPLS_INPUT; l3_offset0 = ((next0 == SSVM_ETH_INPUT_NEXT_IP4_INPUT || diff --git a/src/vnet/dhcp/dhcp6_proxy_node.c b/src/vnet/dhcp/dhcp6_proxy_node.c index 524cb095..de73154d 100644 --- a/src/vnet/dhcp/dhcp6_proxy_node.c +++ b/src/vnet/dhcp/dhcp6_proxy_node.c @@ -883,6 +883,7 @@ dhcp6_proxy_set_server (ip46_address_t *addr, mfib_table_entry_update(rx_fib_index, &all_dhcp_servers, MFIB_SOURCE_DHCP, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_ACCEPT_ALL_ITF); mfib_table_lock(rx_fib_index, FIB_PROTOCOL_IP6); } diff --git a/src/vnet/dpo/dpo.c b/src/vnet/dpo/dpo.c index d8e075a7..dfc2bd92 100644 --- a/src/vnet/dpo/dpo.c +++ b/src/vnet/dpo/dpo.c @@ -37,6 +37,8 @@ #include #include #include +#include +#include /** * Array of char* names for the DPO types and protos @@ -182,6 +184,12 @@ dpo_set (dpo_id_t *dpo, case IP_LOOKUP_NEXT_MIDCHAIN: dpo->dpoi_type = DPO_ADJACENCY_MIDCHAIN; break; + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: + dpo->dpoi_type = DPO_ADJACENCY_MCAST_MIDCHAIN; + break; + case IP_LOOKUP_NEXT_MCAST: + dpo->dpoi_type = DPO_ADJACENCY_MCAST; + break; default: break; } @@ -453,6 +461,8 @@ dpo_module_init (vlib_main_t * vm) lookup_dpo_module_init(); ip_null_dpo_module_init(); replicate_module_init(); + interface_dpo_module_init(); + mpls_disp_dpo_module_init(); return (NULL); } diff --git a/src/vnet/dpo/dpo.h b/src/vnet/dpo/dpo.h index 48b92d3d..5aa4e2d2 100644 --- a/src/vnet/dpo/dpo.h +++ b/src/vnet/dpo/dpo.h @@ -108,12 +108,15 @@ typedef enum dpo_type_t_ { DPO_ADJACENCY_MIDCHAIN, DPO_ADJACENCY_GLEAN, DPO_ADJACENCY_MCAST, + DPO_ADJACENCY_MCAST_MIDCHAIN, DPO_RECEIVE, DPO_LOOKUP, DPO_LISP_CP, DPO_CLASSIFY, DPO_MPLS_LABEL, + DPO_MPLS_DISPOSITION, DPO_MFIB_ENTRY, + DPO_INTERFACE, DPO_LAST, } __attribute__((packed)) dpo_type_t; @@ -129,6 +132,7 @@ typedef enum dpo_type_t_ { [DPO_ADJACENCY_MIDCHAIN] = "dpo-adjacency-midcahin", \ [DPO_ADJACENCY_GLEAN] = "dpo-glean", \ [DPO_ADJACENCY_MCAST] = "dpo-adj-mcast", \ + [DPO_ADJACENCY_MCAST_MIDCHAIN] = "dpo-adj-mcast-midchain", \ [DPO_RECEIVE] = "dpo-receive", \ [DPO_LOOKUP] = "dpo-lookup", \ [DPO_LOAD_BALANCE] = "dpo-load-balance", \ @@ -136,7 +140,9 @@ typedef enum dpo_type_t_ { [DPO_LISP_CP] = "dpo-lisp-cp", \ [DPO_CLASSIFY] = "dpo-classify", \ [DPO_MPLS_LABEL] = "dpo-mpls-label", \ - [DPO_MFIB_ENTRY] = "dpo-mfib_entry" \ + [DPO_MPLS_DISPOSITION] = "dpo-mpls-diposition", \ + [DPO_MFIB_ENTRY] = "dpo-mfib_entry", \ + [DPO_INTERFACE] = "dpo-interface" \ } /** diff --git a/src/vnet/dpo/interface_dpo.c b/src/vnet/dpo/interface_dpo.c new file mode 100644 index 00000000..50ca756f --- /dev/null +++ b/src/vnet/dpo/interface_dpo.c @@ -0,0 +1,416 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +/* + * The 'DB' of interface DPOs. + * There is only one per-interface per-protocol, so this is a per-interface + * vector + */ +static index_t *interface_dpo_db[DPO_PROTO_NUM]; + +static interface_dpo_t * +interface_dpo_alloc (void) +{ + interface_dpo_t *ido; + + pool_get(interface_dpo_pool, ido); + + return (ido); +} + +static inline interface_dpo_t * +interface_dpo_get_from_dpo (const dpo_id_t *dpo) +{ + ASSERT(DPO_INTERFACE == dpo->dpoi_type); + + return (interface_dpo_get(dpo->dpoi_index)); +} + +static inline index_t +interface_dpo_get_index (interface_dpo_t *ido) +{ + return (ido - interface_dpo_pool); +} + +static void +interface_dpo_lock (dpo_id_t *dpo) +{ + interface_dpo_t *ido; + + ido = interface_dpo_get_from_dpo(dpo); + ido->ido_locks++; +} + +static void +interface_dpo_unlock (dpo_id_t *dpo) +{ + interface_dpo_t *ido; + + ido = interface_dpo_get_from_dpo(dpo); + ido->ido_locks--; + + if (0 == ido->ido_locks) + { + interface_dpo_db[ido->ido_proto][ido->ido_sw_if_index] = + INDEX_INVALID; + pool_put(interface_dpo_pool, ido); + } +} + +/* + * interface_dpo_add_or_lock + * + * Add/create and lock a new or lock an existing for the interface DPO + * on the interface and protocol given + */ +void +interface_dpo_add_or_lock (dpo_proto_t proto, + u32 sw_if_index, + dpo_id_t *dpo) +{ + interface_dpo_t *ido; + + vec_validate_init_empty(interface_dpo_db[proto], + sw_if_index, + INDEX_INVALID); + + if (INDEX_INVALID == interface_dpo_db[proto][sw_if_index]) + { + ido = interface_dpo_alloc(); + + ido->ido_sw_if_index = sw_if_index; + ido->ido_proto = proto; + + interface_dpo_db[proto][sw_if_index] = + interface_dpo_get_index(ido); + } + else + { + ido = interface_dpo_get(interface_dpo_db[proto][sw_if_index]); + } + + dpo_set(dpo, DPO_INTERFACE, proto, interface_dpo_get_index(ido)); +} + + +static clib_error_t * +interface_dpo_interface_state_change (vnet_main_t * vnm, + u32 sw_if_index, + u32 flags) +{ + /* + */ + return (NULL); +} + +VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION( + interface_dpo_interface_state_change); + +/** + * @brief Registered callback for HW interface state changes + */ +static clib_error_t * +interface_dpo_hw_interface_state_change (vnet_main_t * vnm, + u32 hw_if_index, + u32 flags) +{ + return (NULL); +} + +VNET_HW_INTERFACE_LINK_UP_DOWN_FUNCTION( + interface_dpo_hw_interface_state_change); + +static clib_error_t * +interface_dpo_interface_delete (vnet_main_t * vnm, + u32 sw_if_index, + u32 is_add) +{ + return (NULL); +} + +VNET_SW_INTERFACE_ADD_DEL_FUNCTION( + interface_dpo_interface_delete); + +u8* +format_interface_dpo (u8* s, va_list *ap) +{ + index_t index = va_arg(*ap, index_t); + CLIB_UNUSED(u32 indent) = va_arg(*ap, u32); + vnet_main_t * vnm = vnet_get_main(); + interface_dpo_t *ido = interface_dpo_get(index); + + return (format(s, "%U-dpo: %U", + format_vnet_sw_interface_name, + vnm, + vnet_get_sw_interface(vnm, ido->ido_sw_if_index), + format_dpo_proto, ido->ido_proto)); +} + +static void +interface_dpo_mem_show (void) +{ + fib_show_memory_usage("Interface", + pool_elts(interface_dpo_pool), + pool_len(interface_dpo_pool), + sizeof(interface_dpo_t)); +} + + +const static dpo_vft_t interface_dpo_vft = { + .dv_lock = interface_dpo_lock, + .dv_unlock = interface_dpo_unlock, + .dv_format = format_interface_dpo, + .dv_mem_show = interface_dpo_mem_show, +}; + +/** + * @brief The per-protocol VLIB graph nodes that are assigned to a glean + * object. + * + * this means that these graph nodes are ones from which a glean is the + * parent object in the DPO-graph. + */ +const static char* const interface_dpo_ip4_nodes[] = +{ + "interface-dpo-ip4", + NULL, +}; +const static char* const interface_dpo_ip6_nodes[] = +{ + "interface-dpo-ip4", + NULL, +}; + +const static char* const * const interface_dpo_nodes[DPO_PROTO_NUM] = +{ + [DPO_PROTO_IP4] = interface_dpo_ip4_nodes, + [DPO_PROTO_IP6] = interface_dpo_ip6_nodes, + [DPO_PROTO_MPLS] = NULL, +}; + +void +interface_dpo_module_init (void) +{ + dpo_register(DPO_INTERFACE, + &interface_dpo_vft, + interface_dpo_nodes); +} + +/** + * @brief Interface DPO trace data + */ +typedef struct interface_dpo_trace_t_ +{ + u32 sw_if_index; +} interface_dpo_trace_t; + +typedef enum interface_dpo_next_t_ +{ + INTERFACE_DPO_DROP = 0, + INTERFACE_DPO_INPUT = 1, +} interface_dpo_next_t; + +always_inline uword +interface_dpo_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + u32 n_left_from, next_index, * from, * to_next; + u32 cpu_index = os_get_cpu_number(); + vnet_interface_main_t *im; + + im = &vnet_get_main ()->interface_main; + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next > 2) + { + const interface_dpo_t *ido0, *ido1; + u32 bi0, idoi0, bi1, idoi1; + vlib_buffer_t *b0, *b1; + + bi0 = from[0]; + to_next[0] = bi0; + bi1 = from[1]; + to_next[1] = bi1; + from += 2; + to_next += 2; + n_left_from -= 2; + n_left_to_next -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + idoi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX]; + idoi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX]; + ido0 = interface_dpo_get(idoi0); + ido1 = interface_dpo_get(idoi1); + + vnet_buffer(b0)->sw_if_index[VLIB_RX] = ido0->ido_sw_if_index; + vnet_buffer(b1)->sw_if_index[VLIB_RX] = ido1->ido_sw_if_index; + + vlib_increment_combined_counter (im->combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + cpu_index, + ido0->ido_sw_if_index, + 1, + vlib_buffer_length_in_chain (vm, b0)); + vlib_increment_combined_counter (im->combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + cpu_index, + ido1->ido_sw_if_index, + 1, + vlib_buffer_length_in_chain (vm, b1)); + + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + interface_dpo_trace_t *tr0; + + tr0 = vlib_add_trace (vm, node, b0, sizeof (*tr0)); + tr0->sw_if_index = ido0->ido_sw_if_index; + } + if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED)) + { + interface_dpo_trace_t *tr1; + + tr1 = vlib_add_trace (vm, node, b1, sizeof (*tr1)); + tr1->sw_if_index = ido1->ido_sw_if_index; + } + + vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, + n_left_to_next, bi0, bi1, + INTERFACE_DPO_INPUT, + INTERFACE_DPO_INPUT); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + const interface_dpo_t * ido0; + vlib_buffer_t * b0; + u32 bi0, idoi0; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + idoi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX]; + ido0 = interface_dpo_get(idoi0); + + /* Swap the RX interface of the packet to the one the + * interface DPR represents */ + vnet_buffer(b0)->sw_if_index[VLIB_RX] = ido0->ido_sw_if_index; + + /* Bump the interface's RX coutners */ + vlib_increment_combined_counter (im->combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + cpu_index, + ido0->ido_sw_if_index, + 1, + vlib_buffer_length_in_chain (vm, b0)); + + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + interface_dpo_trace_t *tr; + + tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); + tr->sw_if_index = ido0->ido_sw_if_index; + } + + vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, + n_left_to_next, bi0, + INTERFACE_DPO_INPUT); + } + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + return from_frame->n_vectors; +} + +static u8 * +format_interface_dpo_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + interface_dpo_trace_t * t = va_arg (*args, interface_dpo_trace_t *); + uword indent = format_get_indent (s); + s = format (s, "%U sw_if_index:%d", + format_white_space, indent, + t->sw_if_index); + return s; +} + +static uword +interface_dpo_ip4 (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return (interface_dpo_inline(vm, node, from_frame)); +} + +static uword +interface_dpo_ip6 (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return (interface_dpo_inline(vm, node, from_frame)); +} + +VLIB_REGISTER_NODE (interface_dpo_ip4_node) = { + .function = interface_dpo_ip4, + .name = "interface-dpo-ip4", + .vector_size = sizeof (u32), + .format_trace = format_interface_dpo_trace, + + .n_next_nodes = 2, + .next_nodes = { + [INTERFACE_DPO_DROP] = "ip4-drop", + [INTERFACE_DPO_INPUT] = "ip4-input", + }, +}; + +VLIB_NODE_FUNCTION_MULTIARCH (interface_dpo_ip4_node, + interface_dpo_ip4) + +VLIB_REGISTER_NODE (interface_dpo_ip6_node) = { + .function = interface_dpo_ip6, + .name = "interface-dpo-ip6", + .vector_size = sizeof (u32), + .format_trace = format_interface_dpo_trace, + + .n_next_nodes = 2, + .next_nodes = { + [INTERFACE_DPO_DROP] = "ip6-drop", + [INTERFACE_DPO_INPUT] = "ip6-input", + }, +}; + +VLIB_NODE_FUNCTION_MULTIARCH (interface_dpo_ip6_node, + interface_dpo_ip6) + diff --git a/src/vnet/dpo/interface_dpo.h b/src/vnet/dpo/interface_dpo.h new file mode 100644 index 00000000..1538dfbb --- /dev/null +++ b/src/vnet/dpo/interface_dpo.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @brief + * The data-path object representing interfaceing the packet, i.e. it's for-us + */ + +#ifndef __INTERFACE_DPO_H__ +#define __INTERFACE_DPO_H__ + +#include + +typedef struct interface_dpo_t_ +{ + /** + * The Software interface index that the packets will be given + * as the ingress/rx interface + */ + u32 ido_sw_if_index; + + /** + * next VLIB node. A '-input' node. + */ + u32 ido_next_node; + + /** + * DPO protocol that the packets will have as they 'ingress' + * on this interface + */ + dpo_proto_t ido_proto; + + /** + * number of locks. + */ + u16 ido_locks; +} interface_dpo_t; + +extern void interface_dpo_add_or_lock (dpo_proto_t proto, + u32 sw_if_index, + dpo_id_t *dpo); + +extern void interface_dpo_module_init(void); + +/** + * @brief pool of all interface DPOs + */ +interface_dpo_t *interface_dpo_pool; + +static inline interface_dpo_t * +interface_dpo_get (index_t index) +{ + return (pool_elt_at_index(interface_dpo_pool, index)); +} + +#endif diff --git a/src/vnet/dpo/lookup_dpo.c b/src/vnet/dpo/lookup_dpo.c index 97ad0a44..e5b00a79 100644 --- a/src/vnet/dpo/lookup_dpo.c +++ b/src/vnet/dpo/lookup_dpo.c @@ -21,8 +21,12 @@ #include #include #include +#include +#include +#include static const char *const lookup_input_names[] = LOOKUP_INPUTS; +static const char *const lookup_cast_names[] = LOOKUP_CASTS; /** * @brief Enumeration of the lookup subtypes @@ -31,6 +35,7 @@ typedef enum lookup_sub_type_t_ { LOOKUP_SUB_TYPE_SRC, LOOKUP_SUB_TYPE_DST, + LOOKUP_SUB_TYPE_DST_MCAST, LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE, } lookup_sub_type_t; #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1) @@ -67,6 +72,7 @@ lookup_dpo_get_index (lookup_dpo_t *lkd) static void lookup_dpo_add_or_lock_i (fib_node_index_t fib_index, dpo_proto_t proto, + lookup_cast_t cast, lookup_input_t input, lookup_table_t table_config, dpo_id_t *dpo) @@ -79,6 +85,7 @@ lookup_dpo_add_or_lock_i (fib_node_index_t fib_index, lkd->lkd_proto = proto; lkd->lkd_input = input; lkd->lkd_table = table_config; + lkd->lkd_cast = cast; /* * use the input type to select the lookup sub-type @@ -100,6 +107,10 @@ lookup_dpo_add_or_lock_i (fib_node_index_t fib_index, type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST]; break; } + if (LOOKUP_MULTICAST == cast) + { + type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST]; + } } if (0 == type) @@ -115,20 +126,29 @@ lookup_dpo_add_or_lock_i (fib_node_index_t fib_index, void lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index, dpo_proto_t proto, + lookup_cast_t cast, lookup_input_t input, lookup_table_t table_config, dpo_id_t *dpo) { if (LOOKUP_TABLE_FROM_CONFIG == table_config) { - fib_table_lock(fib_index, dpo_proto_to_fib(proto)); + if (LOOKUP_UNICAST == cast) + { + fib_table_lock(fib_index, dpo_proto_to_fib(proto)); + } + else + { + mfib_table_lock(fib_index, dpo_proto_to_fib(proto)); + } } - lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo); + lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo); } void lookup_dpo_add_or_lock_w_table_id (u32 table_id, dpo_proto_t proto, + lookup_cast_t cast, lookup_input_t input, lookup_table_t table_config, dpo_id_t *dpo) @@ -137,13 +157,22 @@ lookup_dpo_add_or_lock_w_table_id (u32 table_id, if (LOOKUP_TABLE_FROM_CONFIG == table_config) { - fib_index = - fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto), - table_id); + if (LOOKUP_UNICAST == cast) + { + fib_index = + fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto), + table_id); + } + else + { + fib_index = + mfib_table_find_or_create_and_lock(dpo_proto_to_fib(proto), + table_id); + } } ASSERT(FIB_NODE_INDEX_INVALID != fib_index); - lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo); + lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo); } u8* @@ -156,16 +185,29 @@ format_lookup_dpo (u8 *s, va_list *args) if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table) { - s = format(s, "%s lookup in interface's %U table", + s = format(s, "%s,%s lookup in interface's %U table", lookup_input_names[lkd->lkd_input], + lookup_cast_names[lkd->lkd_cast], format_dpo_proto, lkd->lkd_proto); } else { - s = format(s, "%s lookup in %U", - lookup_input_names[lkd->lkd_input], - format_fib_table_name, lkd->lkd_fib_index, - dpo_proto_to_fib(lkd->lkd_proto)); + if (LOOKUP_UNICAST == lkd->lkd_cast) + { + s = format(s, "%s,%s lookup in %U", + lookup_input_names[lkd->lkd_input], + lookup_cast_names[lkd->lkd_cast], + format_fib_table_name, lkd->lkd_fib_index, + dpo_proto_to_fib(lkd->lkd_proto)); + } + else + { + s = format(s, "%s,%s lookup in %U", + lookup_input_names[lkd->lkd_input], + lookup_cast_names[lkd->lkd_cast], + format_mfib_table_name, lkd->lkd_fib_index, + dpo_proto_to_fib(lkd->lkd_proto)); + } } return (s); } @@ -193,8 +235,16 @@ lookup_dpo_unlock (dpo_id_t *dpo) { if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table) { - fib_table_unlock(lkd->lkd_fib_index, - dpo_proto_to_fib(lkd->lkd_proto)); + if (LOOKUP_UNICAST == lkd->lkd_cast) + { + fib_table_unlock(lkd->lkd_fib_index, + dpo_proto_to_fib(lkd->lkd_proto)); + } + else + { + mfib_table_unlock(lkd->lkd_fib_index, + dpo_proto_to_fib(lkd->lkd_proto)); + } } pool_put(lookup_dpo_pool, lkd); } @@ -1069,6 +1119,123 @@ VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = { }; VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_itf_node, lookup_mpls_dst_itf) +typedef enum lookup_ip_dst_mcast_next_t_ { + LOOKUP_IP_DST_MCAST_NEXT_RPF, + LOOKUP_IP_DST_MCAST_N_NEXT, +} mfib_forward_lookup_next_t; + +always_inline uword +lookup_dpo_ip_dst_mcast_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame, + int is_v4) +{ + u32 n_left_from, next_index, * from, * to_next; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = LOOKUP_IP_DST_MCAST_NEXT_RPF; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next); + + /* while (n_left_from >= 4 && n_left_to_next >= 2) */ + /* } */ + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0, lkdi0, fib_index0, next0; + const lookup_dpo_t * lkd0; + fib_node_index_t mfei0; + vlib_buffer_t * b0; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + /* dst lookup was done by mpls lookup */ + lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX]; + lkd0 = lookup_dpo_get(lkdi0); + fib_index0 = lkd0->lkd_fib_index; + next0 = LOOKUP_IP_DST_MCAST_NEXT_RPF; + + if (is_v4) + { + ip4_header_t * ip0; + + ip0 = vlib_buffer_get_current (b0); + mfei0 = ip4_mfib_table_lookup(ip4_mfib_get(fib_index0), + &ip0->src_address, + &ip0->dst_address, + 64); + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + lookup_trace_t *tr = vlib_add_trace (vm, node, + b0, sizeof (*tr)); + tr->fib_index = fib_index0; + tr->lbi = mfei0; + tr->addr.ip4 = ip0->dst_address; + } + } + else + { + ip6_header_t * ip0; + + ip0 = vlib_buffer_get_current (b0); + mfei0 = ip6_mfib_table_lookup2(ip6_mfib_get(fib_index0), + &ip0->src_address, + &ip0->dst_address); + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + lookup_trace_t *tr = vlib_add_trace (vm, node, + b0, sizeof (*tr)); + tr->fib_index = fib_index0; + tr->lbi = mfei0; + tr->addr.ip6 = ip0->dst_address; + } + } + + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = mfei0; + + vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + return from_frame->n_vectors; +} + +always_inline uword +lookup_ip4_dst_mcast (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 1)); +} + +VLIB_REGISTER_NODE (lookup_ip4_dst_mcast_node) = { + .function = lookup_ip4_dst_mcast, + .name = "lookup-ip4-dst-mcast", + .vector_size = sizeof (u32), + + .format_trace = format_lookup_trace, + .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT, + .next_nodes = { + [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip4-mfib-forward-rpf", + }, +}; +VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_mcast_node, + lookup_ip4_dst_mcast) + static void lookup_dpo_mem_show (void) { @@ -1129,6 +1296,22 @@ const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] = [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes, }; +const static char* const lookup_dst_mcast_ip4_nodes[] = +{ + "lookup-ip4-dst-mcast", + NULL, +}; +const static char* const lookup_dst_mcast_ip6_nodes[] = +{ + "lookup-ip6-dst-mcast", + NULL, +}; +const static char* const * const lookup_dst_mcast_nodes[DPO_PROTO_NUM] = +{ + [DPO_PROTO_IP4] = lookup_dst_mcast_ip4_nodes, + [DPO_PROTO_IP6] = lookup_dst_mcast_ip6_nodes, +}; + const static char* const lookup_dst_from_interface_ip4_nodes[] = { "lookup-ip4-dst-itf", @@ -1168,6 +1351,8 @@ lookup_dpo_module_init (void) dpo_register_new_type(&lkd_vft, lookup_src_nodes); lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] = dpo_register_new_type(&lkd_vft, lookup_dst_nodes); + lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST] = + dpo_register_new_type(&lkd_vft, lookup_dst_mcast_nodes); lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] = dpo_register_new_type(&lkd_vft, lookup_dst_from_interface_nodes); } diff --git a/src/vnet/dpo/lookup_dpo.h b/src/vnet/dpo/lookup_dpo.h index ff283388..7dfd0385 100644 --- a/src/vnet/dpo/lookup_dpo.h +++ b/src/vnet/dpo/lookup_dpo.h @@ -46,6 +46,19 @@ typedef enum lookup_table_t_ { [LOOKUP_INPUT_DST_ADDR] = "table-configured", \ } +/** + * Switch to use the packet's source or destination address for lookup + */ +typedef enum lookup_cast_t_ { + LOOKUP_UNICAST, + LOOKUP_MULTICAST, +} __attribute__ ((packed)) lookup_cast_t; + +#define LOOKUP_CASTS { \ + [LOOKUP_UNICAST] = "unicast", \ + [LOOKUP_MULTICAST] = "multicast", \ +} + /** * A representation of an MPLS label for imposition in the data-path */ @@ -73,6 +86,11 @@ typedef struct lookup_dpo_t */ lookup_table_t lkd_table; + /** + * Unicast of rmulticast FIB lookup + */ + lookup_cast_t lkd_cast; + /** * Number of locks */ @@ -81,11 +99,13 @@ typedef struct lookup_dpo_t extern void lookup_dpo_add_or_lock_w_fib_index(fib_node_index_t fib_index, dpo_proto_t proto, + lookup_cast_t cast, lookup_input_t input, lookup_table_t table, dpo_id_t *dpo); extern void lookup_dpo_add_or_lock_w_table_id(u32 table_id, dpo_proto_t proto, + lookup_cast_t cast, lookup_input_t input, lookup_table_t table, dpo_id_t *dpo); diff --git a/src/vnet/dpo/mpls_disposition.c b/src/vnet/dpo/mpls_disposition.c new file mode 100644 index 00000000..5dc33fcf --- /dev/null +++ b/src/vnet/dpo/mpls_disposition.c @@ -0,0 +1,364 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +/* + * pool of all MPLS Label DPOs + */ +mpls_disp_dpo_t *mpls_disp_dpo_pool; + +static mpls_disp_dpo_t * +mpls_disp_dpo_alloc (void) +{ + mpls_disp_dpo_t *mdd; + + pool_get_aligned(mpls_disp_dpo_pool, mdd, CLIB_CACHE_LINE_BYTES); + memset(mdd, 0, sizeof(*mdd)); + + dpo_reset(&mdd->mdd_dpo); + + return (mdd); +} + +static index_t +mpls_disp_dpo_get_index (mpls_disp_dpo_t *mdd) +{ + return (mdd - mpls_disp_dpo_pool); +} + +index_t +mpls_disp_dpo_create (dpo_proto_t payload_proto, + fib_rpf_id_t rpf_id, + const dpo_id_t *dpo) +{ + mpls_disp_dpo_t *mdd; + + mdd = mpls_disp_dpo_alloc(); + + mdd->mdd_payload_proto = payload_proto; + mdd->mdd_rpf_id = rpf_id; + + dpo_stack(DPO_MPLS_DISPOSITION, + mdd->mdd_payload_proto, + &mdd->mdd_dpo, + dpo); + + return (mpls_disp_dpo_get_index(mdd)); +} + +u8* +format_mpls_disp_dpo (u8 *s, va_list *args) +{ + index_t index = va_arg (*args, index_t); + u32 indent = va_arg (*args, u32); + mpls_disp_dpo_t *mdd; + + mdd = mpls_disp_dpo_get(index); + + s = format(s, "mpls-disposition:[%d]:[%U]", + index, + format_dpo_proto, mdd->mdd_payload_proto); + + s = format(s, "\n%U", format_white_space, indent); + s = format(s, "%U", format_dpo_id, &mdd->mdd_dpo, indent+2); + + return (s); +} + +static void +mpls_disp_dpo_lock (dpo_id_t *dpo) +{ + mpls_disp_dpo_t *mdd; + + mdd = mpls_disp_dpo_get(dpo->dpoi_index); + + mdd->mdd_locks++; +} + +static void +mpls_disp_dpo_unlock (dpo_id_t *dpo) +{ + mpls_disp_dpo_t *mdd; + + mdd = mpls_disp_dpo_get(dpo->dpoi_index); + + mdd->mdd_locks--; + + if (0 == mdd->mdd_locks) + { + dpo_reset(&mdd->mdd_dpo); + pool_put(mpls_disp_dpo_pool, mdd); + } +} + +/** + * @brief A struct to hold tracing information for the MPLS label disposition + * node. + */ +typedef struct mpls_label_disposition_trace_t_ +{ + index_t mdd; +} mpls_label_disposition_trace_t; + +always_inline uword +mpls_label_disposition_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame, + u8 payload_is_ip4, + u8 payload_is_ip6) +{ + u32 n_left_from, next_index, * from, * to_next; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + mpls_disp_dpo_t *mdd0, *mdd1; + u32 bi0, mddi0, bi1, mddi1; + vlib_buffer_t * b0, *b1; + u32 next0, next1; + + bi0 = to_next[0] = from[0]; + bi1 = to_next[1] = from[1]; + + /* Prefetch next iteration. */ + { + vlib_buffer_t * p2, * p3; + + p2 = vlib_get_buffer (vm, from[2]); + p3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (p2, STORE); + vlib_prefetch_buffer_header (p3, STORE); + + CLIB_PREFETCH (p2->data, sizeof (ip6_header_t), STORE); + CLIB_PREFETCH (p3->data, sizeof (ip6_header_t), STORE); + } + + from += 2; + to_next += 2; + n_left_from -= 2; + n_left_to_next -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + /* dst lookup was done by ip4 lookup */ + mddi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX]; + mddi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX]; + mdd0 = mpls_disp_dpo_get(mddi0); + mdd1 = mpls_disp_dpo_get(mddi1); + + if (payload_is_ip4) + { + /* + * decrement the TTL on ingress to the LSP + */ + } + else if (payload_is_ip6) + { + /* + * decrement the TTL on ingress to the LSP + */ + } + + next0 = mdd0->mdd_dpo.dpoi_next_node; + next1 = mdd1->mdd_dpo.dpoi_next_node; + vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mdd0->mdd_dpo.dpoi_index; + vnet_buffer(b1)->ip.adj_index[VLIB_TX] = mdd1->mdd_dpo.dpoi_index; + vnet_buffer(b0)->ip.rpf_id = mdd0->mdd_rpf_id; + vnet_buffer(b1)->ip.rpf_id = mdd1->mdd_rpf_id; + + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_label_disposition_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + + tr->mdd = mddi0; + } + if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_label_disposition_trace_t *tr = + vlib_add_trace (vm, node, b1, sizeof (*tr)); + tr->mdd = mddi1; + } + + vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, + n_left_to_next, + bi0, bi1, next0, next1); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + mpls_disp_dpo_t *mdd0; + vlib_buffer_t * b0; + u32 bi0, mddi0; + u32 next0; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + /* dst lookup was done by ip4 lookup */ + mddi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX]; + mdd0 = mpls_disp_dpo_get(mddi0); + + if (payload_is_ip4) + { + /* + * decrement the TTL on ingress to the LSP + */ + } + else if (payload_is_ip6) + { + /* + * decrement the TTL on ingress to the LSP + */ + } + else + { + } + + next0 = mdd0->mdd_dpo.dpoi_next_node; + vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mdd0->mdd_dpo.dpoi_index; + vnet_buffer(b0)->ip.rpf_id = mdd0->mdd_rpf_id; + + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_label_disposition_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + tr->mdd = mddi0; + } + + vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + return from_frame->n_vectors; +} + +static u8 * +format_mpls_label_disposition_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + CLIB_UNUSED (mpls_label_disposition_trace_t * t); + + t = va_arg (*args, mpls_label_disposition_trace_t *); + + s = format(s, "disp:%d", t->mdd); + return (s); +} + +static uword +ip4_mpls_label_disposition (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return (mpls_label_disposition_inline(vm, node, frame, 1, 0)); +} + +VLIB_REGISTER_NODE (ip4_mpls_label_disposition_node) = { + .function = ip4_mpls_label_disposition, + .name = "ip4-mpls-label-disposition", + .vector_size = sizeof (u32), + + .format_trace = format_mpls_label_disposition_trace, + .n_next_nodes = 1, + .next_nodes = { + [0] = "ip4-drop", + } +}; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_mpls_label_disposition_node, + ip4_mpls_label_disposition) + +static uword +ip6_mpls_label_disposition (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return (mpls_label_disposition_inline(vm, node, frame, 0, 1)); +} + +VLIB_REGISTER_NODE (ip6_mpls_label_disposition_node) = { + .function = ip6_mpls_label_disposition, + .name = "ip6-mpls-label-disposition", + .vector_size = sizeof (u32), + + .format_trace = format_mpls_label_disposition_trace, + .n_next_nodes = 1, + .next_nodes = { + [0] = "ip6-drop", + } +}; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_mpls_label_disposition_node, + ip6_mpls_label_disposition) + +static void +mpls_disp_dpo_mem_show (void) +{ + fib_show_memory_usage("MPLS label", + pool_elts(mpls_disp_dpo_pool), + pool_len(mpls_disp_dpo_pool), + sizeof(mpls_disp_dpo_t)); +} + +const static dpo_vft_t mdd_vft = { + .dv_lock = mpls_disp_dpo_lock, + .dv_unlock = mpls_disp_dpo_unlock, + .dv_format = format_mpls_disp_dpo, + .dv_mem_show = mpls_disp_dpo_mem_show, +}; + +const static char* const mpls_label_disp_ip4_nodes[] = +{ + "ip4-mpls-label-disposition", + NULL, +}; +const static char* const mpls_label_disp_ip6_nodes[] = +{ + "ip6-mpls-label-disposition", + NULL, +}; +const static char* const * const mpls_label_disp_nodes[DPO_PROTO_NUM] = +{ + [DPO_PROTO_IP4] = mpls_label_disp_ip4_nodes, + [DPO_PROTO_IP6] = mpls_label_disp_ip6_nodes, +}; + + +void +mpls_disp_dpo_module_init (void) +{ + dpo_register(DPO_MPLS_DISPOSITION, &mdd_vft, mpls_label_disp_nodes); +} diff --git a/src/vnet/dpo/mpls_disposition.h b/src/vnet/dpo/mpls_disposition.h new file mode 100644 index 00000000..9c015083 --- /dev/null +++ b/src/vnet/dpo/mpls_disposition.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MPLS_DISP_DPO_H__ +#define __MPLS_DISP_DPO_H__ + +#include +#include +#include +#include + +/** + * A representation of an MPLS label for imposition in the data-path + */ +typedef struct mpls_disp_dpo_t +{ + /** + * Next DPO in the graph + */ + dpo_id_t mdd_dpo; + + /** + * The protocol of the payload/packets that are being encapped + */ + dpo_proto_t mdd_payload_proto; + + /** + * RPF-ID (if this is an mcast disposition) + */ + fib_rpf_id_t mdd_rpf_id; + + /** + * Number of locks/users of the label + */ + u16 mdd_locks; +} mpls_disp_dpo_t; + +/** + * @brief Assert that the MPLS label object is less than a cache line in size. + * Should this get any bigger then we will need to reconsider how many labels + * can be pushed in one object. + */ +_Static_assert((sizeof(mpls_disp_dpo_t) <= CLIB_CACHE_LINE_BYTES), + "MPLS Disposition DPO is larger than one cache line."); + +/** + * @brief Create an MPLS label object + * + * @param payload_proto The ptocool of the payload packets that will + * be imposed with this label header. + * @param dpo The parent of the created MPLS label object + */ +extern index_t mpls_disp_dpo_create(dpo_proto_t payload_proto, + fib_rpf_id_t rpf_id, + const dpo_id_t *dpo); + +extern u8* format_mpls_disp_dpo(u8 *s, va_list *args); + + +/* + * Encapsulation violation for fast data-path access + */ +extern mpls_disp_dpo_t *mpls_disp_dpo_pool; + +static inline mpls_disp_dpo_t * +mpls_disp_dpo_get (index_t index) +{ + return (pool_elt_at_index(mpls_disp_dpo_pool, index)); +} + +extern void mpls_disp_dpo_module_init(void); + +#endif diff --git a/src/vnet/dpo/mpls_label_dpo.c b/src/vnet/dpo/mpls_label_dpo.c index be9b2850..4d84b900 100644 --- a/src/vnet/dpo/mpls_label_dpo.c +++ b/src/vnet/dpo/mpls_label_dpo.c @@ -562,7 +562,7 @@ VLIB_REGISTER_NODE (mpls_label_imposition_node) = { .format_trace = format_mpls_label_imposition_trace, .n_next_nodes = 1, .next_nodes = { - [0] = "error-drop", + [0] = "mpls-drop", } }; VLIB_NODE_FUNCTION_MULTIARCH (mpls_label_imposition_node, @@ -584,7 +584,7 @@ VLIB_REGISTER_NODE (ip4_mpls_label_imposition_node) = { .format_trace = format_mpls_label_imposition_trace, .n_next_nodes = 1, .next_nodes = { - [0] = "error-drop", + [0] = "ip4-drop", } }; VLIB_NODE_FUNCTION_MULTIARCH (ip4_mpls_label_imposition_node, @@ -606,7 +606,7 @@ VLIB_REGISTER_NODE (ip6_mpls_label_imposition_node) = { .format_trace = format_mpls_label_imposition_trace, .n_next_nodes = 1, .next_nodes = { - [0] = "error-drop", + [0] = "ip6-drop", } }; VLIB_NODE_FUNCTION_MULTIARCH (ip6_mpls_label_imposition_node, diff --git a/src/vnet/dpo/replicate_dpo.c b/src/vnet/dpo/replicate_dpo.c index e25ceae9..9fdb9a05 100644 --- a/src/vnet/dpo/replicate_dpo.c +++ b/src/vnet/dpo/replicate_dpo.c @@ -17,6 +17,7 @@ #include #include #include +#include #undef REP_DEBUG @@ -106,6 +107,7 @@ replicate_format (index_t repi, dpo_id_t *buckets; u32 i; + repi &= ~MPLS_IS_REPLICATE; rep = replicate_get(repi); vlib_get_combined_counter(&(replicate_main.repm_counters), repi, &to); buckets = replicate_get_buckets(rep); @@ -187,6 +189,7 @@ replicate_set_bucket (index_t repi, replicate_t *rep; dpo_id_t *buckets; + repi &= ~MPLS_IS_REPLICATE; rep = replicate_get(repi); buckets = replicate_get_buckets(rep); @@ -199,11 +202,13 @@ int replicate_is_drop (const dpo_id_t *dpo) { replicate_t *rep; + index_t repi; if (DPO_REPLICATE != dpo->dpoi_type) return (0); - rep = replicate_get(dpo->dpoi_index); + repi = dpo->dpoi_index & ~MPLS_IS_REPLICATE; + rep = replicate_get(repi); if (1 == rep->rep_n_buckets) { @@ -218,6 +223,7 @@ replicate_get_bucket (index_t repi, { replicate_t *rep; + repi &= ~MPLS_IS_REPLICATE; rep = replicate_get(repi); return (replicate_get_bucket_i(rep, bucket)); @@ -288,9 +294,11 @@ replicate_multipath_update (const dpo_id_t *dpo, dpo_id_t *tmp_dpo; u32 ii, n_buckets; replicate_t *rep; + index_t repi; ASSERT(DPO_REPLICATE == dpo->dpoi_type); - rep = replicate_get(dpo->dpoi_index); + repi = dpo->dpoi_index & ~MPLS_IS_REPLICATE; + rep = replicate_get(repi); nhs = replicate_multipath_next_hop_fixup(next_hops, rep->rep_proto); n_buckets = vec_len(nhs); @@ -718,7 +726,7 @@ format_replicate_trace (u8 * s, va_list * args) s = format (s, "replicate: %d via %U", t->rep_index, - format_dpo_id, &t->dpo); + format_dpo_id, &t->dpo, 0); return s; } @@ -731,7 +739,7 @@ ip4_replicate (vlib_main_t * vm, } /** - * @brief + * @brief IP4 replication node */ VLIB_REGISTER_NODE (ip4_replicate_node) = { .function = ip4_replicate, @@ -744,7 +752,7 @@ VLIB_REGISTER_NODE (ip4_replicate_node) = { .format_trace = format_replicate_trace, .n_next_nodes = 1, .next_nodes = { - [0] = "error-drop", + [0] = "ip4-drop", }, }; @@ -757,7 +765,7 @@ ip6_replicate (vlib_main_t * vm, } /** - * @brief + * @brief IPv6 replication node */ VLIB_REGISTER_NODE (ip6_replicate_node) = { .function = ip6_replicate, @@ -770,7 +778,33 @@ VLIB_REGISTER_NODE (ip6_replicate_node) = { .format_trace = format_replicate_trace, .n_next_nodes = 1, .next_nodes = { - [0] = "error-drop", + [0] = "ip6-drop", + }, +}; + +static uword +mpls_replicate (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return (replicate_inline (vm, node, frame)); +} + +/** + * @brief MPLS replication node + */ +VLIB_REGISTER_NODE (mpls_replicate_node) = { + .function = mpls_replicate, + .name = "mpls-replicate", + .vector_size = sizeof (u32), + + .n_errors = ARRAY_LEN(replicate_dpo_error_strings), + .error_strings = replicate_dpo_error_strings, + + .format_trace = format_replicate_trace, + .n_next_nodes = 1, + .next_nodes = { + [0] = "mpls-drop", }, }; diff --git a/src/vnet/dpo/replicate_dpo.h b/src/vnet/dpo/replicate_dpo.h index 77273015..7383184a 100644 --- a/src/vnet/dpo/replicate_dpo.h +++ b/src/vnet/dpo/replicate_dpo.h @@ -25,6 +25,7 @@ #include #include #include +#include /** * replicate main @@ -119,6 +120,7 @@ extern replicate_t *replicate_pool; static inline replicate_t* replicate_get (index_t repi) { + repi &= ~MPLS_IS_REPLICATE; return (pool_elt_at_index(replicate_pool, repi)); } diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c index c74a097e..dd509193 100644 --- a/src/vnet/ethernet/arp.c +++ b/src/vnet/ethernet/arp.c @@ -507,6 +507,7 @@ arp_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai) case IP_LOOKUP_NEXT_PUNT: case IP_LOOKUP_NEXT_LOCAL: case IP_LOOKUP_NEXT_REWRITE: + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: case IP_LOOKUP_NEXT_MIDCHAIN: case IP_LOOKUP_NEXT_ICMP_ERROR: case IP_LOOKUP_N_NEXT: diff --git a/src/vnet/ethernet/interface.c b/src/vnet/ethernet/interface.c index 335e3f9f..9ac30bc6 100644 --- a/src/vnet/ethernet/interface.c +++ b/src/vnet/ethernet/interface.c @@ -115,7 +115,7 @@ ethernet_build_rewrite (vnet_main_t * vnm, #define _(a,b) case VNET_LINK_##a: type = ETHERNET_TYPE_##b; break _(IP4, IP4); _(IP6, IP6); - _(MPLS, MPLS_UNICAST); + _(MPLS, MPLS); _(ARP, ARP); #undef _ default: diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index f7787ed2..5305012f 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -249,7 +249,7 @@ determine_next_node (ethernet_main_t * em, { *next0 = em->l3_next.input_next_ip6; } - else if (type0 == ETHERNET_TYPE_MPLS_UNICAST) + else if (type0 == ETHERNET_TYPE_MPLS) { *next0 = em->l3_next.input_next_mpls; @@ -1252,7 +1252,7 @@ next_by_ethertype_register (next_by_ethertype_t * l3_next, { l3_next->input_next_ip6 = next_index; } - else if (ethertype == ETHERNET_TYPE_MPLS_UNICAST) + else if (ethertype == ETHERNET_TYPE_MPLS) { l3_next->input_next_mpls = next_index; } diff --git a/src/vnet/ethernet/types.def b/src/vnet/ethernet/types.def index 643f3152..7dab8ee1 100644 --- a/src/vnet/ethernet/types.def +++ b/src/vnet/ethernet/types.def @@ -85,8 +85,8 @@ ethernet_type (0x876D, SECURE_DATA) ethernet_type (0x8808, MAC_CONTROL) ethernet_type (0x8809, SLOW_PROTOCOLS) ethernet_type (0x880B, PPP) -ethernet_type (0x8847, MPLS_UNICAST) -ethernet_type (0x8848, MPLS_MULTICAST) +ethernet_type (0x8847, MPLS) +ethernet_type (0x8848, MPLS_UPSTREAM_ASSIGNED) ethernet_type (0x8863, PPPOE_DISCOVERY) ethernet_type (0x8864, PPPOE_SESSION) ethernet_type (0x886D, INTEL_ANS) diff --git a/src/vnet/fib/fib_api.h b/src/vnet/fib/fib_api.h index f8275317..10d0cb58 100644 --- a/src/vnet/fib/fib_api.h +++ b/src/vnet/fib/fib_api.h @@ -24,6 +24,7 @@ add_del_route_check (fib_protocol_t table_proto, fib_protocol_t next_hop_table_proto, u32 next_hop_table_id, u8 create_missing_tables, + u8 is_rpf_id, u32 * fib_index, u32 * next_hop_fib_index); int @@ -33,10 +34,13 @@ add_del_route_t_handler (u8 is_multipath, u8 is_unreach, u8 is_prohibit, u8 is_local, + u8 is_multicast, u8 is_classify, u32 classify_table_index, u8 is_resolve_host, u8 is_resolve_attached, + u8 is_interface_rx, + u8 is_rpf_id, u32 fib_index, const fib_prefix_t * prefix, u8 next_hop_proto_is_ip4, diff --git a/src/vnet/fib/fib_entry.c b/src/vnet/fib/fib_entry.c index dac1fce9..6f811aa1 100644 --- a/src/vnet/fib/fib_entry.c +++ b/src/vnet/fib/fib_entry.c @@ -75,13 +75,7 @@ fib_entry_get_default_chain_type (const fib_entry_t *fib_entry) return (FIB_FORW_CHAIN_TYPE_UNICAST_IP6); case FIB_PROTOCOL_MPLS: if (MPLS_EOS == fib_entry->fe_prefix.fp_eos) - /* - * If the entry being asked is a eos-MPLS label entry, - * then use the payload-protocol field, that we stashed there - * for just this purpose - */ - return (fib_forw_chain_type_from_dpo_proto( - fib_entry->fe_prefix.fp_payload_proto)); + return (FIB_FORW_CHAIN_TYPE_MPLS_EOS); else return (FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS); } @@ -370,6 +364,35 @@ fib_entry_contribute_urpf (fib_node_index_t entry_index, return (fib_path_list_contribute_urpf(fib_entry->fe_parent, urpf)); } +/* + * If the client is request a chain for multicast forwarding then swap + * the chain type to one that can provide such transport. + */ +static fib_forward_chain_type_t +fib_entry_chain_type_mcast_to_ucast (fib_forward_chain_type_t fct) +{ + switch (fct) + { + case FIB_FORW_CHAIN_TYPE_MCAST_IP4: + case FIB_FORW_CHAIN_TYPE_MCAST_IP6: + /* + * we can only transport IP multicast packets if there is an + * LSP. + */ + fct = FIB_FORW_CHAIN_TYPE_MPLS_EOS; + break; + case FIB_FORW_CHAIN_TYPE_MPLS_EOS: + case FIB_FORW_CHAIN_TYPE_UNICAST_IP4: + case FIB_FORW_CHAIN_TYPE_UNICAST_IP6: + case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS: + case FIB_FORW_CHAIN_TYPE_ETHERNET: + case FIB_FORW_CHAIN_TYPE_NSH: + break; + } + + return (fct); +} + /* * fib_entry_contribute_forwarding * @@ -385,6 +408,11 @@ fib_entry_contribute_forwarding (fib_node_index_t fib_entry_index, fib_entry = fib_entry_get(fib_entry_index); + /* + * mfib children ask for mcast chains. fix these to the appropriate ucast types. + */ + fct = fib_entry_chain_type_mcast_to_ucast(fct); + if (fct == fib_entry_get_default_chain_type(fib_entry)) { dpo_copy(dpo, &fib_entry->fe_lb); @@ -414,6 +442,11 @@ fib_entry_contribute_forwarding (fib_node_index_t fib_entry_index, dpo_copy(dpo, &fed->fd_dpo); } + /* + * don't allow the special index indicating replicate.vs.load-balance + * to escape to the clients + */ + dpo->dpoi_index &= ~MPLS_IS_REPLICATE; } const dpo_id_t * diff --git a/src/vnet/fib/fib_entry.h b/src/vnet/fib/fib_entry.h index a3f75e60..b17a0b64 100644 --- a/src/vnet/fib/fib_entry.h +++ b/src/vnet/fib/fib_entry.h @@ -192,6 +192,11 @@ typedef enum fib_entry_attribute_t_ { * The prefix/address is local to this device */ FIB_ENTRY_ATTRIBUTE_LOCAL, + /** + * The prefix/address is a multicast prefix. + * this aplies only to MPLS. IP multicast is handled by mfib + */ + FIB_ENTRY_ATTRIBUTE_MULTICAST, /** * The prefix/address exempted from loose uRPF check * To be used with caution @@ -200,7 +205,7 @@ typedef enum fib_entry_attribute_t_ { /** * Marker. add new entries before this one. */ - FIB_ENTRY_ATTRIBUTE_LAST = FIB_ENTRY_ATTRIBUTE_URPF_EXEMPT, + FIB_ENTRY_ATTRIBUTE_LAST = FIB_ENTRY_ATTRIBUTE_MULTICAST, } fib_entry_attribute_t; /** @@ -215,7 +220,8 @@ typedef enum fib_entry_attribute_t_ { [FIB_ENTRY_ATTRIBUTE_DROP] = "drop", \ [FIB_ENTRY_ATTRIBUTE_EXCLUSIVE] = "exclusive", \ [FIB_ENTRY_ATTRIBUTE_LOCAL] = "local", \ - [FIB_ENTRY_ATTRIBUTE_URPF_EXEMPT] = "uRPF-exempt" \ + [FIB_ENTRY_ATTRIBUTE_URPF_EXEMPT] = "uRPF-exempt", \ + [FIB_ENTRY_ATTRIBUTE_MULTICAST] = "multicast", \ } #define FOR_EACH_FIB_ATTRIBUTE(_item) \ @@ -232,6 +238,7 @@ typedef enum fib_entry_flag_t_ { FIB_ENTRY_FLAG_LOCAL = (1 << FIB_ENTRY_ATTRIBUTE_LOCAL), FIB_ENTRY_FLAG_IMPORT = (1 << FIB_ENTRY_ATTRIBUTE_IMPORT), FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT = (1 << FIB_ENTRY_ATTRIBUTE_URPF_EXEMPT), + FIB_ENTRY_FLAG_MULTICAST = (1 << FIB_ENTRY_ATTRIBUTE_MULTICAST), } __attribute__((packed)) fib_entry_flag_t; /** @@ -396,7 +403,7 @@ typedef struct fib_entry_t_ { * paint the header straight on without the need to check the packet * type to derive the EOS bit value. */ - dpo_id_t fe_lb; // [FIB_FORW_CHAIN_MPLS_NUM]; + dpo_id_t fe_lb; /** * Vector of source infos. * Most entries will only have 1 source. So we optimise for memory usage, diff --git a/src/vnet/fib/fib_entry_src.c b/src/vnet/fib/fib_entry_src.c index aa1d5a24..a700282e 100644 --- a/src/vnet/fib/fib_entry_src.c +++ b/src/vnet/fib/fib_entry_src.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -229,8 +230,6 @@ fib_forward_chain_type_t fib_entry_chain_type_fixup (const fib_entry_t *entry, fib_forward_chain_type_t fct) { - ASSERT(FIB_FORW_CHAIN_TYPE_MPLS_EOS == fct); - /* * The EOS chain is a tricky since one cannot know the adjacency * to link to without knowing what the packets payload protocol @@ -238,6 +237,11 @@ fib_entry_chain_type_fixup (const fib_entry_t *entry, */ fib_forward_chain_type_t dfct; + if (FIB_FORW_CHAIN_TYPE_MPLS_EOS != fct) + { + return (fct); + } + dfct = fib_entry_get_default_chain_type(entry); if (FIB_FORW_CHAIN_TYPE_MPLS_EOS == dfct) @@ -303,7 +307,12 @@ fib_entry_src_collect_forwarding (fib_node_index_t pl_index, * found a matching extension. stack it to obtain the forwarding * info for this path. */ - ctx->next_hops = fib_path_ext_stack(path_ext, ctx->fib_entry, ctx->fct, ctx->next_hops); + ctx->next_hops = + fib_path_ext_stack(path_ext, + ctx->fct, + fib_entry_chain_type_fixup(ctx->fib_entry, + ctx->fct), + ctx->next_hops); } else { @@ -355,6 +364,9 @@ fib_entry_src_collect_forwarding (fib_node_index_t pl_index, fib_entry_chain_type_fixup(ctx->fib_entry, ctx->fct), &nh->path_dpo); + fib_path_stack_mpls_disp(path_index, + ctx->fib_entry->fe_prefix.fp_payload_proto, + &nh->path_dpo); break; } @@ -424,50 +436,70 @@ fib_entry_src_mk_lb (fib_entry_t *fib_entry, /* * first time create */ - flow_hash_config_t fhc; - - fhc = fib_table_get_flow_hash_config(fib_entry->fe_fib_index, - dpo_proto_to_fib(lb_proto)); - dpo_set(dpo_lb, - DPO_LOAD_BALANCE, - lb_proto, - load_balance_create(0, lb_proto, fhc)); + if (esrc->fes_entry_flags & FIB_ENTRY_FLAG_MULTICAST) + { + dpo_set(dpo_lb, + DPO_REPLICATE, + lb_proto, + MPLS_IS_REPLICATE | replicate_create(0, lb_proto)); + } + else + { + flow_hash_config_t fhc; + + fhc = fib_table_get_flow_hash_config(fib_entry->fe_fib_index, + dpo_proto_to_fib(lb_proto)); + dpo_set(dpo_lb, + DPO_LOAD_BALANCE, + lb_proto, + load_balance_create(0, lb_proto, fhc)); + } } - load_balance_multipath_update(dpo_lb, - ctx.next_hops, - fib_entry_calc_lb_flags(&ctx)); - vec_free(ctx.next_hops); - - /* - * if this entry is sourced by the uRPF-exempt source then we - * append the always present local0 interface (index 0) to the - * uRPF list so it is not empty. that way packets pass the loose check. - */ - index_t ui = fib_path_list_get_urpf(esrc->fes_pl); - - if ((fib_entry_is_sourced(fib_entry_get_index(fib_entry), - FIB_SOURCE_URPF_EXEMPT) || - (esrc->fes_entry_flags & FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT))&& - (0 == fib_urpf_check_size(ui))) + if (esrc->fes_entry_flags & FIB_ENTRY_FLAG_MULTICAST) { - /* - * The uRPF list we get from the path-list is shared by all - * other users of the list, but the uRPF exemption applies - * only to this prefix. So we need our own list. - */ - ui = fib_urpf_list_alloc_and_lock(); - fib_urpf_list_append(ui, 0); - fib_urpf_list_bake(ui); - load_balance_set_urpf(dpo_lb->dpoi_index, ui); - fib_urpf_list_unlock(ui); + /* + * MPLS multicast + */ + replicate_multipath_update(dpo_lb, ctx.next_hops); } else { - load_balance_set_urpf(dpo_lb->dpoi_index, ui); + load_balance_multipath_update(dpo_lb, + ctx.next_hops, + fib_entry_calc_lb_flags(&ctx)); + vec_free(ctx.next_hops); + + /* + * if this entry is sourced by the uRPF-exempt source then we + * append the always present local0 interface (index 0) to the + * uRPF list so it is not empty. that way packets pass the loose check. + */ + index_t ui = fib_path_list_get_urpf(esrc->fes_pl); + + if ((fib_entry_is_sourced(fib_entry_get_index(fib_entry), + FIB_SOURCE_URPF_EXEMPT) || + (esrc->fes_entry_flags & FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT))&& + (0 == fib_urpf_check_size(ui))) + { + /* + * The uRPF list we get from the path-list is shared by all + * other users of the list, but the uRPF exemption applies + * only to this prefix. So we need our own list. + */ + ui = fib_urpf_list_alloc_and_lock(); + fib_urpf_list_append(ui, 0); + fib_urpf_list_bake(ui); + load_balance_set_urpf(dpo_lb->dpoi_index, ui); + fib_urpf_list_unlock(ui); + } + else + { + load_balance_set_urpf(dpo_lb->dpoi_index, ui); + } + load_balance_set_fib_entry_flags(dpo_lb->dpoi_index, + fib_entry_get_flags_i(fib_entry)); } - load_balance_set_fib_entry_flags(dpo_lb->dpoi_index, - fib_entry_get_flags_i(fib_entry)); } void @@ -887,21 +919,6 @@ fib_entry_src_action_remove (fib_entry_t *fib_entry, return (sflags); } -static inline int -fib_route_recurses_via_self (const fib_prefix_t *prefix, - const fib_route_path_t *rpath) -{ - /* - * not all zeros next hop && - * is recursive path && - * nexthop is same as the route's address - */ - return ((!ip46_address_is_zero(&rpath->frp_addr)) && - (~0 == rpath->frp_sw_if_index) && - (0 == ip46_address_cmp(&rpath->frp_addr, &prefix->fp_addr))); - -} - /* * fib_route_attached_cross_table * @@ -962,14 +979,14 @@ fib_entry_src_flags_2_path_list_flags (fib_entry_flag_t eflags) { plf |= FIB_PATH_LIST_FLAG_DROP; } - if (eflags & FIB_ENTRY_FLAG_LOCAL) - { - plf |= FIB_PATH_LIST_FLAG_LOCAL; - } if (eflags & FIB_ENTRY_FLAG_EXCLUSIVE) { plf |= FIB_PATH_LIST_FLAG_EXCLUSIVE; } + if (eflags & FIB_ENTRY_FLAG_LOCAL) + { + plf |= FIB_PATH_LIST_FLAG_LOCAL; + } return (plf); } @@ -980,25 +997,6 @@ fib_entry_flags_update (const fib_entry_t *fib_entry, fib_path_list_flags_t *pl_flags, fib_entry_src_t *esrc) { - /* - * don't allow the addition of a recursive looped path for prefix - * via itself. - */ - if (fib_route_recurses_via_self(&fib_entry->fe_prefix, rpath)) - { - /* - * force the install of a drop path-list. - * we want the entry to have some path-list, mainly so - * the dodgy path can be rmeoved when the source stops playing - * silly buggers. - */ - *pl_flags |= FIB_PATH_LIST_FLAG_DROP; - } - else - { - *pl_flags &= ~FIB_PATH_LIST_FLAG_DROP; - } - if ((esrc->fes_src == FIB_SOURCE_API) || (esrc->fes_src == FIB_SOURCE_CLI)) { diff --git a/src/vnet/fib/fib_internal.h b/src/vnet/fib/fib_internal.h index 2d980bcc..8abc0e07 100644 --- a/src/vnet/fib/fib_internal.h +++ b/src/vnet/fib/fib_internal.h @@ -25,6 +25,7 @@ #undef FIB_DEBUG extern void fib_prefix_from_mpls_label(mpls_label_t label, + mpls_eos_bit_t eos, fib_prefix_t *prf); extern int fib_route_path_cmp(const fib_route_path_t *rpath1, diff --git a/src/vnet/fib/fib_path.c b/src/vnet/fib/fib_path.c index 6b202a97..f81f4170 100644 --- a/src/vnet/fib/fib_path.c +++ b/src/vnet/fib/fib_path.c @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include @@ -66,6 +68,10 @@ typedef enum fib_path_type_t_ { * deag. Link to a lookup adj in the next table */ FIB_PATH_TYPE_DEAG, + /** + * interface receive. + */ + FIB_PATH_TYPE_INTF_RX, /** * receive. it's for-us. */ @@ -88,6 +94,7 @@ typedef enum fib_path_type_t_ { [FIB_PATH_TYPE_SPECIAL] = "special", \ [FIB_PATH_TYPE_EXCLUSIVE] = "exclusive", \ [FIB_PATH_TYPE_DEAG] = "deag", \ + [FIB_PATH_TYPE_INTF_RX] = "intf-rx", \ [FIB_PATH_TYPE_RECEIVE] = "receive", \ } @@ -220,10 +227,16 @@ typedef struct fib_path_t_ { * The next-hop */ ip46_address_t fp_ip; - /** - * The local label to resolve through. - */ - mpls_label_t fp_local_label; + struct { + /** + * The local label to resolve through. + */ + mpls_label_t fp_local_label; + /** + * The EOS bit of the resolving label + */ + mpls_eos_bit_t fp_eos; + }; } fp_nh; /** * The FIB table index in which to find the next-hop. @@ -254,6 +267,10 @@ typedef struct fib_path_t_ { * The FIB index in which to perfom the next lookup */ fib_node_index_t fp_tbl_id; + /** + * The RPF-ID to tag the packets with + */ + fib_rpf_id_t fp_rpf_id; } deag; struct { } special; @@ -273,6 +290,12 @@ typedef struct fib_path_t_ { */ ip46_address_t fp_addr; } receive; + struct { + /** + * The interface on which the packets will be input. + */ + u32 fp_interface; + } intf_rx; }; STRUCT_MARK(path_hash_end); @@ -444,9 +467,11 @@ format_fib_path (u8 * s, va_list * args) case FIB_PATH_TYPE_RECURSIVE: if (FIB_PROTOCOL_MPLS == path->fp_nh_proto) { - s = format (s, "via %U", + s = format (s, "via %U %U", format_mpls_unicast_label, - path->recursive.fp_nh.fp_local_label); + path->recursive.fp_nh.fp_local_label, + format_mpls_eos_bit, + path->recursive.fp_nh.fp_eos); } else { @@ -465,6 +490,7 @@ format_fib_path (u8 * s, va_list * args) break; case FIB_PATH_TYPE_RECEIVE: + case FIB_PATH_TYPE_INTF_RX: case FIB_PATH_TYPE_SPECIAL: case FIB_PATH_TYPE_DEAG: case FIB_PATH_TYPE_EXCLUSIVE: @@ -736,6 +762,7 @@ fib_path_unresolve (fib_path_t *path) break; case FIB_PATH_TYPE_SPECIAL: case FIB_PATH_TYPE_RECEIVE: + case FIB_PATH_TYPE_INTF_RX: case FIB_PATH_TYPE_DEAG: /* * these hold only the path's DPO, which is reset below. @@ -754,16 +781,24 @@ fib_path_unresolve (fib_path_t *path) } static fib_forward_chain_type_t -fib_path_proto_to_chain_type (fib_protocol_t proto) +fib_path_to_chain_type (const fib_path_t *path) { - switch (proto) + switch (path->fp_nh_proto) { case FIB_PROTOCOL_IP4: return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4); case FIB_PROTOCOL_IP6: return (FIB_FORW_CHAIN_TYPE_UNICAST_IP6); case FIB_PROTOCOL_MPLS: - return (FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS); + if (FIB_PATH_TYPE_RECURSIVE == path->fp_type && + MPLS_EOS == path->recursive.fp_nh.fp_eos) + { + return (FIB_FORW_CHAIN_TYPE_MPLS_EOS); + } + else + { + return (FIB_FORW_CHAIN_TYPE_MPLS_EOS); + } } return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4); } @@ -793,7 +828,7 @@ fib_path_back_walk_notify (fib_node_t *node, */ fib_path_recursive_adj_update( path, - fib_path_proto_to_chain_type(path->fp_nh_proto), + fib_path_to_chain_type(path), &path->fp_dpo); } if ((FIB_NODE_BW_REASON_FLAG_ADJ_UPDATE & ctx->fnbw_reason) || @@ -931,6 +966,8 @@ FIXME comment path->fp_oper_flags |= FIB_PATH_OPER_FLAG_DROP; } break; + case FIB_PATH_TYPE_INTF_RX: + ASSERT(0); case FIB_PATH_TYPE_DEAG: /* * FIXME When VRF delete is allowed this will need a poke. @@ -986,6 +1023,14 @@ fib_path_route_flags_to_cfg_flags (const fib_route_path_t *rpath) cfg_flags |= FIB_PATH_CFG_FLAG_LOCAL; if (rpath->frp_flags & FIB_ROUTE_PATH_ATTACHED) cfg_flags |= FIB_PATH_CFG_FLAG_ATTACHED; + if (rpath->frp_flags & FIB_ROUTE_PATH_INTF_RX) + cfg_flags |= FIB_PATH_CFG_FLAG_INTF_RX; + if (rpath->frp_flags & FIB_ROUTE_PATH_RPF_ID) + cfg_flags |= FIB_PATH_CFG_FLAG_RPF_ID; + if (rpath->frp_flags & FIB_ROUTE_PATH_EXCLUSIVE) + cfg_flags |= FIB_PATH_CFG_FLAG_EXCLUSIVE; + if (rpath->frp_flags & FIB_ROUTE_PATH_DROP) + cfg_flags |= FIB_PATH_CFG_FLAG_DROP; return (cfg_flags); } @@ -998,8 +1043,6 @@ fib_path_route_flags_to_cfg_flags (const fib_route_path_t *rpath) */ fib_node_index_t fib_path_create (fib_node_index_t pl_index, - fib_protocol_t nh_proto, - fib_path_cfg_flags_t flags, const fib_route_path_t *rpath) { fib_path_t *path; @@ -1012,7 +1055,7 @@ fib_path_create (fib_node_index_t pl_index, dpo_reset(&path->fp_dpo); path->fp_pl_index = pl_index; - path->fp_nh_proto = nh_proto; + path->fp_nh_proto = rpath->frp_proto; path->fp_via_fib = FIB_NODE_INDEX_INVALID; path->fp_weight = rpath->frp_weight; if (0 == path->fp_weight) @@ -1023,8 +1066,7 @@ fib_path_create (fib_node_index_t pl_index, */ path->fp_weight = 1; } - path->fp_cfg_flags = flags; - path->fp_cfg_flags |= fib_path_route_flags_to_cfg_flags(rpath); + path->fp_cfg_flags = fib_path_route_flags_to_cfg_flags(rpath); /* * deduce the path's tpye from the parementers and save what is needed. @@ -1035,6 +1077,17 @@ fib_path_create (fib_node_index_t pl_index, path->receive.fp_interface = rpath->frp_sw_if_index; path->receive.fp_addr = rpath->frp_addr; } + else if (path->fp_cfg_flags & FIB_PATH_CFG_FLAG_INTF_RX) + { + path->fp_type = FIB_PATH_TYPE_INTF_RX; + path->intf_rx.fp_interface = rpath->frp_sw_if_index; + } + else if (path->fp_cfg_flags & FIB_PATH_CFG_FLAG_RPF_ID) + { + path->fp_type = FIB_PATH_TYPE_DEAG; + path->deag.fp_tbl_id = rpath->frp_fib_index; + path->deag.fp_rpf_id = rpath->frp_rpf_id; + } else if (~0 != rpath->frp_sw_if_index) { if (ip46_address_is_zero(&rpath->frp_addr)) @@ -1069,6 +1122,7 @@ fib_path_create (fib_node_index_t pl_index, if (FIB_PROTOCOL_MPLS == path->fp_nh_proto) { path->recursive.fp_nh.fp_local_label = rpath->frp_local_label; + path->recursive.fp_nh.fp_eos = rpath->frp_eos; } else { @@ -1238,17 +1292,13 @@ fib_path_cmp_i (const fib_path_t *path1, res = ip46_address_cmp(&path1->attached_next_hop.fp_nh, &path2->attached_next_hop.fp_nh); if (0 == res) { - res = vnet_sw_interface_compare( - vnet_get_main(), - path1->attached_next_hop.fp_interface, - path2->attached_next_hop.fp_interface); + res = (path1->attached_next_hop.fp_interface - + path2->attached_next_hop.fp_interface); } break; case FIB_PATH_TYPE_ATTACHED: - res = vnet_sw_interface_compare( - vnet_get_main(), - path1->attached.fp_interface, - path2->attached.fp_interface); + res = (path1->attached.fp_interface - + path2->attached.fp_interface); break; case FIB_PATH_TYPE_RECURSIVE: res = ip46_address_cmp(&path1->recursive.fp_nh, @@ -1261,6 +1311,13 @@ fib_path_cmp_i (const fib_path_t *path1, break; case FIB_PATH_TYPE_DEAG: res = (path1->deag.fp_tbl_id - path2->deag.fp_tbl_id); + if (0 == res) + { + res = (path1->deag.fp_rpf_id - path2->deag.fp_rpf_id); + } + break; + case FIB_PATH_TYPE_INTF_RX: + res = (path1->intf_rx.fp_interface - path2->intf_rx.fp_interface); break; case FIB_PATH_TYPE_SPECIAL: case FIB_PATH_TYPE_RECEIVE: @@ -1336,22 +1393,22 @@ fib_path_cmp_w_route_path (fib_node_index_t path_index, &rpath->frp_addr); if (0 == res) { - res = vnet_sw_interface_compare( - vnet_get_main(), - path->attached_next_hop.fp_interface, - rpath->frp_sw_if_index); + res = (path->attached_next_hop.fp_interface - + rpath->frp_sw_if_index); } break; case FIB_PATH_TYPE_ATTACHED: - res = vnet_sw_interface_compare( - vnet_get_main(), - path->attached.fp_interface, - rpath->frp_sw_if_index); + res = (path->attached.fp_interface - rpath->frp_sw_if_index); break; case FIB_PATH_TYPE_RECURSIVE: if (FIB_PROTOCOL_MPLS == path->fp_nh_proto) { res = path->recursive.fp_nh.fp_local_label - rpath->frp_local_label; + + if (res == 0) + { + res = path->recursive.fp_nh.fp_eos - rpath->frp_eos; + } } else { @@ -1364,9 +1421,16 @@ fib_path_cmp_w_route_path (fib_node_index_t path_index, res = (path->recursive.fp_tbl_id - rpath->frp_fib_index); } break; + case FIB_PATH_TYPE_INTF_RX: + res = (path->intf_rx.fp_interface - rpath->frp_sw_if_index); + break; case FIB_PATH_TYPE_DEAG: res = (path->deag.fp_tbl_id - rpath->frp_fib_index); - break; + if (0 == res) + { + res = (path->deag.fp_rpf_id - rpath->frp_rpf_id); + } + break; case FIB_PATH_TYPE_SPECIAL: case FIB_PATH_TYPE_RECEIVE: case FIB_PATH_TYPE_EXCLUSIVE: @@ -1465,6 +1529,7 @@ fib_path_recursive_loop_detect (fib_node_index_t path_index, case FIB_PATH_TYPE_SPECIAL: case FIB_PATH_TYPE_DEAG: case FIB_PATH_TYPE_RECEIVE: + case FIB_PATH_TYPE_INTF_RX: case FIB_PATH_TYPE_EXCLUSIVE: /* * these path types cannot be part of a loop, since they are the leaves @@ -1563,7 +1628,9 @@ fib_path_resolve (fib_node_index_t path_index) if (FIB_PROTOCOL_MPLS == path->fp_nh_proto) { - fib_prefix_from_mpls_label(path->recursive.fp_nh.fp_local_label, &pfx); + fib_prefix_from_mpls_label(path->recursive.fp_nh.fp_local_label, + path->recursive.fp_nh.fp_eos, + &pfx); } else { @@ -1592,7 +1659,7 @@ fib_path_resolve (fib_node_index_t path_index) */ fib_path_recursive_adj_update( path, - fib_path_proto_to_chain_type(path->fp_nh_proto), + fib_path_to_chain_type(path), &path->fp_dpo); break; @@ -1605,16 +1672,25 @@ fib_path_resolve (fib_node_index_t path_index) drop_dpo_get(fib_proto_to_dpo(path->fp_nh_proto))); break; case FIB_PATH_TYPE_DEAG: + { /* * Resolve via a lookup DPO. * FIXME. control plane should add routes with a table ID */ - lookup_dpo_add_or_lock_w_fib_index(path->deag.fp_tbl_id, - fib_proto_to_dpo(path->fp_nh_proto), - LOOKUP_INPUT_DST_ADDR, - LOOKUP_TABLE_FROM_CONFIG, - &path->fp_dpo); + lookup_cast_t cast; + + cast = (path->fp_cfg_flags & FIB_PATH_CFG_FLAG_RPF_ID ? + LOOKUP_MULTICAST : + LOOKUP_UNICAST); + + lookup_dpo_add_or_lock_w_fib_index(path->deag.fp_tbl_id, + fib_proto_to_dpo(path->fp_nh_proto), + cast, + LOOKUP_INPUT_DST_ADDR, + LOOKUP_TABLE_FROM_CONFIG, + &path->fp_dpo); break; + } case FIB_PATH_TYPE_RECEIVE: /* * Resolve via a receive DPO. @@ -1624,6 +1700,15 @@ fib_path_resolve (fib_node_index_t path_index) &path->receive.fp_addr, &path->fp_dpo); break; + case FIB_PATH_TYPE_INTF_RX: { + /* + * Resolve via a receive DPO. + */ + interface_dpo_add_or_lock(fib_proto_to_dpo(path->fp_nh_proto), + path->intf_rx.fp_interface, + &path->fp_dpo); + break; + } case FIB_PATH_TYPE_EXCLUSIVE: /* * Resolve via the user provided DPO @@ -1652,6 +1737,7 @@ fib_path_get_resolving_interface (fib_node_index_t path_index) return (path->receive.fp_interface); case FIB_PATH_TYPE_RECURSIVE: return (fib_entry_get_resolving_interface(path->fp_via_fib)); + case FIB_PATH_TYPE_INTF_RX: case FIB_PATH_TYPE_SPECIAL: case FIB_PATH_TYPE_DEAG: case FIB_PATH_TYPE_EXCLUSIVE: @@ -1743,6 +1829,7 @@ fib_path_contribute_urpf (fib_node_index_t path_index, case FIB_PATH_TYPE_DEAG: case FIB_PATH_TYPE_RECEIVE: + case FIB_PATH_TYPE_INTF_RX: /* * these path types don't link to an adj */ @@ -1750,6 +1837,44 @@ fib_path_contribute_urpf (fib_node_index_t path_index, } } +void +fib_path_stack_mpls_disp (fib_node_index_t path_index, + dpo_proto_t payload_proto, + dpo_id_t *dpo) +{ + fib_path_t *path; + + path = fib_path_get(path_index); + + ASSERT(path); + + switch (path->fp_type) + { + case FIB_PATH_TYPE_DEAG: + { + dpo_id_t tmp = DPO_INVALID; + + dpo_copy(&tmp, dpo); + dpo_set(dpo, + DPO_MPLS_DISPOSITION, + payload_proto, + mpls_disp_dpo_create(payload_proto, + path->deag.fp_rpf_id, + &tmp)); + dpo_reset(&tmp); + break; + } + case FIB_PATH_TYPE_RECEIVE: + case FIB_PATH_TYPE_ATTACHED: + case FIB_PATH_TYPE_ATTACHED_NEXT_HOP: + case FIB_PATH_TYPE_RECURSIVE: + case FIB_PATH_TYPE_INTF_RX: + case FIB_PATH_TYPE_EXCLUSIVE: + case FIB_PATH_TYPE_SPECIAL: + break; + } +} + void fib_path_contribute_forwarding (fib_node_index_t path_index, fib_forward_chain_type_t fct, @@ -1769,7 +1894,7 @@ fib_path_contribute_forwarding (fib_node_index_t path_index, * This then represents the path's 'native' protocol; IP. * For all others will need to go find something else. */ - if (fib_path_proto_to_chain_type(path->fp_nh_proto) == fct) + if (fib_path_to_chain_type(path) == fct) { dpo_copy(dpo, &path->fp_dpo); } @@ -1813,10 +1938,10 @@ fib_path_contribute_forwarding (fib_node_index_t path_index, case FIB_FORW_CHAIN_TYPE_UNICAST_IP4: case FIB_FORW_CHAIN_TYPE_UNICAST_IP6: case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS: - fib_path_recursive_adj_update(path, fct, dpo); - break; case FIB_FORW_CHAIN_TYPE_MCAST_IP4: case FIB_FORW_CHAIN_TYPE_MCAST_IP6: + fib_path_recursive_adj_update(path, fct, dpo); + break; case FIB_FORW_CHAIN_TYPE_ETHERNET: case FIB_FORW_CHAIN_TYPE_NSH: ASSERT(0); @@ -1829,13 +1954,14 @@ fib_path_contribute_forwarding (fib_node_index_t path_index, case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS: lookup_dpo_add_or_lock_w_table_id(MPLS_FIB_DEFAULT_TABLE_ID, DPO_PROTO_MPLS, + LOOKUP_UNICAST, LOOKUP_INPUT_DST_ADDR, LOOKUP_TABLE_FROM_CONFIG, dpo); break; + case FIB_FORW_CHAIN_TYPE_MPLS_EOS: case FIB_FORW_CHAIN_TYPE_UNICAST_IP4: case FIB_FORW_CHAIN_TYPE_UNICAST_IP6: - case FIB_FORW_CHAIN_TYPE_MPLS_EOS: dpo_copy(dpo, &path->fp_dpo); break; case FIB_FORW_CHAIN_TYPE_MCAST_IP4: @@ -1870,7 +1996,7 @@ fib_path_contribute_forwarding (fib_node_index_t path_index, ai = adj_mcast_add_or_lock(path->fp_nh_proto, fib_forw_chain_type_to_link_type(fct), path->attached.fp_interface); - dpo_set(dpo, DPO_ADJACENCY_MCAST, + dpo_set(dpo, DPO_ADJACENCY, fib_forw_chain_type_to_dpo_proto(fct), ai); adj_unlock(ai); @@ -1878,6 +2004,14 @@ fib_path_contribute_forwarding (fib_node_index_t path_index, break; } break; + case FIB_PATH_TYPE_INTF_RX: + /* + * Create the adj needed for sending IP multicast traffic + */ + interface_dpo_add_or_lock(fib_forw_chain_type_to_dpo_proto(fct), + path->attached.fp_interface, + dpo); + break; case FIB_PATH_TYPE_RECEIVE: case FIB_PATH_TYPE_SPECIAL: dpo_copy(dpo, &path->fp_dpo); diff --git a/src/vnet/fib/fib_path.h b/src/vnet/fib/fib_path.h index 14efc1ab..334be6f5 100644 --- a/src/vnet/fib/fib_path.h +++ b/src/vnet/fib/fib_path.h @@ -69,6 +69,14 @@ typedef enum fib_path_cfg_attribute_t_ { /** * The path is a for-us path */ + FIB_PATH_CFG_ATTRIBUTE_INTF_RX, + /** + * The path is a deag with rpf-id + */ + FIB_PATH_CFG_ATTRIBUTE_RPF_ID, + /** + * The path is an interface recieve + */ FIB_PATH_CFG_ATTRIBUTE_LOCAL, /** * Marker. Add new types before this one, then update it. @@ -88,6 +96,8 @@ typedef enum fib_path_cfg_attribute_t_ { [FIB_PATH_CFG_ATTRIBUTE_RESOLVE_ATTACHED] = "resolve-attached", \ [FIB_PATH_CFG_ATTRIBUTE_LOCAL] = "local", \ [FIB_PATH_CFG_ATTRIBUTE_ATTACHED] = "attached", \ + [FIB_PATH_CFG_ATTRIBUTE_INTF_RX] = "interface-rx", \ + [FIB_PATH_CFG_ATTRIBUTE_RPF_ID] = "rpf-id", \ } #define FOR_EACH_FIB_PATH_CFG_ATTRIBUTE(_item) \ @@ -106,6 +116,8 @@ typedef enum fib_path_cfg_flags_t_ { FIB_PATH_CFG_FLAG_RESOLVE_ATTACHED = (1 << FIB_PATH_CFG_ATTRIBUTE_RESOLVE_ATTACHED), FIB_PATH_CFG_FLAG_LOCAL = (1 << FIB_PATH_CFG_ATTRIBUTE_LOCAL), FIB_PATH_CFG_FLAG_ATTACHED = (1 << FIB_PATH_CFG_ATTRIBUTE_ATTACHED), + FIB_PATH_CFG_FLAG_INTF_RX = (1 << FIB_PATH_CFG_ATTRIBUTE_INTF_RX), + FIB_PATH_CFG_FLAG_RPF_ID = (1 << FIB_PATH_CFG_ATTRIBUTE_RPF_ID), } __attribute__ ((packed)) fib_path_cfg_flags_t; @@ -117,8 +129,6 @@ extern u8 *fib_path_adj_format(fib_node_index_t pi, extern u8 * format_fib_path(u8 * s, va_list * args); extern fib_node_index_t fib_path_create(fib_node_index_t pl_index, - fib_protocol_t nh_proto, - fib_path_cfg_flags_t flags, const fib_route_path_t *path); extern fib_node_index_t fib_path_create_special(fib_node_index_t pl_index, fib_protocol_t nh_proto, @@ -145,6 +155,9 @@ extern load_balance_path_t * fib_path_append_nh_for_multipath_hash( fib_node_index_t path_index, fib_forward_chain_type_t fct, load_balance_path_t *hash_key); +extern void fib_path_stack_mpls_disp(fib_node_index_t path_index, + dpo_proto_t payload_proto, + dpo_id_t *dpo); extern void fib_path_contribute_forwarding(fib_node_index_t path_index, fib_forward_chain_type_t type, dpo_id_t *dpo); diff --git a/src/vnet/fib/fib_path_ext.c b/src/vnet/fib/fib_path_ext.c index f75b5626..08293bcf 100644 --- a/src/vnet/fib/fib_path_ext.c +++ b/src/vnet/fib/fib_path_ext.c @@ -103,8 +103,8 @@ fib_path_ext_is_imp_null (fib_path_ext_t *path_ext) load_balance_path_t * fib_path_ext_stack (fib_path_ext_t *path_ext, - const fib_entry_t *entry, fib_forward_chain_type_t child_fct, + fib_forward_chain_type_t imp_null_fct, load_balance_path_t *nhs) { fib_forward_chain_type_t parent_fct; @@ -129,7 +129,7 @@ fib_path_ext_stack (fib_path_ext_t *path_ext, */ if (fib_path_ext_is_imp_null(path_ext)) { - parent_fct = fib_entry_chain_type_fixup(entry, child_fct); + parent_fct = imp_null_fct; } else { diff --git a/src/vnet/fib/fib_path_ext.h b/src/vnet/fib/fib_path_ext.h index cf8f8df0..d617700d 100644 --- a/src/vnet/fib/fib_path_ext.h +++ b/src/vnet/fib/fib_path_ext.h @@ -18,6 +18,7 @@ #include #include +#include /** * A path extension is a per-entry addition to the forwarding information @@ -61,8 +62,8 @@ extern void fib_path_ext_resolve(fib_path_ext_t *path_ext, fib_node_index_t path_list_index); extern load_balance_path_t *fib_path_ext_stack(fib_path_ext_t *path_ext, - const struct fib_entry_t_ *entry, fib_forward_chain_type_t fct, + fib_forward_chain_type_t imp_null_fct, load_balance_path_t *nhs); #endif diff --git a/src/vnet/fib/fib_path_list.c b/src/vnet/fib/fib_path_list.c index b9a391b3..ea6565dd 100644 --- a/src/vnet/fib/fib_path_list.c +++ b/src/vnet/fib/fib_path_list.c @@ -40,13 +40,6 @@ typedef struct fib_path_list_t_ { */ fib_path_list_flags_t fpl_flags; - /** - * The next-hop protocol for the paths in this path list. - * Note that fixing the proto here means we don't support a mix of - * v4 and v6 paths. ho hum. - */ - fib_protocol_t fpl_nh_proto; - /** * Vector of paths indicies for all configured paths. * For shareable path-lists this list MUST not change. @@ -57,6 +50,11 @@ typedef struct fib_path_list_t_ { * the RPF list calculated for this path list */ fib_node_index_t fpl_urpf; + + /** + * Hash table of paths. valid only with INDEXED flag + */ + uword *fpl_db; } fib_path_list_t; /* @@ -131,7 +129,6 @@ format_fib_path_list (u8 * s, va_list * args) s = format (s, " index:%u", fib_path_list_get_index(path_list)); s = format (s, " locks:%u", path_list->fpl_node.fn_locks); - s = format (s, " proto:%U", format_fib_protocol, path_list->fpl_nh_proto); if (FIB_PATH_LIST_FLAG_NONE != path_list->fpl_flags) { @@ -155,26 +152,6 @@ format_fib_path_list (u8 * s, va_list * args) return (s); } -u8 * -fib_path_list_adjs_format (fib_node_index_t path_list_index, - u32 indent, - u8 * s) -{ - fib_path_list_t *path_list; - u32 i; - - path_list = fib_path_list_get(path_list_index); - - vec_foreach_index (i, path_list->fpl_paths) - { - s = fib_path_adj_format(path_list->fpl_paths[i], - indent, s); - } - - return (s); -} - - u8 * fib_path_list_format (fib_node_index_t path_list_index, u8 * s) @@ -648,27 +625,6 @@ fib_path_list_is_looped (fib_node_index_t path_list_index) return (path_list->fpl_flags & FIB_PATH_LIST_FLAG_LOOPED); } -static fib_path_cfg_flags_t -fib_path_list_flags_2_path_flags (fib_path_list_flags_t plf) -{ - fib_path_cfg_flags_t pf = FIB_PATH_CFG_FLAG_NONE; - - if (plf & FIB_PATH_LIST_FLAG_LOCAL) - { - pf |= FIB_PATH_CFG_FLAG_LOCAL; - } - if (plf & FIB_PATH_LIST_FLAG_DROP) - { - pf |= FIB_PATH_CFG_FLAG_DROP; - } - if (plf & FIB_PATH_LIST_FLAG_EXCLUSIVE) - { - pf |= FIB_PATH_CFG_FLAG_EXCLUSIVE; - } - - return (pf); -} - static fib_path_list_flags_t fib_path_list_flags_fixup (fib_path_list_flags_t flags) { @@ -695,18 +651,15 @@ fib_path_list_create (fib_path_list_flags_t flags, flags = fib_path_list_flags_fixup(flags); path_list = fib_path_list_alloc(&path_list_index); path_list->fpl_flags = flags; - /* - * we'll assume for now all paths are the same next-hop protocol - */ - path_list->fpl_nh_proto = rpaths[0].frp_proto; - vec_foreach_index(i, rpaths) + if (NULL != rpaths) { - vec_add1(path_list->fpl_paths, - fib_path_create(path_list_index, - path_list->fpl_nh_proto, - fib_path_list_flags_2_path_flags(flags), - &rpaths[i])); + vec_foreach_index(i, rpaths) + { + vec_add1(path_list->fpl_paths, + fib_path_create(path_list_index, + &rpaths[i])); + } } /* @@ -748,6 +701,27 @@ fib_path_list_create (fib_path_list_flags_t flags, return (path_list_index); } +static fib_path_cfg_flags_t +fib_path_list_flags_2_path_flags (fib_path_list_flags_t plf) +{ + fib_path_cfg_flags_t pf = FIB_PATH_CFG_FLAG_NONE; + + if (plf & FIB_PATH_LIST_FLAG_DROP) + { + pf |= FIB_PATH_CFG_FLAG_DROP; + } + if (plf & FIB_PATH_LIST_FLAG_EXCLUSIVE) + { + pf |= FIB_PATH_CFG_FLAG_EXCLUSIVE; + } + if (plf & FIB_PATH_LIST_FLAG_LOCAL) + { + pf |= FIB_PATH_CFG_FLAG_LOCAL; + } + + return (pf); +} + fib_node_index_t fib_path_list_create_special (fib_protocol_t nh_proto, fib_path_list_flags_t flags, @@ -758,11 +732,10 @@ fib_path_list_create_special (fib_protocol_t nh_proto, path_list = fib_path_list_alloc(&path_list_index); path_list->fpl_flags = flags; - path_list->fpl_nh_proto = nh_proto; path_index = fib_path_create_special(path_list_index, - path_list->fpl_nh_proto, + nh_proto, fib_path_list_flags_2_path_flags(flags), dpo); vec_add1(path_list->fpl_paths, path_index); @@ -775,6 +748,30 @@ fib_path_list_create_special (fib_protocol_t nh_proto, return (path_list_index); } +/* + * return the index info the path-lists's vector of paths, of the matching path. + * ~0 if not found + */ +u32 +fib_path_list_find_rpath (fib_node_index_t path_list_index, + const fib_route_path_t *rpath) +{ + fib_path_list_t *path_list; + u32 ii; + + path_list = fib_path_list_get(path_list_index); + + vec_foreach_index (ii, path_list->fpl_paths) + { + if (!fib_path_cmp_w_route_path(path_list->fpl_paths[ii], rpath)) + { + return (ii); + } + } + return (~0); +} + + /* * fib_path_list_copy_and_path_add * @@ -782,13 +779,62 @@ fib_path_list_create_special (fib_protocol_t nh_proto, * The path-list returned could either have been newly created, or * can be a shared path-list from the data-base. */ +fib_node_index_t +fib_path_list_path_add (fib_node_index_t path_list_index, + const fib_route_path_t *rpaths) +{ + fib_node_index_t new_path_index, *orig_path_index; + fib_path_list_t *path_list; + + /* + * alloc the new list before we retrieve the old one, lest + * the alloc result in a realloc + */ + path_list = fib_path_list_get(path_list_index); + + ASSERT(1 == vec_len(rpaths)); + ASSERT(!(path_list->fpl_flags & FIB_PATH_LIST_FLAG_SHARED)); + + FIB_PATH_LIST_DBG(orig_path_list, "path-add"); + + new_path_index = fib_path_create(path_list_index, + rpaths); + + vec_foreach (orig_path_index, path_list->fpl_paths) + { + /* + * don't add duplicate paths + */ + if (0 == fib_path_cmp(new_path_index, *orig_path_index)) + { + return (*orig_path_index); + } + } + + /* + * Add the new path - no sort, no sharing, no key.. + */ + vec_add1(path_list->fpl_paths, new_path_index); + + FIB_PATH_LIST_DBG(path_list, "path-added"); + + /* + * no shared path list requested. resolve and use the one + * just created. + */ + fib_path_resolve(new_path_index); + + return (new_path_index); +} + fib_node_index_t fib_path_list_copy_and_path_add (fib_node_index_t orig_path_list_index, - fib_path_list_flags_t flags, - const fib_route_path_t *rpaths) + fib_path_list_flags_t flags, + const fib_route_path_t *rpaths) { fib_node_index_t path_index, new_path_index, *orig_path_index; fib_path_list_t *path_list, *orig_path_list; + fib_node_index_t exist_path_list_index; fib_node_index_t path_list_index; fib_node_index_t pi; @@ -806,13 +852,11 @@ fib_path_list_copy_and_path_add (fib_node_index_t orig_path_list_index, flags = fib_path_list_flags_fixup(flags); path_list->fpl_flags = flags; - path_list->fpl_nh_proto = orig_path_list->fpl_nh_proto; + vec_validate(path_list->fpl_paths, vec_len(orig_path_list->fpl_paths)); pi = 0; new_path_index = fib_path_create(path_list_index, - path_list->fpl_nh_proto, - fib_path_list_flags_2_path_flags(flags), rpaths); vec_foreach (orig_path_index, orig_path_list->fpl_paths) @@ -845,46 +889,79 @@ fib_path_list_copy_and_path_add (fib_node_index_t orig_path_list_index, FIB_PATH_LIST_DBG(path_list, "path-added"); /* - * If a shared path list is requested, consult the DB for a match + * check for a matching path-list in the DB. + * If we find one then we can return the existing one and destroy the + * new one just created. */ - if (path_list->fpl_flags & FIB_PATH_LIST_FLAG_SHARED) + exist_path_list_index = fib_path_list_db_find(path_list); + if (FIB_NODE_INDEX_INVALID != exist_path_list_index) { - fib_node_index_t exist_path_list_index; - /* - * check for a matching path-list in the DB. - * If we find one then we can return the existing one and destroy the - * new one just created. - */ - exist_path_list_index = fib_path_list_db_find(path_list); - if (FIB_NODE_INDEX_INVALID != exist_path_list_index) - { - fib_path_list_destroy(path_list); + fib_path_list_destroy(path_list); - path_list_index = exist_path_list_index; - } - else - { - /* - * if there was not a matching path-list, then this - * new one will need inserting into the DB and resolving. - */ - fib_path_list_db_insert(path_list_index); - - path_list = fib_path_list_resolve(path_list); - } + path_list_index = exist_path_list_index; } else { - /* - * no shared path list requested. resolve and use the one - * just created. - */ - path_list = fib_path_list_resolve(path_list); + /* + * if there was not a matching path-list, then this + * new one will need inserting into the DB and resolving. + */ + fib_path_list_db_insert(path_list_index); + + path_list = fib_path_list_resolve(path_list); } return (path_list_index); } +/* + * fib_path_list_path_remove + */ +fib_node_index_t +fib_path_list_path_remove (fib_node_index_t path_list_index, + const fib_route_path_t *rpaths) +{ + fib_node_index_t match_path_index, tmp_path_index; + fib_path_list_t *path_list; + fib_node_index_t pi; + + path_list = fib_path_list_get(path_list_index); + + ASSERT(1 == vec_len(rpaths)); + ASSERT(!(path_list->fpl_flags & FIB_PATH_LIST_FLAG_SHARED)); + + FIB_PATH_LIST_DBG(orig_path_list, "path-remove"); + + /* + * create a representation of the path to be removed, so it + * can be used as a comparison object during the copy. + */ + tmp_path_index = fib_path_create(path_list_index, + rpaths); + match_path_index = FIB_NODE_INDEX_INVALID; + + vec_foreach_index (pi, path_list->fpl_paths) + { + if (0 == fib_path_cmp(tmp_path_index, + path_list->fpl_paths[pi])) + { + /* + * match - remove it + */ + match_path_index = path_list->fpl_paths[pi]; + fib_path_destroy(match_path_index); + vec_del1(path_list->fpl_paths, pi); + } + } + + /* + * done with the temporary now + */ + fib_path_destroy(tmp_path_index); + + return (match_path_index); +} + /* * fib_path_list_copy_and_path_remove * @@ -911,7 +988,6 @@ fib_path_list_copy_and_path_remove (fib_node_index_t orig_path_list_index, FIB_PATH_LIST_DBG(orig_path_list, "copy-remove"); path_list->fpl_flags = flags; - path_list->fpl_nh_proto = orig_path_list->fpl_nh_proto; /* * allocate as many paths as we might need in one go, rather than * using vec_add to do a few at a time. @@ -927,8 +1003,6 @@ fib_path_list_copy_and_path_remove (fib_node_index_t orig_path_list_index, * can be used as a comparison object during the copy. */ tmp_path_index = fib_path_create(path_list_index, - path_list->fpl_nh_proto, - fib_path_list_flags_2_path_flags(flags), rpaths); vec_foreach (orig_path_index, orig_path_list->fpl_paths) diff --git a/src/vnet/fib/fib_path_list.h b/src/vnet/fib/fib_path_list.h index b4971add..9d246211 100644 --- a/src/vnet/fib/fib_path_list.h +++ b/src/vnet/fib/fib_path_list.h @@ -38,6 +38,11 @@ typedef enum fib_path_list_attribute_t_ { * be searched for each route update. */ FIB_PATH_LIST_ATTRIBUTE_SHARED = FIB_PATH_LIST_ATTRIBUTE_FIRST, + /** + * Indexed means the path-list keeps a hash table of all paths for + * fast lookup. The lookup result is the fib_node_index of the path. + */ + FIB_PATH_LIST_ATTRIBUTE_INDEXED, /** * explicit drop path-list. Used when the entry source needs to * force a drop, despite the fact the path info is present. @@ -73,6 +78,7 @@ typedef enum fib_path_list_attribute_t_ { typedef enum fib_path_list_flags_t_ { FIB_PATH_LIST_FLAG_NONE = 0, FIB_PATH_LIST_FLAG_SHARED = (1 << FIB_PATH_LIST_ATTRIBUTE_SHARED), + FIB_PATH_LIST_FLAG_INDEXED = (1 << FIB_PATH_LIST_ATTRIBUTE_INDEXED), FIB_PATH_LIST_FLAG_DROP = (1 << FIB_PATH_LIST_ATTRIBUTE_DROP), FIB_PATH_LIST_FLAG_LOCAL = (1 << FIB_PATH_LIST_ATTRIBUTE_LOCAL), FIB_PATH_LIST_FLAG_EXCLUSIVE = (1 << FIB_PATH_LIST_ATTRIBUTE_EXCLUSIVE), @@ -83,10 +89,11 @@ typedef enum fib_path_list_flags_t_ { #define FIB_PATH_LIST_ATTRIBUTES { \ [FIB_PATH_LIST_ATTRIBUTE_SHARED] = "shared", \ + [FIB_PATH_LIST_ATTRIBUTE_INDEXED] = "indexed", \ [FIB_PATH_LIST_ATTRIBUTE_RESOLVED] = "resolved", \ [FIB_PATH_LIST_ATTRIBUTE_DROP] = "drop", \ [FIB_PATH_LIST_ATTRIBUTE_EXCLUSIVE] = "exclusive", \ - [FIB_PATH_LIST_ATTRIBUTE_LOCAL] = "local", \ + [FIB_PATH_LIST_ATTRIBUTE_LOCAL] = "local", \ [FIB_PATH_LIST_ATTRIBUTE_LOOPED] = "looped", \ [FIB_PATH_LIST_ATTRIBUTE_NO_URPF] = "no-uRPF", \ } @@ -110,6 +117,13 @@ extern fib_node_index_t fib_path_list_copy_and_path_remove( fib_node_index_t pl_index, fib_path_list_flags_t flags, const fib_route_path_t *path); +extern fib_node_index_t fib_path_list_path_add ( + fib_node_index_t path_list_index, + const fib_route_path_t *rpaths); +extern fib_node_index_t fib_path_list_path_remove ( + fib_node_index_t path_list_index, + const fib_route_path_t *rpaths); + extern u32 fib_path_list_get_n_paths(fib_node_index_t pl_index); extern void fib_path_list_contribute_forwarding(fib_node_index_t path_list_index, @@ -137,11 +151,11 @@ extern int fib_path_list_is_looped(fib_node_index_t path_list_index); extern fib_protocol_t fib_path_list_get_proto(fib_node_index_t path_list_index); extern u8 * fib_path_list_format(fib_node_index_t pl_index, u8 * s); -extern u8 * fib_path_list_adjs_format(fib_node_index_t pl_index, - u32 indent, - u8 * s); extern index_t fib_path_list_lb_map_add_or_lock(fib_node_index_t pl_index, const fib_node_index_t *pis); +extern u32 fib_path_list_find_rpath (fib_node_index_t path_list_index, + const fib_route_path_t *rpath); + /** * A callback function type for walking a path-list's paths */ diff --git a/src/vnet/fib/fib_table.c b/src/vnet/fib/fib_table.c index 6c3162e7..b31f35e3 100644 --- a/src/vnet/fib/fib_table.c +++ b/src/vnet/fib/fib_table.c @@ -475,8 +475,21 @@ fib_table_entry_special_remove (u32 fib_index, */ static void fib_table_route_path_fixup (const fib_prefix_t *prefix, + fib_entry_flag_t eflags, fib_route_path_t *path) { + /* + * not all zeros next hop && + * is recursive path && + * nexthop is same as the route's address + */ + if ((!ip46_address_is_zero(&path->frp_addr)) && + (~0 == path->frp_sw_if_index) && + (0 == ip46_address_cmp(&path->frp_addr, &prefix->fp_addr))) + { + /* Prefix recurses via itse;f */ + path->frp_flags |= FIB_ROUTE_PATH_DROP; + } if (fib_prefix_is_host(prefix) && ip46_address_is_zero(&path->frp_addr) && path->frp_sw_if_index != ~0) @@ -484,7 +497,19 @@ fib_table_route_path_fixup (const fib_prefix_t *prefix, path->frp_addr = prefix->fp_addr; path->frp_flags |= FIB_ROUTE_PATH_ATTACHED; } -} + if (eflags & FIB_ENTRY_FLAG_DROP) + { + path->frp_flags |= FIB_ROUTE_PATH_DROP; + } + if (eflags & FIB_ENTRY_FLAG_LOCAL) + { + path->frp_flags |= FIB_ROUTE_PATH_LOCAL; + } + if (eflags & FIB_ENTRY_FLAG_EXCLUSIVE) + { + path->frp_flags |= FIB_ROUTE_PATH_EXCLUSIVE; + } +} fib_node_index_t fib_table_entry_path_add (u32 fib_index, @@ -536,7 +561,7 @@ fib_table_entry_path_add2 (u32 fib_index, for (ii = 0; ii < vec_len(rpath); ii++) { - fib_table_route_path_fixup(prefix, &rpath[ii]); + fib_table_route_path_fixup(prefix, flags, &rpath[ii]); } if (FIB_NODE_INDEX_INVALID == fib_entry_index) @@ -583,11 +608,6 @@ fib_table_entry_path_remove2 (u32 fib_index, fib_table = fib_table_get(fib_index, prefix->fp_proto); fib_entry_index = fib_table_lookup_exact_match_i(fib_table, prefix); - for (ii = 0; ii < vec_len(rpath); ii++) - { - fib_table_route_path_fixup(prefix, &rpath[ii]); - } - if (FIB_NODE_INDEX_INVALID == fib_entry_index) { /* @@ -605,6 +625,15 @@ fib_table_entry_path_remove2 (u32 fib_index, fib_entry_lock(fib_entry_index); was_sourced = fib_entry_is_sourced(fib_entry_index, source); + for (ii = 0; ii < vec_len(rpath); ii++) + { + fib_table_route_path_fixup( + prefix, + fib_entry_get_flags_for_source(fib_entry_index, + source), + &rpath[ii]); + } + src_flag = fib_entry_path_remove(fib_entry_index, source, rpath); if (!(FIB_ENTRY_SRC_FLAG_ADDED & src_flag)) @@ -661,7 +690,6 @@ fib_table_entry_path_remove (u32 fib_index, }; fib_route_path_t *paths = NULL; - fib_table_route_path_fixup(prefix, &path); vec_add1(paths, path); fib_table_entry_path_remove2(fib_index, prefix, source, paths); @@ -692,7 +720,7 @@ fib_table_entry_update (u32 fib_index, for (ii = 0; ii < vec_len(paths); ii++) { - fib_table_route_path_fixup(prefix, &paths[ii]); + fib_table_route_path_fixup(prefix, flags, &paths[ii]); } /* * sort the paths provided by the control plane. this means @@ -750,7 +778,6 @@ fib_table_entry_update_one_path (u32 fib_index, }; fib_route_path_t *paths = NULL; - fib_table_route_path_fixup(prefix, &path); vec_add1(paths, path); fib_entry_index = diff --git a/src/vnet/fib/fib_test.c b/src/vnet/fib/fib_test.c index 3c9b8a38..e4a8a70e 100644 --- a/src/vnet/fib/fib_test.c +++ b/src/vnet/fib/fib_test.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include @@ -271,6 +273,7 @@ typedef enum fib_test_lb_bucket_type_t_ { FT_LB_O_LB, FT_LB_SPECIAL, FT_LB_ADJ, + FT_LB_INTF, } fib_test_lb_bucket_type_t; typedef struct fib_test_lb_bucket_t_ { @@ -315,6 +318,31 @@ typedef struct fib_test_lb_bucket_t_ { }; } fib_test_lb_bucket_t; +typedef enum fib_test_rep_bucket_type_t_ { + FT_REP_LABEL_O_ADJ, + FT_REP_DISP_MFIB_LOOKUP, + FT_REP_INTF, +} fib_test_rep_bucket_type_t; + +typedef struct fib_test_rep_bucket_t_ { + fib_test_rep_bucket_type_t type; + + union + { + struct + { + mpls_eos_bit_t eos; + mpls_label_t label; + u8 ttl; + adj_index_t adj; + } label_o_adj; + struct + { + adj_index_t adj; + } adj; + }; +} fib_test_rep_bucket_t; + #define FIB_TEST_LB(_cond, _comment, _args...) \ { \ if (!FIB_TEST_I(_cond, _comment, ##_args)) { \ @@ -322,7 +350,83 @@ typedef struct fib_test_lb_bucket_t_ { } \ } -static int +int +fib_test_validate_rep_v (const replicate_t *rep, + u16 n_buckets, + va_list ap) +{ + const fib_test_rep_bucket_t *exp; + const dpo_id_t *dpo; + int bucket; + + FIB_TEST_LB((n_buckets == rep->rep_n_buckets), + "n_buckets = %d", rep->rep_n_buckets); + + for (bucket = 0; bucket < n_buckets; bucket++) + { + exp = va_arg(ap, fib_test_rep_bucket_t*); + + dpo = replicate_get_bucket_i(rep, bucket); + + switch (exp->type) + { + case FT_REP_LABEL_O_ADJ: + { + const mpls_label_dpo_t *mld; + mpls_label_t hdr; + FIB_TEST_LB((DPO_MPLS_LABEL == dpo->dpoi_type), + "bucket %d stacks on %U", + bucket, + format_dpo_type, dpo->dpoi_type); + + mld = mpls_label_dpo_get(dpo->dpoi_index); + hdr = clib_net_to_host_u32(mld->mld_hdr[0].label_exp_s_ttl); + + FIB_TEST_LB((vnet_mpls_uc_get_label(hdr) == + exp->label_o_adj.label), + "bucket %d stacks on label %d", + bucket, + exp->label_o_adj.label); + + FIB_TEST_LB((vnet_mpls_uc_get_s(hdr) == + exp->label_o_adj.eos), + "bucket %d stacks on label %d %U", + bucket, + exp->label_o_adj.label, + format_mpls_eos_bit, exp->label_o_adj.eos); + + FIB_TEST_LB((DPO_ADJACENCY_INCOMPLETE == mld->mld_dpo.dpoi_type), + "bucket %d label stacks on %U", + bucket, + format_dpo_type, mld->mld_dpo.dpoi_type); + + FIB_TEST_LB((exp->label_o_adj.adj == mld->mld_dpo.dpoi_index), + "bucket %d label stacks on adj %d", + bucket, + exp->label_o_adj.adj); + } + break; + case FT_REP_INTF: + FIB_TEST_LB((DPO_INTERFACE == dpo->dpoi_type), + "bucket %d stacks on %U", + bucket, + format_dpo_type, dpo->dpoi_type); + + FIB_TEST_LB((exp->adj.adj == dpo->dpoi_index), + "bucket %d stacks on adj %d", + bucket, + exp->adj.adj); + break; + case FT_REP_DISP_MFIB_LOOKUP: +// ASSERT(0); + break; + } + } + + return (!0); +} + +int fib_test_validate_lb_v (const load_balance_t *lb, u16 n_buckets, va_list ap) @@ -484,6 +588,16 @@ fib_test_validate_lb_v (const load_balance_t *lb, bucket, exp->adj.adj); break; + case FT_LB_INTF: + FIB_TEST_I((DPO_INTERFACE == dpo->dpoi_type), + "bucket %d stacks on %U", + bucket, + format_dpo_type, dpo->dpoi_type); + FIB_TEST_LB((exp->adj.adj == dpo->dpoi_index), + "bucket %d stacks on adj %d", + bucket, + exp->adj.adj); + break; case FT_LB_O_LB: FIB_TEST_I((DPO_LOAD_BALANCE == dpo->dpoi_type), "bucket %d stacks on %U", @@ -509,14 +623,13 @@ fib_test_validate_lb_v (const load_balance_t *lb, return (!0); } -static int +int fib_test_validate_entry (fib_node_index_t fei, fib_forward_chain_type_t fct, u16 n_buckets, ...) { dpo_id_t dpo = DPO_INVALID; - const load_balance_t *lb; fib_prefix_t pfx; index_t fw_lbi; u32 fib_index; @@ -529,47 +642,59 @@ fib_test_validate_entry (fib_node_index_t fei, fib_index = fib_entry_get_fib_index(fei); fib_entry_contribute_forwarding(fei, fct, &dpo); - FIB_TEST_LB((DPO_LOAD_BALANCE == dpo.dpoi_type), - "Entry links to %U", - format_dpo_type, dpo.dpoi_type); - lb = load_balance_get(dpo.dpoi_index); - - res = fib_test_validate_lb_v(lb, n_buckets, ap); + if (DPO_REPLICATE == dpo.dpoi_type) + { + const replicate_t *rep; - /* - * ensure that the LB contributed by the entry is the - * same as the LB in the forwarding tables - */ - if (fct == fib_entry_get_default_chain_type(fib_entry_get(fei))) + rep = replicate_get(dpo.dpoi_index); + res = fib_test_validate_rep_v(rep, n_buckets, ap); + } + else { - switch (pfx.fp_proto) - { - case FIB_PROTOCOL_IP4: - fw_lbi = ip4_fib_forwarding_lookup(fib_index, &pfx.fp_addr.ip4); - break; - case FIB_PROTOCOL_IP6: - fw_lbi = ip6_fib_table_fwding_lookup(&ip6_main, fib_index, &pfx.fp_addr.ip6); - break; - case FIB_PROTOCOL_MPLS: - { - mpls_unicast_header_t hdr = { - .label_exp_s_ttl = 0, - }; + const load_balance_t *lb; + + FIB_TEST_LB((DPO_LOAD_BALANCE == dpo.dpoi_type), + "Entry links to %U", + format_dpo_type, dpo.dpoi_type); - vnet_mpls_uc_set_label(&hdr.label_exp_s_ttl, pfx.fp_label); - vnet_mpls_uc_set_s(&hdr.label_exp_s_ttl, pfx.fp_eos); - hdr.label_exp_s_ttl = clib_host_to_net_u32(hdr.label_exp_s_ttl); + lb = load_balance_get(dpo.dpoi_index); + res = fib_test_validate_lb_v(lb, n_buckets, ap); - fw_lbi = mpls_fib_table_forwarding_lookup(fib_index, &hdr); + /* + * ensure that the LB contributed by the entry is the + * same as the LB in the forwarding tables + */ + if (fct == fib_entry_get_default_chain_type(fib_entry_get(fei))) + { + switch (pfx.fp_proto) + { + case FIB_PROTOCOL_IP4: + fw_lbi = ip4_fib_forwarding_lookup(fib_index, &pfx.fp_addr.ip4); + break; + case FIB_PROTOCOL_IP6: + fw_lbi = ip6_fib_table_fwding_lookup(&ip6_main, fib_index, &pfx.fp_addr.ip6); break; + case FIB_PROTOCOL_MPLS: + { + mpls_unicast_header_t hdr = { + .label_exp_s_ttl = 0, + }; + + vnet_mpls_uc_set_label(&hdr.label_exp_s_ttl, pfx.fp_label); + vnet_mpls_uc_set_s(&hdr.label_exp_s_ttl, pfx.fp_eos); + hdr.label_exp_s_ttl = clib_host_to_net_u32(hdr.label_exp_s_ttl); + + fw_lbi = mpls_fib_table_forwarding_lookup(fib_index, &hdr); + break; + } + default: + fw_lbi = 0; } - default: - fw_lbi = 0; + FIB_TEST_LB((fw_lbi == dpo.dpoi_index), + "Contributed LB = FW LB: %U\n %U", + format_load_balance, fw_lbi, 0, + format_load_balance, dpo.dpoi_index, 0); } - FIB_TEST_LB((fw_lbi == dpo.dpoi_index), - "Contributed LB = FW LB: %U\n %U", - format_load_balance, fw_lbi, 0, - format_load_balance, dpo.dpoi_index, 0); } dpo_reset(&dpo); @@ -1289,6 +1414,7 @@ fib_test_v4 (void) lookup_dpo_add_or_lock_w_fib_index(fib_index, DPO_PROTO_IP4, + LOOKUP_UNICAST, LOOKUP_INPUT_DST_ADDR, LOOKUP_TABLE_FROM_CONFIG, &ex_dpo); @@ -2605,7 +2731,6 @@ fib_test_v4 (void) NULL, FIB_ROUTE_PATH_FLAG_NONE); - fei = fib_table_lookup(fib_index, &pfx_5_5_5_6_s_32); dpo1 = fib_entry_contribute_ip_forwarding(fei); @@ -7493,6 +7618,7 @@ lfib_test (void) fib_route_path_t *rpaths = NULL, rpath = { .frp_proto = FIB_PROTOCOL_MPLS, .frp_local_label = 1200, + .frp_eos = MPLS_NON_EOS, .frp_sw_if_index = ~0, // recurive .frp_fib_index = 0, // Default MPLS fib .frp_weight = 1, @@ -7607,6 +7733,146 @@ lfib_test (void) dpo_reset(&ip_1200); + /* + * An rx-interface route. + * like the tail of an mcast LSP + */ + dpo_id_t idpo = DPO_INVALID; + + interface_dpo_add_or_lock(DPO_PROTO_IP4, + tm->hw[0]->sw_if_index, + &idpo); + + fib_prefix_t pfx_2500 = { + .fp_len = 21, + .fp_proto = FIB_PROTOCOL_MPLS, + .fp_label = 2500, + .fp_eos = MPLS_EOS, + .fp_payload_proto = DPO_PROTO_IP4, + }; + fib_test_lb_bucket_t rx_intf_0 = { + .type = FT_LB_INTF, + .adj = { + .adj = idpo.dpoi_index, + }, + }; + + lfe = fib_table_entry_update_one_path(fib_index, + &pfx_2500, + FIB_SOURCE_API, + FIB_ENTRY_FLAG_NONE, + FIB_PROTOCOL_IP4, + NULL, + tm->hw[0]->sw_if_index, + ~0, // invalid fib index + 0, + NULL, + FIB_ROUTE_PATH_INTF_RX); + FIB_TEST(fib_test_validate_entry(lfe, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + 1, + &rx_intf_0), + "2500 rx-interface 0"); + fib_table_entry_delete(fib_index, &pfx_2500, FIB_SOURCE_API); + + /* + * An MPLS mulicast entry + */ + fib_prefix_t pfx_3500 = { + .fp_len = 21, + .fp_proto = FIB_PROTOCOL_MPLS, + .fp_label = 3500, + .fp_eos = MPLS_EOS, + .fp_payload_proto = DPO_PROTO_IP4, + }; + fib_test_rep_bucket_t mc_0 = { + .type = FT_REP_LABEL_O_ADJ, + .label_o_adj = { + .adj = ai_mpls_10_10_10_1, + .label = 3300, + .eos = MPLS_EOS, + }, + }; + fib_test_rep_bucket_t mc_intf_0 = { + .type = FT_REP_INTF, + .adj = { + .adj = idpo.dpoi_index, + }, + }; + mpls_label_t *l3300 = NULL; + vec_add1(l3300, 3300); + + lfe = fib_table_entry_update_one_path(lfib_index, + &pfx_3500, + FIB_SOURCE_API, + FIB_ENTRY_FLAG_MULTICAST, + FIB_PROTOCOL_IP4, + &nh_10_10_10_1, + tm->hw[0]->sw_if_index, + ~0, // invalid fib index + 1, + l3300, + FIB_ROUTE_PATH_FLAG_NONE); + FIB_TEST(fib_test_validate_entry(lfe, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + 1, + &mc_0), + "3500 via replicate over 10.10.10.1"); + + /* + * MPLS Bud-node. Add a replication via an interface-receieve path + */ + lfe = fib_table_entry_path_add(lfib_index, + &pfx_3500, + FIB_SOURCE_API, + FIB_ENTRY_FLAG_MULTICAST, + FIB_PROTOCOL_IP4, + NULL, + tm->hw[0]->sw_if_index, + ~0, // invalid fib index + 0, + NULL, + FIB_ROUTE_PATH_INTF_RX); + FIB_TEST(fib_test_validate_entry(lfe, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + 2, + &mc_0, + &mc_intf_0), + "3500 via replicate over 10.10.10.1 and interface-rx"); + + /* + * Add a replication via an interface-free for-us path + */ + fib_test_rep_bucket_t mc_disp = { + .type = FT_REP_DISP_MFIB_LOOKUP, + .adj = { + .adj = idpo.dpoi_index, + }, + }; + lfe = fib_table_entry_path_add(lfib_index, + &pfx_3500, + FIB_SOURCE_API, + FIB_ENTRY_FLAG_MULTICAST, + FIB_PROTOCOL_IP4, + NULL, + 5, // rpf-id + 0, // default table + 0, + NULL, + FIB_ROUTE_PATH_RPF_ID); + FIB_TEST(fib_test_validate_entry(lfe, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + 3, + &mc_0, + &mc_disp, + &mc_intf_0), + "3500 via replicate over 10.10.10.1 and interface-rx"); + + + + fib_table_entry_delete(fib_index, &pfx_3500, FIB_SOURCE_API); + dpo_reset(&idpo); + /* * cleanup */ @@ -7617,6 +7883,9 @@ lfib_test (void) FIB_TEST(lb_count == pool_elts(load_balance_pool), "Load-balance resources freed %d of %d", lb_count, pool_elts(load_balance_pool)); + FIB_TEST(0 == pool_elts(interface_dpo_pool), + "interface_dpo resources freed %d of %d", + 0, pool_elts(interface_dpo_pool)); return (0); } diff --git a/src/vnet/fib/fib_test.h b/src/vnet/fib/fib_test.h new file mode 100644 index 00000000..b98680bf --- /dev/null +++ b/src/vnet/fib/fib_test.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __FIB_TEST_H__ +#define __FIB_TEST_H__ + +#include + +typedef enum fib_test_lb_bucket_type_t_ { + FT_LB_LABEL_O_ADJ, + FT_LB_LABEL_STACK_O_ADJ, + FT_LB_LABEL_O_LB, + FT_LB_O_LB, + FT_LB_SPECIAL, + FT_LB_ADJ, + FT_LB_INTF, +} fib_test_lb_bucket_type_t; + +typedef struct fib_test_lb_bucket_t_ { + fib_test_lb_bucket_type_t type; + + union + { + struct + { + mpls_eos_bit_t eos; + mpls_label_t label; + u8 ttl; + adj_index_t adj; + } label_o_adj; + struct + { + mpls_eos_bit_t eos; + mpls_label_t label_stack[8]; + u8 label_stack_size; + u8 ttl; + adj_index_t adj; + } label_stack_o_adj; + struct + { + mpls_eos_bit_t eos; + mpls_label_t label; + u8 ttl; + index_t lb; + } label_o_lb; + struct + { + index_t adj; + } adj; + struct + { + index_t lb; + } lb; + struct + { + index_t adj; + } special; + }; +} fib_test_lb_bucket_t; + +typedef enum fib_test_rep_bucket_type_t_ { + FT_REP_LABEL_O_ADJ, + FT_REP_INTF, +} fib_test_rep_bucket_type_t; + +typedef struct fib_test_rep_bucket_t_ { + fib_test_rep_bucket_type_t type; + + union + { + struct + { + mpls_eos_bit_t eos; + mpls_label_t label; + u8 ttl; + adj_index_t adj; + } label_o_adj; + struct + { + adj_index_t adj; + } adj; + }; +} fib_test_rep_bucket_t; + + +extern int fib_test_validate_rep_v(const replicate_t *rep, + u16 n_buckets, + va_list ap); + +extern int fib_test_validate_lb_v(const load_balance_t *lb, + u16 n_buckets, + va_list ap); + +extern int fib_test_validate_entry(fib_node_index_t fei, + fib_forward_chain_type_t fct, + u16 n_buckets, + ...); + +#endif diff --git a/src/vnet/fib/fib_types.c b/src/vnet/fib/fib_types.c index 2837a59d..8165f3eb 100644 --- a/src/vnet/fib/fib_types.c +++ b/src/vnet/fib/fib_types.c @@ -66,12 +66,13 @@ fib_prefix_from_ip46_addr (const ip46_address_t *addr, void fib_prefix_from_mpls_label (mpls_label_t label, + mpls_eos_bit_t eos, fib_prefix_t *pfx) { pfx->fp_proto = FIB_PROTOCOL_MPLS; pfx->fp_len = 21; pfx->fp_label = label; - pfx->fp_eos = MPLS_NON_EOS; + pfx->fp_eos = eos; } int @@ -194,17 +195,7 @@ fib_route_path_cmp (const fib_route_path_t *rpath1, if (0 != res) return (res); - if (~0 != rpath1->frp_sw_if_index && - ~0 != rpath2->frp_sw_if_index) - { - res = vnet_sw_interface_compare(vnet_get_main(), - rpath1->frp_sw_if_index, - rpath2->frp_sw_if_index); - } - else - { - res = rpath1->frp_sw_if_index - rpath2->frp_sw_if_index; - } + res = (rpath1->frp_sw_if_index - rpath2->frp_sw_if_index); if (0 != res) return (res); diff --git a/src/vnet/fib/fib_types.h b/src/vnet/fib/fib_types.h index 1c5299a9..4cb73e8a 100644 --- a/src/vnet/fib/fib_types.h +++ b/src/vnet/fib/fib_types.h @@ -286,8 +286,36 @@ typedef enum fib_route_path_flags_t_ * Attached path */ FIB_ROUTE_PATH_ATTACHED = (1 << 3), + /** + * A Drop path - resolve the path on the drop DPO + */ + FIB_ROUTE_PATH_DROP = (1 << 4), + /** + * Don't resolve the path, use the DPO the client provides + */ + FIB_ROUTE_PATH_EXCLUSIVE = (1 << 5), + /** + * A path that result in received traffic being recieved/recirculated + * so that it appears to have arrived on the new interface + */ + FIB_ROUTE_PATH_INTF_RX = (1 << 6), + /** + * A local path with a RPF-ID => multicast traffic + */ + FIB_ROUTE_PATH_RPF_ID = (1 << 7), } fib_route_path_flags_t; +/** + * An RPF-ID is numerical value that is used RPF validate. An entry + * has-a RPF-ID, when a packet egress from (e.g. an LSP) it gains an + * RPF-ID, these two are compared for the RPF check. + * This replaces the interfce based chack (since the LSP has no associated + * interface. + */ +typedef u32 fib_rpf_id_t; + +#define MFIB_RPF_ID_NONE (0) + /** * @brief * A representation of a path as described by a route producer. @@ -321,17 +349,29 @@ typedef struct fib_route_path_t_ { */ ip46_address_t frp_addr; - /** - * The MPLS local Label to reursively resolve through. - * This is valid when the path type is MPLS. - */ - mpls_label_t frp_local_label; + struct { + /** + * The MPLS local Label to reursively resolve through. + * This is valid when the path type is MPLS. + */ + mpls_label_t frp_local_label; + /** + * EOS bit for the resolving label + */ + mpls_eos_bit_t frp_eos; + }; + }; + union { + /** + * The interface. + * Will be invalid for recursive paths. + */ + u32 frp_sw_if_index; + /** + * The RPF-ID + */ + fib_rpf_id_t frp_rpf_id; }; - /** - * The interface. - * Will be invalid for recursive paths. - */ - u32 frp_sw_if_index; /** * The FIB index to lookup the nexthop * Only valid for recursive paths. diff --git a/src/vnet/fib/mpls_fib.c b/src/vnet/fib/mpls_fib.c index 4b2b76ea..19f9f3c1 100644 --- a/src/vnet/fib/mpls_fib.c +++ b/src/vnet/fib/mpls_fib.c @@ -165,6 +165,7 @@ mpls_fib_create_with_table_id (u32 table_id) lookup_dpo_add_or_lock_w_fib_index(0, // unused DPO_PROTO_IP4, + LOOKUP_UNICAST, LOOKUP_INPUT_DST_ADDR, LOOKUP_TABLE_FROM_INPUT_INTERFACE, &dpo); @@ -179,6 +180,7 @@ mpls_fib_create_with_table_id (u32 table_id) lookup_dpo_add_or_lock_w_fib_index(0, //unsued DPO_PROTO_MPLS, + LOOKUP_UNICAST, LOOKUP_INPUT_DST_ADDR, LOOKUP_TABLE_FROM_INPUT_INTERFACE, &dpo); @@ -197,6 +199,7 @@ mpls_fib_create_with_table_id (u32 table_id) lookup_dpo_add_or_lock_w_fib_index(0, //unused DPO_PROTO_IP6, + LOOKUP_UNICAST, LOOKUP_INPUT_DST_ADDR, LOOKUP_TABLE_FROM_INPUT_INTERFACE, &dpo); @@ -210,6 +213,7 @@ mpls_fib_create_with_table_id (u32 table_id) prefix.fp_eos = MPLS_NON_EOS; lookup_dpo_add_or_lock_w_fib_index(0, // unsued DPO_PROTO_MPLS, + LOOKUP_UNICAST, LOOKUP_INPUT_DST_ADDR, LOOKUP_TABLE_FROM_INPUT_INTERFACE, &dpo); @@ -320,8 +324,15 @@ mpls_fib_forwarding_table_update (mpls_fib_t *mf, { mpls_label_t key; - ASSERT(DPO_LOAD_BALANCE == dpo->dpoi_type); - + ASSERT((DPO_LOAD_BALANCE == dpo->dpoi_type) || + (DPO_REPLICATE == dpo->dpoi_type)); + if (CLIB_DEBUG > 0) + { + if (DPO_REPLICATE == dpo->dpoi_type) + ASSERT(dpo->dpoi_index & MPLS_IS_REPLICATE); + if (DPO_LOAD_BALANCE == dpo->dpoi_type) + ASSERT(!(dpo->dpoi_index & MPLS_IS_REPLICATE)); + } key = mpls_fib_entry_mk_key(label, eos); mf->mf_lbs[key] = dpo->dpoi_index; diff --git a/src/vnet/handoff.h b/src/vnet/handoff.h index 815206a9..04ba8bfb 100644 --- a/src/vnet/handoff.h +++ b/src/vnet/handoff.h @@ -150,7 +150,7 @@ eth_get_sym_key (ethernet_header_t * h0) ip->dst_address.as_u64[0] ^ ip->dst_address.as_u64[1] ^ ip->protocol); } - else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST)) + else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) { hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1)); } @@ -179,8 +179,7 @@ eth_get_sym_key (ethernet_header_t * h0) ip->dst_address.as_u64[0] ^ ip->dst_address.as_u64[1] ^ ip->protocol); } - else if (outer->type == - clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST)) + else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) { hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1)); } @@ -210,7 +209,7 @@ eth_get_key (ethernet_header_t * h0) { hash_key = ipv6_get_key ((ip6_header_t *) (h0 + 1)); } - else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST)) + else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) { hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1)); } @@ -230,8 +229,7 @@ eth_get_key (ethernet_header_t * h0) { hash_key = ipv6_get_key ((ip6_header_t *) (outer + 1)); } - else if (outer->type == - clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST)) + else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) { hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1)); } diff --git a/src/vnet/interface.c b/src/vnet/interface.c index 2a1e70e8..45417b2f 100644 --- a/src/vnet/interface.c +++ b/src/vnet/interface.c @@ -1360,7 +1360,7 @@ vnet_link_to_l3_proto (vnet_link_t link) case VNET_LINK_IP6: return (VNET_L3_PACKET_TYPE_IP6); case VNET_LINK_MPLS: - return (VNET_L3_PACKET_TYPE_MPLS_UNICAST); + return (VNET_L3_PACKET_TYPE_MPLS); case VNET_LINK_ARP: return (VNET_L3_PACKET_TYPE_ARP); case VNET_LINK_ETHERNET: diff --git a/src/vnet/ip/ip.api b/src/vnet/ip/ip.api index 5c2df32c..6af1714f 100644 --- a/src/vnet/ip/ip.api +++ b/src/vnet/ip/ip.api @@ -478,6 +478,7 @@ define ip_mroute_add_del u32 table_id; u32 entry_flags; u32 itf_flags; + u32 rpf_id; u16 grp_address_length; u8 create_vrf_if_needed; u8 is_add; @@ -518,6 +519,8 @@ manual_endian manual_print define ip_mfib_details { u32 context; u32 table_id; + u32 entry_flags; + u32 rpf_id; u8 address_length; u8 grp_address[4]; u8 src_address[4]; diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c index fdfe7f63..9fdf9b3c 100644 --- a/src/vnet/ip/ip4_forward.c +++ b/src/vnet/ip/ip4_forward.c @@ -2752,6 +2752,16 @@ ip4_rewrite_mcast (vlib_main_t * vm, return ip4_rewrite_inline (vm, node, frame, 0, 0, 1); } +static uword +ip4_mcast_midchain (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + if (adj_are_counters_enabled ()) + return ip4_rewrite_inline (vm, node, frame, 1, 1, 1); + else + return ip4_rewrite_inline (vm, node, frame, 0, 1, 1); +} + /* *INDENT-OFF* */ VLIB_REGISTER_NODE (ip4_rewrite_node) = { .function = ip4_rewrite, @@ -2778,6 +2788,16 @@ VLIB_REGISTER_NODE (ip4_rewrite_mcast_node) = { }; VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_mcast_node, ip4_rewrite_mcast) +VLIB_REGISTER_NODE (ip4_mcast_midchain_node, static) = { + .function = ip4_mcast_midchain, + .name = "ip4-mcast-midchain", + .vector_size = sizeof (u32), + + .format_trace = format_ip4_rewrite_trace, + .sibling_of = "ip4-rewrite", +}; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_mcast_midchain_node, ip4_mcast_midchain) + VLIB_REGISTER_NODE (ip4_midchain_node) = { .function = ip4_midchain, .name = "ip4-midchain", diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index c2fc4f87..a369f79f 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -2246,6 +2246,16 @@ ip6_midchain (vlib_main_t * vm, return ip6_rewrite_inline (vm, node, frame, 0, 1, 0); } +static uword +ip6_mcast_midchain (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + if (adj_are_counters_enabled ()) + return ip6_rewrite_inline (vm, node, frame, 1, 1, 1); + else + return ip6_rewrite_inline (vm, node, frame, 1, 1, 1); +} + /* *INDENT-OFF* */ VLIB_REGISTER_NODE (ip6_midchain_node) = { @@ -2290,6 +2300,19 @@ VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) = VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_mcast_node, ip6_rewrite_mcast); +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (ip6_mcast_midchain_node, static) = +{ + .function = ip6_mcast_midchain, + .name = "ip6-mcast-midchain", + .vector_size = sizeof (u32), + .format_trace = format_ip6_rewrite_trace, + .sibling_of = "ip6-rewrite", +}; +/* *INDENT-ON* */ + +VLIB_NODE_FUNCTION_MULTIARCH (ip6_mcast_midchain_node, ip6_mcast_midchain); + /* * Hop-by-Hop handling */ diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c index 2af546df..58b997aa 100644 --- a/src/vnet/ip/ip6_neighbor.c +++ b/src/vnet/ip/ip6_neighbor.c @@ -557,6 +557,7 @@ ip6_ethernet_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai) case IP_LOOKUP_NEXT_PUNT: case IP_LOOKUP_NEXT_LOCAL: case IP_LOOKUP_NEXT_REWRITE: + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: case IP_LOOKUP_NEXT_MIDCHAIN: case IP_LOOKUP_NEXT_ICMP_ERROR: case IP_LOOKUP_N_NEXT: diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c index b9f1782b..9c9cb4a4 100644 --- a/src/vnet/ip/ip_api.c +++ b/src/vnet/ip/ip_api.c @@ -438,17 +438,20 @@ vl_api_ip6_fib_dump_t_handler (vl_api_ip6_fib_dump_t * mp) } static void -send_ip_mfib_details (vpe_api_main_t * am, - unix_shared_memory_queue_t * q, - u32 table_id, - mfib_prefix_t * pfx, - fib_route_path_encode_t * api_rpaths, u32 context) +send_ip_mfib_details (unix_shared_memory_queue_t * q, + u32 context, u32 table_id, fib_node_index_t mfei) { + fib_route_path_encode_t *api_rpath, *api_rpaths = NULL; vl_api_ip_mfib_details_t *mp; - fib_route_path_encode_t *api_rpath; + mfib_entry_t *mfib_entry; vl_api_fib_path_t *fp; + mfib_prefix_t pfx; int path_count; + mfib_entry = mfib_entry_get (mfei); + mfib_entry_get_prefix (mfei, &pfx); + mfib_entry_encode (mfei, &api_rpaths); + path_count = vec_len (api_rpaths); mp = vl_msg_api_alloc (sizeof (*mp) + path_count * sizeof (*fp)); if (!mp) @@ -457,12 +460,14 @@ send_ip_mfib_details (vpe_api_main_t * am, mp->_vl_msg_id = ntohs (VL_API_IP_FIB_DETAILS); mp->context = context; + mp->rpf_id = mfib_entry->mfe_rpf_id; + mp->entry_flags = mfib_entry->mfe_flags; mp->table_id = htonl (table_id); - mp->address_length = pfx->fp_len; - memcpy (mp->grp_address, &pfx->fp_grp_addr.ip4, - sizeof (pfx->fp_grp_addr.ip4)); - memcpy (mp->src_address, &pfx->fp_src_addr.ip4, - sizeof (pfx->fp_src_addr.ip4)); + mp->address_length = pfx.fp_len; + memcpy (mp->grp_address, &pfx.fp_grp_addr.ip4, + sizeof (pfx.fp_grp_addr.ip4)); + memcpy (mp->src_address, &pfx.fp_src_addr.ip4, + sizeof (pfx.fp_src_addr.ip4)); mp->count = htonl (path_count); fp = mp->path; @@ -475,6 +480,7 @@ send_ip_mfib_details (vpe_api_main_t * am, copy_fib_next_hop (api_rpath, fp); fp++; } + vec_free (api_rpaths); vl_msg_api_send_shmem (q, (u8 *) & mp); } @@ -497,13 +503,10 @@ vl_api_ip_mfib_table_dump_walk (fib_node_index_t fei, void *arg) static void vl_api_ip_mfib_dump_t_handler (vl_api_ip_mfib_dump_t * mp) { - vpe_api_main_t *am = &vpe_api_main; unix_shared_memory_queue_t *q; ip4_main_t *im = &ip4_main; mfib_table_t *mfib_table; fib_node_index_t *mfeip; - mfib_prefix_t pfx; - fib_route_path_encode_t *api_rpaths = NULL; vl_api_ip_mfib_dump_ctc_t ctx = { .entries = NULL, }; @@ -524,21 +527,16 @@ vl_api_ip_mfib_dump_t_handler (vl_api_ip_mfib_dump_t * mp) vec_foreach (mfeip, ctx.entries) { - mfib_entry_get_prefix (*mfeip, &pfx); - mfib_entry_encode (*mfeip, &api_rpaths); - send_ip_mfib_details (am, q, + send_ip_mfib_details (q, mp->context, mfib_table->mft_table_id, - &pfx, api_rpaths, - mp->context); + *mfeip); } - vec_reset_length (api_rpaths); vec_reset_length (ctx.entries); })); /* *INDENT-ON* */ vec_free (ctx.entries); - vec_free (api_rpaths); } static void @@ -705,10 +703,13 @@ add_del_route_t_handler (u8 is_multipath, u8 is_unreach, u8 is_prohibit, u8 is_local, + u8 is_multicast, u8 is_classify, u32 classify_table_index, u8 is_resolve_host, u8 is_resolve_attached, + u8 is_interface_rx, + u8 is_rpf_id, u32 fib_index, const fib_prefix_t * prefix, u8 next_hop_proto_is_ip4, @@ -731,16 +732,24 @@ add_del_route_t_handler (u8 is_multipath, .frp_label_stack = next_hop_out_label_stack, }; fib_route_path_t *paths = NULL; + fib_entry_flag_t entry_flags = FIB_ENTRY_FLAG_NONE; if (MPLS_LABEL_INVALID != next_hop_via_label) { path.frp_proto = FIB_PROTOCOL_MPLS; path.frp_local_label = next_hop_via_label; + path.frp_eos = MPLS_NON_EOS; } if (is_resolve_host) path_flags |= FIB_ROUTE_PATH_RESOLVE_VIA_HOST; if (is_resolve_attached) path_flags |= FIB_ROUTE_PATH_RESOLVE_VIA_ATTACHED; + if (is_interface_rx) + path_flags |= FIB_ROUTE_PATH_INTF_RX; + if (is_rpf_id) + path_flags |= FIB_ROUTE_PATH_RPF_ID; + if (is_multicast) + entry_flags |= FIB_ENTRY_FLAG_MULTICAST; path.frp_flags = path_flags; @@ -754,8 +763,7 @@ add_del_route_t_handler (u8 is_multipath, if (is_add) fib_table_entry_path_add2 (fib_index, prefix, - FIB_SOURCE_API, - FIB_ENTRY_FLAG_NONE, paths); + FIB_SOURCE_API, entry_flags, paths); else fib_table_entry_path_remove2 (fib_index, prefix, FIB_SOURCE_API, paths); @@ -826,8 +834,7 @@ add_del_route_t_handler (u8 is_multipath, { vec_add1 (paths, path); fib_table_entry_update (fib_index, - prefix, - FIB_SOURCE_API, FIB_ENTRY_FLAG_NONE, paths); + prefix, FIB_SOURCE_API, entry_flags, paths); vec_free (paths); } else @@ -847,7 +854,7 @@ add_del_route_check (fib_protocol_t table_proto, fib_protocol_t next_hop_table_proto, u32 next_hop_table_id, u8 create_missing_tables, - u32 * fib_index, u32 * next_hop_fib_index) + u8 is_rpf_id, u32 * fib_index, u32 * next_hop_fib_index) { vnet_main_t *vnm = vnet_get_main (); @@ -866,7 +873,7 @@ add_del_route_check (fib_protocol_t table_proto, } } - if (~0 != ntohl (next_hop_sw_if_index)) + if (!is_rpf_id && ~0 != ntohl (next_hop_sw_if_index)) { if (pool_is_free_index (vnm->interface_main.sw_interfaces, ntohl (next_hop_sw_if_index))) @@ -876,16 +883,27 @@ add_del_route_check (fib_protocol_t table_proto, } else { - *next_hop_fib_index = fib_table_find (next_hop_table_proto, - ntohl (next_hop_table_id)); + if (is_rpf_id) + *next_hop_fib_index = mfib_table_find (next_hop_table_proto, + ntohl (next_hop_table_id)); + else + *next_hop_fib_index = fib_table_find (next_hop_table_proto, + ntohl (next_hop_table_id)); if (~0 == *next_hop_fib_index) { if (create_missing_tables) { - *next_hop_fib_index = - fib_table_find_or_create_and_lock (next_hop_table_proto, - ntohl (next_hop_table_id)); + if (is_rpf_id) + *next_hop_fib_index = + mfib_table_find_or_create_and_lock (next_hop_table_proto, + ntohl + (next_hop_table_id)); + else + *next_hop_fib_index = + fib_table_find_or_create_and_lock (next_hop_table_proto, + ntohl + (next_hop_table_id)); } else { @@ -910,7 +928,7 @@ ip4_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp) mp->next_hop_sw_if_index, FIB_PROTOCOL_IP4, mp->next_hop_table_id, - mp->create_vrf_if_needed, + mp->create_vrf_if_needed, 0, &fib_index, &next_hop_fib_index); if (0 != rv) @@ -943,11 +961,11 @@ ip4_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp) mp->is_drop, mp->is_unreach, mp->is_prohibit, - mp->is_local, + mp->is_local, 0, mp->is_classify, mp->classify_table_index, mp->is_resolve_host, - mp->is_resolve_attached, + mp->is_resolve_attached, 0, 0, fib_index, &pfx, 1, &nh, ntohl (mp->next_hop_sw_if_index), @@ -969,7 +987,7 @@ ip6_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp) mp->next_hop_sw_if_index, FIB_PROTOCOL_IP6, mp->next_hop_table_id, - mp->create_vrf_if_needed, + mp->create_vrf_if_needed, 0, &fib_index, &next_hop_fib_index); if (0 != rv) @@ -1002,11 +1020,11 @@ ip6_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp) mp->is_drop, mp->is_unreach, mp->is_prohibit, - mp->is_local, + mp->is_local, 0, mp->is_classify, mp->classify_table_index, mp->is_resolve_host, - mp->is_resolve_attached, + mp->is_resolve_attached, 0, 0, fib_index, &pfx, 0, &nh, ntohl (mp->next_hop_sw_if_index), next_hop_fib_index, @@ -1075,6 +1093,7 @@ mroute_add_del_handler (u8 is_add, u32 fib_index, const mfib_prefix_t * prefix, u32 entry_flags, + fib_rpf_id_t rpf_id, u32 next_hop_sw_if_index, u32 itf_flags) { stats_dslock_with_hint (1 /* release hint */ , 2 /* tag */ ); @@ -1091,7 +1110,7 @@ mroute_add_del_handler (u8 is_add, if (!is_local && ~0 == next_hop_sw_if_index) { mfib_table_entry_update (fib_index, prefix, - MFIB_SOURCE_API, entry_flags); + MFIB_SOURCE_API, rpf_id, entry_flags); } else { @@ -1152,6 +1171,7 @@ api_mroute_add_del_t_handler (vl_api_ip_mroute_add_del_t * mp) mp->is_local, fib_index, &pfx, ntohl (mp->entry_flags), + ntohl (mp->rpf_id), ntohl (mp->next_hop_sw_if_index), ntohl (mp->itf_flags))); } diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c index ec9a1f97..597de06b 100755 --- a/src/vnet/ip/lookup.c +++ b/src/vnet/ip/lookup.c @@ -450,6 +450,7 @@ vnet_ip_route_cmd (vlib_main_t * vm, unformat_mpls_unicast_label, &rpath.frp_local_label)) { rpath.frp_weight = 1; + rpath.frp_eos = MPLS_NON_EOS; rpath.frp_proto = FIB_PROTOCOL_MPLS; rpath.frp_sw_if_index = ~0; vec_add1 (rpaths, rpath); @@ -923,7 +924,7 @@ vnet_ip_mroute_cmd (vlib_main_t * vm, else if (eflags) { mfib_table_entry_update (fib_index, &pfx, MFIB_SOURCE_CLI, - eflags); + MFIB_RPF_ID_NONE, eflags); } else { diff --git a/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c b/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c index efa724e0..d2954e96 100644 --- a/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c +++ b/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c @@ -88,6 +88,7 @@ ip_dst_fib_add_route (u32 dst_fib_index, const ip_prefix_t * dst_prefix) (ip_prefix_version (dst_prefix) == IP6 ? DPO_PROTO_IP6 : DPO_PROTO_IP4), + LOOKUP_UNICAST, LOOKUP_INPUT_SRC_ADDR, LOOKUP_TABLE_FROM_CONFIG, &src_lkup_dpo); diff --git a/src/vnet/mfib/ip4_mfib.c b/src/vnet/mfib/ip4_mfib.c index 164cafa1..3ed7cba7 100644 --- a/src/vnet/mfib/ip4_mfib.c +++ b/src/vnet/mfib/ip4_mfib.c @@ -72,6 +72,7 @@ ip4_create_mfib_with_table_id (u32 table_id) mfib_table_entry_update(mfib_table->mft_index, &prefix, MFIB_SOURCE_DEFAULT_ROUTE, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_DROP); } diff --git a/src/vnet/mfib/ip6_mfib.c b/src/vnet/mfib/ip6_mfib.c index 991b91c6..116fee22 100644 --- a/src/vnet/mfib/ip6_mfib.c +++ b/src/vnet/mfib/ip6_mfib.c @@ -195,6 +195,7 @@ ip6_create_mfib_with_table_id (u32 table_id) mfib_table_entry_update(mfib_table->mft_index, &all_zeros, MFIB_SOURCE_DEFAULT_ROUTE, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_DROP); /* diff --git a/src/vnet/mfib/mfib_entry.c b/src/vnet/mfib/mfib_entry.c index 1aa8e086..847f25e7 100644 --- a/src/vnet/mfib/mfib_entry.c +++ b/src/vnet/mfib/mfib_entry.c @@ -48,6 +48,15 @@ #define MFIB_ENTRY_DBG(_e, _fmt, _args...) #endif +/** + * MFIB extensions to each path + */ +typedef struct mfib_path_ext_t_ +{ + mfib_itf_flags_t mfpe_flags; + fib_node_index_t mfpe_path; +} mfib_path_ext_t; + /** * The source of an MFIB entry */ @@ -58,22 +67,39 @@ typedef struct mfib_entry_src_t_ */ mfib_source_t mfes_src; + /** + * Route flags + */ + mfib_entry_flags_t mfes_flags; + /** * The path-list of forwarding interfaces */ fib_node_index_t mfes_pl; /** - * Route flags + * RPF-ID */ - mfib_entry_flags_t mfes_flags; + fib_rpf_id_t mfes_rpf_id; + + /** + * Hash table of path extensions + */ + mfib_path_ext_t *mfes_exts; /** - * The hash table of all interfaces + * The hash table of all interfaces. + * This is forwarding time information derived from the paths + * and their extensions. */ mfib_itf_t *mfes_itfs; } mfib_entry_src_t; +/** + * Pool of path extensions + */ +static mfib_path_ext_t *mfib_path_ext_pool; + /** * String names for each source */ @@ -123,6 +149,24 @@ format_mfib_entry_dpo (u8 * s, va_list * args) MFIB_ENTRY_FORMAT_BRIEF)); } +static inline mfib_path_ext_t * +mfib_entry_path_ext_get (index_t mi) +{ + return (pool_elt_at_index(mfib_path_ext_pool, mi)); +} + +static u8 * +format_mfib_entry_path_ext (u8 * s, va_list * args) +{ + mfib_path_ext_t *path_ext; + index_t mpi = va_arg(*args, index_t); + + path_ext = mfib_entry_path_ext_get(mpi); + return (format(s, "path:%d flags:%U", + path_ext->mfpe_path, + format_mfib_itf_flags, path_ext->mfpe_flags)); +} + u8 * format_mfib_entry (u8 * s, va_list * args) { @@ -141,6 +185,8 @@ format_mfib_entry (u8 * s, va_list * args) if (level >= MFIB_ENTRY_FORMAT_DETAIL) { + fib_node_index_t path_index, mpi; + s = format (s, "\n"); s = format (s, " fib:%d", mfib_entry->mfe_fib_index); s = format (s, " index:%d", mfib_entry_get_index(mfib_entry)); @@ -153,6 +199,14 @@ format_mfib_entry (u8 * s, va_list * args) { s = fib_path_list_format(msrc->mfes_pl, s); } + s = format (s, " Extensions:\n", + mfib_source_names[msrc->mfes_src]); + hash_foreach(path_index, mpi, msrc->mfes_exts, + ({ + s = format(s, " %U\n", format_mfib_entry_path_ext, mpi); + })); + s = format (s, " Interface-Forwarding:\n", + mfib_source_names[msrc->mfes_src]); hash_foreach(sw_if_index, mfi, msrc->mfes_itfs, ({ s = format(s, " %U\n", format_mfib_itf, mfi); @@ -165,7 +219,7 @@ format_mfib_entry (u8 * s, va_list * args) ({ s = format(s, "\n %U", format_mfib_itf, mfi); })); - + s = format(s, "\n RPF-ID:%d", mfib_entry->mfe_rpf_id); s = format(s, "\n %U-chain\n %U", format_fib_forw_chain_type, mfib_entry_get_default_chain_type(mfib_entry), @@ -314,13 +368,6 @@ mfib_entry_src_remove (mfib_entry_t *mfib_entry, } } -static int -mfib_entry_src_n_itfs (const mfib_entry_src_t *msrc) -{ - return (hash_elts(msrc->mfes_itfs)); -} - - static void mfib_entry_last_lock_gone (fib_node_t *node) { @@ -338,7 +385,6 @@ mfib_entry_last_lock_gone (fib_node_t *node) mfib_entry_src_flush(msrc); } - fib_path_list_unlock(mfib_entry->mfe_parent); vec_free(mfib_entry->mfe_srcs); fib_node_deinit(&mfib_entry->mfe_node); @@ -417,10 +463,9 @@ mfib_entry_alloc (u32 fib_index, mfib_entry->mfe_flags = 0; mfib_entry->mfe_fib_index = fib_index; mfib_entry->mfe_prefix = *prefix; - mfib_entry->mfe_parent = FIB_NODE_INDEX_INVALID; - mfib_entry->mfe_sibling = FIB_NODE_INDEX_INVALID; mfib_entry->mfe_srcs = NULL; mfib_entry->mfe_itfs = NULL; + mfib_entry->mfe_rpf_id = MFIB_RPF_ID_NONE; dpo_reset(&mfib_entry->mfe_rep); @@ -431,10 +476,57 @@ mfib_entry_alloc (u32 fib_index, return (mfib_entry); } +static inline mfib_path_ext_t * +mfib_entry_path_ext_find (mfib_path_ext_t *exts, + fib_node_index_t path_index) +{ + uword *p; + + p = hash_get(exts, path_index); + + if (NULL != p) + { + return (mfib_entry_path_ext_get(p[0])); + } + + return (NULL); +} + +static mfib_path_ext_t* +mfib_path_ext_add (mfib_entry_src_t *msrc, + fib_node_index_t path_index, + mfib_itf_flags_t mfi_flags) +{ + mfib_path_ext_t *path_ext; + + pool_get(mfib_path_ext_pool, path_ext); + + path_ext->mfpe_flags = mfi_flags; + path_ext->mfpe_path = path_index; + + hash_set(msrc->mfes_exts, path_index, + path_ext - mfib_path_ext_pool); + + return (path_ext); +} + +static void +mfib_path_ext_remove (mfib_entry_src_t *msrc, + fib_node_index_t path_index) +{ + mfib_path_ext_t *path_ext; + + path_ext = mfib_entry_path_ext_find(msrc->mfes_exts, path_index); + + hash_unset(msrc->mfes_exts, path_index); + pool_put(mfib_path_ext_pool, path_ext); +} + typedef struct mfib_entry_collect_forwarding_ctx_t_ { load_balance_path_t * next_hops; fib_forward_chain_type_t fct; + mfib_entry_src_t *msrc; } mfib_entry_collect_forwarding_ctx_t; static int @@ -455,6 +547,20 @@ mfib_entry_src_collect_forwarding (fib_node_index_t pl_index, return (!0); } + /* + * If the path is not forwarding to use it + */ + mfib_path_ext_t *path_ext; + + path_ext = mfib_entry_path_ext_find(ctx->msrc->mfes_exts, + path_index); + + if (NULL != path_ext && + !(path_ext->mfpe_flags & MFIB_ITF_FLAG_FORWARD)) + { + return (!0); + } + switch (ctx->fct) { case FIB_FORW_CHAIN_TYPE_MCAST_IP4: @@ -483,46 +589,61 @@ mfib_entry_src_collect_forwarding (fib_node_index_t pl_index, } static void -mfib_entry_stack (mfib_entry_t *mfib_entry) +mfib_entry_stack (mfib_entry_t *mfib_entry, + mfib_entry_src_t *msrc) { dpo_proto_t dp; dp = fib_proto_to_dpo(mfib_entry_get_proto(mfib_entry)); - if (FIB_NODE_INDEX_INVALID != mfib_entry->mfe_parent) + if (NULL != msrc && + FIB_NODE_INDEX_INVALID != msrc->mfes_pl) { mfib_entry_collect_forwarding_ctx_t ctx = { .next_hops = NULL, .fct = mfib_entry_get_default_chain_type(mfib_entry), + .msrc = msrc, }; - fib_path_list_walk(mfib_entry->mfe_parent, + fib_path_list_walk(msrc->mfes_pl, mfib_entry_src_collect_forwarding, &ctx); if (!(MFIB_ENTRY_FLAG_EXCLUSIVE & mfib_entry->mfe_flags)) { - /* - * each path contirbutes a next-hop. form a replicate - * from those choices. - */ - if (!dpo_id_is_valid(&mfib_entry->mfe_rep) || - dpo_is_drop(&mfib_entry->mfe_rep)) + if (NULL == ctx.next_hops) { - dpo_id_t tmp_dpo = DPO_INVALID; - - dpo_set(&tmp_dpo, - DPO_REPLICATE, dp, - replicate_create(0, dp)); - + /* + * no next-hops, stack directly on the drop + */ dpo_stack(DPO_MFIB_ENTRY, dp, &mfib_entry->mfe_rep, - &tmp_dpo); - - dpo_reset(&tmp_dpo); + drop_dpo_get(dp)); + } + else + { + /* + * each path contirbutes a next-hop. form a replicate + * from those choices. + */ + if (!dpo_id_is_valid(&mfib_entry->mfe_rep) || + dpo_is_drop(&mfib_entry->mfe_rep)) + { + dpo_id_t tmp_dpo = DPO_INVALID; + + dpo_set(&tmp_dpo, + DPO_REPLICATE, dp, + replicate_create(0, dp)); + + dpo_stack(DPO_MFIB_ENTRY, dp, + &mfib_entry->mfe_rep, + &tmp_dpo); + + dpo_reset(&tmp_dpo); + } + replicate_multipath_update(&mfib_entry->mfe_rep, + ctx.next_hops); } - replicate_multipath_update(&mfib_entry->mfe_rep, - ctx.next_hops); } else { @@ -548,11 +669,11 @@ mfib_entry_stack (mfib_entry_t *mfib_entry) } } -static void -mfib_entry_forwarding_path_add (mfib_entry_src_t *msrc, - const fib_route_path_t *rpath) +static fib_node_index_t +mfib_entry_src_path_add (mfib_entry_src_t *msrc, + const fib_route_path_t *rpath) { - fib_node_index_t old_pl_index; + fib_node_index_t path_index; fib_route_path_t *rpaths; ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_flags)); @@ -563,32 +684,26 @@ mfib_entry_forwarding_path_add (mfib_entry_src_t *msrc, rpaths = NULL; vec_add1(rpaths, rpath[0]); - old_pl_index = msrc->mfes_pl; - if (FIB_NODE_INDEX_INVALID == msrc->mfes_pl) { - msrc->mfes_pl = - fib_path_list_create(FIB_PATH_LIST_FLAG_NO_URPF, - rpaths); - } - else - { - msrc->mfes_pl = - fib_path_list_copy_and_path_add(msrc->mfes_pl, - FIB_PATH_LIST_FLAG_NO_URPF, - rpaths); + /* A non-shared path-list */ + msrc->mfes_pl = fib_path_list_create(FIB_PATH_LIST_FLAG_NO_URPF, + NULL); + fib_path_list_lock(msrc->mfes_pl); } - fib_path_list_lock(msrc->mfes_pl); - fib_path_list_unlock(old_pl_index); + + path_index = fib_path_list_path_add(msrc->mfes_pl, rpaths); vec_free(rpaths); + + return (path_index); } -static int -mfib_entry_forwarding_path_remove (mfib_entry_src_t *msrc, - const fib_route_path_t *rpath) +static fib_node_index_t +mfib_entry_src_path_remove (mfib_entry_src_t *msrc, + const fib_route_path_t *rpath) { - fib_node_index_t old_pl_index; + fib_node_index_t path_index; fib_route_path_t *rpaths; ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_flags)); @@ -599,56 +714,31 @@ mfib_entry_forwarding_path_remove (mfib_entry_src_t *msrc, rpaths = NULL; vec_add1(rpaths, rpath[0]); - old_pl_index = msrc->mfes_pl; - - msrc->mfes_pl = - fib_path_list_copy_and_path_remove(msrc->mfes_pl, - FIB_PATH_LIST_FLAG_NONE, - rpaths); - - fib_path_list_lock(msrc->mfes_pl); - fib_path_list_unlock(old_pl_index); + path_index = fib_path_list_path_remove(msrc->mfes_pl, rpaths); vec_free(rpaths); - return (FIB_NODE_INDEX_INVALID != msrc->mfes_pl); + return (path_index); } static void mfib_entry_recalculate_forwarding (mfib_entry_t *mfib_entry) { - fib_node_index_t old_pl_index; mfib_entry_src_t *bsrc; - old_pl_index = mfib_entry->mfe_parent; - /* * copy the forwarding data from the bast source */ bsrc = mfib_entry_get_best_src(mfib_entry); - if (NULL == bsrc) - { - mfib_entry->mfe_parent = FIB_NODE_INDEX_INVALID; - } - else + if (NULL != bsrc) { - mfib_entry->mfe_parent = bsrc->mfes_pl; mfib_entry->mfe_flags = bsrc->mfes_flags; mfib_entry->mfe_itfs = bsrc->mfes_itfs; + mfib_entry->mfe_rpf_id = bsrc->mfes_rpf_id; } - /* - * re-stack the entry on the best forwarding info. - */ - if (old_pl_index != mfib_entry->mfe_parent || - FIB_NODE_INDEX_INVALID == old_pl_index) - { - mfib_entry_stack(mfib_entry); - - fib_path_list_lock(mfib_entry->mfe_parent); - fib_path_list_unlock(old_pl_index); - } + mfib_entry_stack(mfib_entry, bsrc); } @@ -656,6 +746,7 @@ fib_node_index_t mfib_entry_create (u32 fib_index, mfib_source_t source, const mfib_prefix_t *prefix, + fib_rpf_id_t rpf_id, mfib_entry_flags_t entry_flags) { fib_node_index_t mfib_entry_index; @@ -666,6 +757,7 @@ mfib_entry_create (u32 fib_index, &mfib_entry_index); msrc = mfib_entry_src_find_or_create(mfib_entry, source); msrc->mfes_flags = entry_flags; + msrc->mfes_rpf_id = rpf_id; mfib_entry_recalculate_forwarding(mfib_entry); @@ -682,13 +774,14 @@ static int mfib_entry_src_ok_for_delete (const mfib_entry_src_t *msrc) { return ((MFIB_ENTRY_FLAG_NONE == msrc->mfes_flags && - 0 == mfib_entry_src_n_itfs(msrc))); + 0 == fib_path_list_get_n_paths(msrc->mfes_pl))); } int mfib_entry_update (fib_node_index_t mfib_entry_index, mfib_source_t source, mfib_entry_flags_t entry_flags, + fib_rpf_id_t rpf_id, index_t repi) { mfib_entry_t *mfib_entry; @@ -697,6 +790,7 @@ mfib_entry_update (fib_node_index_t mfib_entry_index, mfib_entry = mfib_entry_get(mfib_entry_index); msrc = mfib_entry_src_find_or_create(mfib_entry, source); msrc->mfes_flags = entry_flags; + msrc->mfes_rpf_id = rpf_id; if (INDEX_INVALID != repi) { @@ -768,55 +862,79 @@ mfib_entry_path_update (fib_node_index_t mfib_entry_index, const fib_route_path_t *rpath, mfib_itf_flags_t itf_flags) { + fib_node_index_t path_index; + mfib_path_ext_t *path_ext; + mfib_itf_flags_t old, new; mfib_entry_t *mfib_entry; mfib_entry_src_t *msrc; - mfib_itf_t *mfib_itf; mfib_entry = mfib_entry_get(mfib_entry_index); ASSERT(NULL != mfib_entry); msrc = mfib_entry_src_find_or_create(mfib_entry, source); /* - * search for the interface in the current set + * add the path to the path-list. If it's a duplicate we'll get + * back the original path. + */ + path_index = mfib_entry_src_path_add(msrc, rpath); + + /* + * find the path extension for that path */ - mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs, - rpath[0].frp_sw_if_index); + path_ext = mfib_entry_path_ext_find(msrc->mfes_exts, path_index); - if (NULL == mfib_itf) + if (NULL == path_ext) { - /* - * this is a path we do not yet have. If it is forwarding then we - * add it to the replication set - */ - if (itf_flags & MFIB_ITF_FLAG_FORWARD) - { - mfib_entry_forwarding_path_add(msrc, rpath); - } - /* - * construct a new ITF for this entry's list - */ - mfib_entry_itf_add(msrc, - rpath[0].frp_sw_if_index, - mfib_itf_create(rpath[0].frp_sw_if_index, - itf_flags)); + old = MFIB_ITF_FLAG_NONE; + path_ext = mfib_path_ext_add(msrc, path_index, itf_flags); } else { - int was_forwarding = !!(mfib_itf->mfi_flags & MFIB_ITF_FLAG_FORWARD); - int is_forwarding = !!(itf_flags & MFIB_ITF_FLAG_FORWARD); + old = path_ext->mfpe_flags; + path_ext->mfpe_flags = itf_flags; + } - if (!was_forwarding && is_forwarding) - { - mfib_entry_forwarding_path_add(msrc, rpath); - } - else if (was_forwarding && !is_forwarding) + /* + * Has the path changed its contribution to the input interface set. + * Which only paths with interfaces can do... + */ + if (~0 != rpath[0].frp_sw_if_index) + { + mfib_itf_t *mfib_itf; + + new = itf_flags; + + if (old != new) { - mfib_entry_forwarding_path_remove(msrc, rpath); + if (MFIB_ITF_FLAG_NONE == new) + { + /* + * no more interface flags on this path, remove + * from the data-plane set + */ + mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index); + } + else if (MFIB_ITF_FLAG_NONE == old) + { + /* + * This interface is now contributing + */ + mfib_entry_itf_add(msrc, + rpath[0].frp_sw_if_index, + mfib_itf_create(rpath[0].frp_sw_if_index, + itf_flags)); + } + else + { + /* + * change of flag contributions + */ + mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs, + rpath[0].frp_sw_if_index); + /* Seen by packets inflight */ + mfib_itf->mfi_flags = new; + } } - /* - * packets in flight see these updates. - */ - mfib_itf->mfi_flags = itf_flags; } mfib_entry_recalculate_forwarding(mfib_entry); @@ -833,9 +951,9 @@ mfib_entry_path_remove (fib_node_index_t mfib_entry_index, mfib_source_t source, const fib_route_path_t *rpath) { + fib_node_index_t path_index; mfib_entry_t *mfib_entry; mfib_entry_src_t *msrc; - mfib_itf_t *mfib_itf; mfib_entry = mfib_entry_get(mfib_entry_index); ASSERT(NULL != mfib_entry); @@ -850,33 +968,23 @@ mfib_entry_path_remove (fib_node_index_t mfib_entry_index, } /* - * search for the interface in the current set + * remove the path from the path-list. If it's not there we'll get + * back invalid */ - mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs, - rpath[0].frp_sw_if_index); + path_index = mfib_entry_src_path_remove(msrc, rpath); - if (NULL == mfib_itf) + if (FIB_NODE_INDEX_INVALID != path_index) { /* - * removing a path that does not exist + * don't need the extension, nor the interface anymore */ - return (mfib_entry_ok_for_delete(mfib_entry)); - } - - /* - * we have this path. If it is forwarding then we - * remove it to the replication set - */ - if (mfib_itf->mfi_flags & MFIB_ITF_FLAG_FORWARD) - { - mfib_entry_forwarding_path_remove(msrc, rpath); + mfib_path_ext_remove(msrc, path_index); + if (~0 != rpath[0].frp_sw_if_index) + { + mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index); + } } - /* - * remove the interface/path from this entry's list - */ - mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index); - if (mfib_entry_src_ok_for_delete(msrc)) { /* @@ -1057,11 +1165,14 @@ mfib_entry_encode (fib_node_index_t mfib_entry_index, fib_route_path_encode_t **api_rpaths) { mfib_entry_t *mfib_entry; + mfib_entry_src_t *bsrc; mfib_entry = mfib_entry_get(mfib_entry_index); - if (FIB_NODE_INDEX_INVALID != mfib_entry->mfe_parent) + bsrc = mfib_entry_get_best_src(mfib_entry); + + if (FIB_NODE_INDEX_INVALID != bsrc->mfes_pl) { - fib_path_list_walk(mfib_entry->mfe_parent, + fib_path_list_walk(bsrc->mfes_pl, fib_path_encode, api_rpaths); } diff --git a/src/vnet/mfib/mfib_entry.h b/src/vnet/mfib/mfib_entry.h index dc8f49aa..4f62b18e 100644 --- a/src/vnet/mfib/mfib_entry.h +++ b/src/vnet/mfib/mfib_entry.h @@ -42,17 +42,6 @@ typedef struct mfib_entry_t_ { * The index of the FIB table this entry is in */ u32 mfe_fib_index; - /** - * the path-list for which this entry is a child. This is also the path-list - * that is contributing forwarding for this entry. - */ - fib_node_index_t mfe_parent; - /** - * index of this entry in the parent's child list. - * This is set when this entry is added as a child, but can also - * be changed by the parent as it manages its list. - */ - u32 mfe_sibling; /** * A vector of sources contributing forwarding @@ -65,7 +54,7 @@ typedef struct mfib_entry_t_ { CLIB_CACHE_LINE_ALIGN_MARK(cacheline1); /** - * The Replicate DPO used for forwarding. + * The DPO used for forwarding; replicate, drop, etc.. */ dpo_id_t mfe_rep; @@ -74,6 +63,11 @@ typedef struct mfib_entry_t_ { */ mfib_entry_flags_t mfe_flags; + /** + * RPF-ID used when the packets ingress not from an interface + */ + fib_rpf_id_t mfe_rpf_id; + /** * A hash table of interfaces */ @@ -90,11 +84,13 @@ extern u8 *format_mfib_entry(u8 * s, va_list * args); extern fib_node_index_t mfib_entry_create(u32 fib_index, mfib_source_t source, const mfib_prefix_t *prefix, + fib_rpf_id_t rpf_id, mfib_entry_flags_t entry_flags); extern int mfib_entry_update(fib_node_index_t fib_entry_index, mfib_source_t source, mfib_entry_flags_t entry_flags, + fib_rpf_id_t rpf_id, index_t rep_dpo); extern void mfib_entry_path_update(fib_node_index_t fib_entry_index, diff --git a/src/vnet/mfib/mfib_forward.c b/src/vnet/mfib/mfib_forward.c index 5fe0a57c..3d8f4f98 100644 --- a/src/vnet/mfib/mfib_forward.c +++ b/src/vnet/mfib/mfib_forward.c @@ -380,13 +380,27 @@ mfib_forward_rpf (vlib_main_t * vm, * for the case of throughput traffic that is not replicated * to the host stack nor sets local flags */ - if (PREDICT_TRUE(NULL != mfi0)) + + /* + * If the mfib entry has a configured RPF-ID check that + * in preference to an interface based RPF + */ + if (MFIB_RPF_ID_NONE != mfe0->mfe_rpf_id) { - iflags0 = mfi0->mfi_flags; + iflags0 = (mfe0->mfe_rpf_id == vnet_buffer(b0)->ip.rpf_id ? + MFIB_ITF_FLAG_ACCEPT : + MFIB_ITF_FLAG_NONE); } else { - iflags0 = MFIB_ITF_FLAG_NONE; + if (PREDICT_TRUE(NULL != mfi0)) + { + iflags0 = mfi0->mfi_flags; + } + else + { + iflags0 = MFIB_ITF_FLAG_NONE; + } } eflags0 = mfe0->mfe_flags; @@ -436,17 +450,16 @@ mfib_forward_rpf (vlib_main_t * vm, { mfib_forward_rpf_trace_t *t0; - t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0])); + t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); t0->entry_index = mfei0; + t0->itf_flags = iflags0; if (NULL == mfi0) { t0->sw_if_index = ~0; - t0->itf_flags = MFIB_ITF_FLAG_NONE; } else { t0->sw_if_index = mfi0->mfi_sw_if_index; - t0->itf_flags = mfi0->mfi_flags; } } vlib_validate_buffer_enqueue_x1 (vm, node, next, @@ -478,7 +491,7 @@ VLIB_REGISTER_NODE (ip4_mfib_forward_rpf_node, static) = { .n_next_nodes = MFIB_FORWARD_RPF_N_NEXT, .next_nodes = { - [MFIB_FORWARD_RPF_NEXT_DROP] = "error-drop", + [MFIB_FORWARD_RPF_NEXT_DROP] = "ip4-drop", }, }; @@ -503,7 +516,7 @@ VLIB_REGISTER_NODE (ip6_mfib_forward_rpf_node, static) = { .n_next_nodes = MFIB_FORWARD_RPF_N_NEXT, .next_nodes = { - [MFIB_FORWARD_RPF_NEXT_DROP] = "error-drop", + [MFIB_FORWARD_RPF_NEXT_DROP] = "ip6-drop", }, }; diff --git a/src/vnet/mfib/mfib_table.c b/src/vnet/mfib/mfib_table.c index 3b4bd985..7ffe8941 100644 --- a/src/vnet/mfib/mfib_table.c +++ b/src/vnet/mfib/mfib_table.c @@ -165,6 +165,7 @@ fib_node_index_t mfib_table_entry_update (u32 fib_index, const mfib_prefix_t *prefix, mfib_source_t source, + fib_rpf_id_t rpf_id, mfib_entry_flags_t entry_flags) { fib_node_index_t mfib_entry_index; @@ -181,7 +182,8 @@ mfib_table_entry_update (u32 fib_index, * update to a non-existing entry with non-zero flags */ mfib_entry_index = mfib_entry_create(fib_index, source, - prefix, entry_flags); + prefix, rpf_id, + entry_flags); mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index); } @@ -198,6 +200,7 @@ mfib_table_entry_update (u32 fib_index, if (mfib_entry_update(mfib_entry_index, source, entry_flags, + rpf_id, INDEX_INVALID)) { /* @@ -230,6 +233,7 @@ mfib_table_entry_path_update (u32 fib_index, mfib_entry_index = mfib_entry_create(fib_index, source, prefix, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_NONE); mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index); @@ -304,6 +308,7 @@ mfib_table_entry_special_add (u32 fib_index, mfib_entry_index = mfib_entry_create(fib_index, source, prefix, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_NONE); mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index); @@ -311,6 +316,7 @@ mfib_table_entry_special_add (u32 fib_index, mfib_entry_update(mfib_entry_index, source, (MFIB_ENTRY_FLAG_EXCLUSIVE | entry_flags), + MFIB_RPF_ID_NONE, rep_dpo); return (mfib_entry_index); diff --git a/src/vnet/mfib/mfib_table.h b/src/vnet/mfib/mfib_table.h index 95239f7c..83aa04ef 100644 --- a/src/vnet/mfib/mfib_table.h +++ b/src/vnet/mfib/mfib_table.h @@ -122,6 +122,7 @@ extern fib_node_index_t mfib_table_lookup_exact_match(u32 fib_index, extern fib_node_index_t mfib_table_entry_update(u32 fib_index, const mfib_prefix_t *prefix, mfib_source_t source, + fib_rpf_id_t rpf_id, mfib_entry_flags_t flags); /** diff --git a/src/vnet/mfib/mfib_test.c b/src/vnet/mfib/mfib_test.c index 36a303e8..7c92ae99 100644 --- a/src/vnet/mfib/mfib_test.c +++ b/src/vnet/mfib/mfib_test.c @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include #include @@ -201,8 +203,8 @@ mfib_test_validate_rep_v (const replicate_t *rep, if (DPO_RECEIVE != dt) { MFIB_TEST_REP((ai == dpo->dpoi_index), - "bucket %d stacks on %U", - bucket, + "bucket %d [exp:%d] stacks on %U", + bucket, ai, format_dpo_id, dpo, 0); } } @@ -734,6 +736,7 @@ mfib_test_i (fib_protocol_t PROTO, mfib_table_entry_update(fib_index, pfx_s_g, MFIB_SOURCE_API, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_SIGNAL); MFIB_TEST(mfib_test_entry(mfei, MFIB_ENTRY_FLAG_SIGNAL, @@ -824,6 +827,7 @@ mfib_test_i (fib_protocol_t PROTO, mfib_table_entry_update(fib_index, pfx_s_g, MFIB_SOURCE_API, + MFIB_RPF_ID_NONE, (MFIB_ENTRY_FLAG_SIGNAL | MFIB_ENTRY_FLAG_CONNECTED)); MFIB_TEST(mfib_test_entry(mfei, @@ -965,6 +969,7 @@ mfib_test_i (fib_protocol_t PROTO, mfib_table_entry_update(fib_index, pfx_s_g, MFIB_SOURCE_API, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_NONE); mfei = mfib_table_lookup_exact_match(fib_index, pfx_s_g); @@ -1073,6 +1078,117 @@ mfib_test_i (fib_protocol_t PROTO, MFIB_SOURCE_SRv6); dpo_reset(&td); + /* + * A Multicast LSP. This a mLDP head-end + */ + fib_node_index_t ai_mpls_10_10_10_1, lfei; + ip46_address_t nh_10_10_10_1 = { + .ip4 = { + .as_u32 = clib_host_to_net_u32(0x0a0a0a01), + }, + }; + ai_mpls_10_10_10_1 = adj_nbr_add_or_lock(FIB_PROTOCOL_IP4, + VNET_LINK_MPLS, + &nh_10_10_10_1, + tm->hw[0]->sw_if_index); + + fib_prefix_t pfx_3500 = { + .fp_len = 21, + .fp_proto = FIB_PROTOCOL_MPLS, + .fp_label = 3500, + .fp_eos = MPLS_EOS, + .fp_payload_proto = DPO_PROTO_IP4, + }; + fib_test_rep_bucket_t mc_0 = { + .type = FT_REP_LABEL_O_ADJ, + .label_o_adj = { + .adj = ai_mpls_10_10_10_1, + .label = 3300, + .eos = MPLS_EOS, + }, + }; + mpls_label_t *l3300 = NULL; + vec_add1(l3300, 3300); + + /* + * MPLS enable an interface so we get the MPLS table created + */ + mpls_sw_interface_enable_disable(&mpls_main, + tm->hw[0]->sw_if_index, + 1); + + lfei = fib_table_entry_update_one_path(0, // default MPLS Table + &pfx_3500, + FIB_SOURCE_API, + FIB_ENTRY_FLAG_MULTICAST, + FIB_PROTOCOL_IP4, + &nh_10_10_10_1, + tm->hw[0]->sw_if_index, + ~0, // invalid fib index + 1, + l3300, + FIB_ROUTE_PATH_FLAG_NONE); + MFIB_TEST(fib_test_validate_entry(lfei, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + 1, + &mc_0), + "3500 via replicate over 10.10.10.1"); + + /* + * An (S,G) that resolves via the mLDP head-end + */ + fib_route_path_t path_via_mldp = { + .frp_proto = FIB_PROTOCOL_MPLS, + .frp_local_label = pfx_3500.fp_label, + .frp_eos = MPLS_EOS, + .frp_sw_if_index = 0xffffffff, + .frp_fib_index = 0, + .frp_weight = 1, + .frp_flags = FIB_ROUTE_PATH_FLAG_NONE, + }; + dpo_id_t mldp_dpo = DPO_INVALID; + + fib_entry_contribute_forwarding(lfei, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + &mldp_dpo); + + mfei = mfib_table_entry_path_update(fib_index, + pfx_s_g, + MFIB_SOURCE_API, + &path_via_mldp, + MFIB_ITF_FLAG_FORWARD); + + MFIB_TEST(mfib_test_entry(mfei, + MFIB_ENTRY_FLAG_NONE, + 1, + DPO_REPLICATE, mldp_dpo.dpoi_index), + "%U over-mLDP replicate OK", + format_mfib_prefix, pfx_s_g); + + /* + * add a for-us path. this tests two types of non-attached paths on one entry + */ + mfei = mfib_table_entry_path_update(fib_index, + pfx_s_g, + MFIB_SOURCE_API, + &path_for_us, + MFIB_ITF_FLAG_FORWARD); + MFIB_TEST(mfib_test_entry(mfei, + MFIB_ENTRY_FLAG_NONE, + 2, + DPO_REPLICATE, mldp_dpo.dpoi_index, + DPO_RECEIVE, 0), + "%U mLDP+for-us replicate OK", + format_mfib_prefix, pfx_s_g); + + mfib_table_entry_delete(fib_index, + pfx_s_g, + MFIB_SOURCE_API); + fib_table_entry_delete(0, + &pfx_3500, + FIB_SOURCE_API); + dpo_reset(&mldp_dpo); + /* * Unlock the table - it's the last lock so should be gone thereafter */ @@ -1086,6 +1202,13 @@ mfib_test_i (fib_protocol_t PROTO, adj_unlock(ai_2); adj_unlock(ai_3); + /* + * MPLS disable the interface + */ + mpls_sw_interface_enable_disable(&mpls_main, + tm->hw[0]->sw_if_index, + 0); + /* * test we've leaked no resources */ diff --git a/src/vnet/mpls/mpls.api b/src/vnet/mpls/mpls.api index 2e3bfaf5..a1e1270a 100644 --- a/src/vnet/mpls/mpls.api +++ b/src/vnet/mpls/mpls.api @@ -55,6 +55,7 @@ define mpls_ip_bind_unbind_reply @param context - sender context, to match reply w/ request @param mt_is_add - Is this a route add or delete @param mt_sw_if_index - The SW interface index of the tunnel to delete + @param mt_is_multicast - Is the tunnel's underlying LSP multicast @param mt_next_hop_proto_is_ip4 - The next-hop is IPV4 @param mt_next_hop_weight - The weight, for UCMP @param mt_next_hop[16] - the nextop address @@ -70,6 +71,7 @@ define mpls_tunnel_add_del u32 mt_sw_if_index; u8 mt_is_add; u8 mt_l2_only; + u8 mt_is_multicast; u8 mt_next_hop_proto_is_ip4; u8 mt_next_hop_weight; u8 mt_next_hop[16]; @@ -102,30 +104,43 @@ define mpls_tunnel_dump i32 tunnel_index; }; -/** \brief mpls eth tunnel operational state response - @param tunnel_index - eth tunnel identifier - @param intfc_address - interface ipv4 addr - @param mask_width - interface ipv4 addr mask - @param hw_if_index - interface id - @param l2_only - - @param tunnel_dst_mac - - @param tx_sw_if_index - - @param encap_index - reference to mpls label table - @param nlabels - number of resolved labels - @param labels - resolved labels +/** \brief FIB path + @param sw_if_index - index of the interface + @param weight - The weight, for UCMP + @param is_local - local if non-zero, else remote + @param is_drop - Drop the packet + @param is_unreach - Drop the packet and rate limit send ICMP unreachable + @param is_prohibit - Drop the packet and rate limit send ICMP prohibited + @param afi - the afi of the next hop, IP46_TYPE_IP4=1, IP46_TYPE_IP6=2 + @param next_hop[16] - the next hop address + + WARNING: this type is replicated, pending cleanup completion + +*/ +typeonly manual_print manual_endian define fib_path2 +{ + u32 sw_if_index; + u32 weight; + u8 is_local; + u8 is_drop; + u8 is_unreach; + u8 is_prohibit; + u8 afi; + u8 next_hop[16]; + u32 labels[16]; +}; + +/** \brief mpls tunnel details */ -define mpls_tunnel_details +manual_endian manual_print define mpls_tunnel_details { u32 context; - u32 tunnel_index; - u8 mt_l2_only; u8 mt_sw_if_index; - u8 mt_next_hop_proto_is_ip4; - u8 mt_next_hop[16]; - u32 mt_next_hop_sw_if_index; - u32 mt_next_hop_table_id; - u32 mt_next_hop_n_labels; - u32 mt_next_hop_out_labels[mt_next_hop_n_labels]; + u8 mt_tunnel_index; + u8 mt_l2_only; + u8 mt_is_multicast; + u32 mt_count; + vl_api_fib_path2_t mt_paths[mt_count]; }; /** \brief MPLS Route Add / del route @@ -140,10 +155,14 @@ define mpls_tunnel_details create them @param mr_is_add - Is this a route add or delete @param mr_is_classify - Is this route result a classify + @param mr_is_multicast - Is this a multicast route @param mr_is_multipath - Is this route update a multipath - i.e. is this a path addition to an existing route @param mr_is_resolve_host - Recurse resolution constraint via a host prefix @param mr_is_resolve_attached - Recurse resolution constraint via attached prefix + @param mr_is_interface_rx - Interface Receive path + @param mr_is_interface_rx - RPF-ID Receive path. The next-hop interface + is used as the RPF-ID @param mr_next_hop_proto_is_ip4 - The next-hop is IPV4 @param mr_next_hop_weight - The weight, for UCMP @param mr_next_hop[16] - the nextop address @@ -164,9 +183,12 @@ define mpls_route_add_del u8 mr_create_table_if_needed; u8 mr_is_add; u8 mr_is_classify; + u8 mr_is_multicast; u8 mr_is_multipath; u8 mr_is_resolve_host; u8 mr_is_resolve_attached; + u8 mr_is_interface_rx; + u8 mr_is_rpf_id; u8 mr_next_hop_proto_is_ip4; u8 mr_next_hop_weight; u8 mr_next_hop[16]; @@ -187,31 +209,6 @@ define mpls_route_add_del_reply i32 retval; }; -/** \brief FIB path - @param sw_if_index - index of the interface - @param weight - The weight, for UCMP - @param is_local - local if non-zero, else remote - @param is_drop - Drop the packet - @param is_unreach - Drop the packet and rate limit send ICMP unreachable - @param is_prohibit - Drop the packet and rate limit send ICMP prohibited - @param afi - the afi of the next hop, IP46_TYPE_IP4=1, IP46_TYPE_IP6=2 - @param next_hop[16] - the next hop address - - WARNING: this type is replicated, pending cleanup completion - -*/ -typeonly manual_print manual_endian define fib_path2 -{ - u32 sw_if_index; - u32 weight; - u8 is_local; - u8 is_drop; - u8 is_unreach; - u8 is_prohibit; - u8 afi; - u8 next_hop[16]; -}; - /** \brief Dump MPLS fib table @param client_index - opaque cookie to identify the sender */ diff --git a/src/vnet/mpls/mpls.c b/src/vnet/mpls/mpls.c index 482577b1..451b15cf 100644 --- a/src/vnet/mpls/mpls.c +++ b/src/vnet/mpls/mpls.c @@ -286,7 +286,15 @@ vnet_mpls_local_label (vlib_main_t * vm, rpath.frp_proto = FIB_PROTOCOL_IP4; vec_add1(rpaths, rpath); } - + else if (unformat (line_input, "rx-ip4 %U", + unformat_vnet_sw_interface, vnm, + &rpath.frp_sw_if_index)) + { + rpath.frp_weight = 1; + rpath.frp_proto = FIB_PROTOCOL_IP4; + rpath.frp_flags = FIB_ROUTE_PATH_INTF_RX; + vec_add1(rpaths, rpath); + } else if (unformat (line_input, "via %U %U", unformat_ip6_address, &rpath.frp_addr.ip6, @@ -512,10 +520,3 @@ mpls_init (vlib_main_t * vm) } VLIB_INIT_FUNCTION (mpls_init); - -mpls_main_t * mpls_get_main (vlib_main_t * vm) -{ - vlib_call_init_function (vm, mpls_init); - return &mpls_main; -} - diff --git a/src/vnet/mpls/mpls_api.c b/src/vnet/mpls/mpls_api.c index f1aef6c9..6bfc491d 100644 --- a/src/vnet/mpls/mpls_api.c +++ b/src/vnet/mpls/mpls_api.c @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -163,6 +164,7 @@ mpls_route_add_del_t_handler (vnet_main_t * vnm, dpo_proto_to_fib (pfx.fp_payload_proto), mp->mr_next_hop_table_id, mp->mr_create_table_if_needed, + mp->mr_is_rpf_id, &fib_index, &next_hop_fib_index); if (0 != rv) @@ -192,10 +194,13 @@ mpls_route_add_del_t_handler (vnet_main_t * vnm, 0, // mp->is_unreach, 0, // mp->is_prohibit, 0, // mp->is_local, + mp->mr_is_multicast, mp->mr_is_classify, mp->mr_classify_table_index, mp->mr_is_resolve_host, mp->mr_is_resolve_attached, + mp->mr_is_interface_rx, + mp->mr_is_rpf_id, fib_index, &pfx, mp->mr_next_hop_proto_is_ip4, &nh, ntohl (mp->mr_next_hop_sw_if_index), @@ -229,46 +234,54 @@ vl_api_mpls_tunnel_add_del_t_handler (vl_api_mpls_tunnel_add_del_t * mp) int rv = 0; u32 tunnel_sw_if_index; int ii; + fib_route_path_t rpath, *rpaths = NULL; + + memset (&rpath, 0, sizeof (rpath)); stats_dslock_with_hint (1 /* release hint */ , 5 /* tag */ ); - if (mp->mt_is_add) + if (mp->mt_next_hop_proto_is_ip4) { - fib_route_path_t rpath, *rpaths = NULL; - mpls_label_t *label_stack = NULL; - - memset (&rpath, 0, sizeof (rpath)); - - if (mp->mt_next_hop_proto_is_ip4) - { - rpath.frp_proto = FIB_PROTOCOL_IP4; - clib_memcpy (&rpath.frp_addr.ip4, - mp->mt_next_hop, sizeof (rpath.frp_addr.ip4)); - } - else - { - rpath.frp_proto = FIB_PROTOCOL_IP6; - clib_memcpy (&rpath.frp_addr.ip6, - mp->mt_next_hop, sizeof (rpath.frp_addr.ip6)); - } - rpath.frp_sw_if_index = ntohl (mp->mt_next_hop_sw_if_index); + rpath.frp_proto = FIB_PROTOCOL_IP4; + clib_memcpy (&rpath.frp_addr.ip4, + mp->mt_next_hop, sizeof (rpath.frp_addr.ip4)); + } + else + { + rpath.frp_proto = FIB_PROTOCOL_IP6; + clib_memcpy (&rpath.frp_addr.ip6, + mp->mt_next_hop, sizeof (rpath.frp_addr.ip6)); + } + rpath.frp_sw_if_index = ntohl (mp->mt_next_hop_sw_if_index); + rpath.frp_weight = 1; + if (mp->mt_is_add) + { for (ii = 0; ii < mp->mt_next_hop_n_out_labels; ii++) - vec_add1 (label_stack, ntohl (mp->mt_next_hop_out_label_stack[ii])); + vec_add1 (rpath.frp_label_stack, + ntohl (mp->mt_next_hop_out_label_stack[ii])); + } - vec_add1 (rpaths, rpath); + vec_add1 (rpaths, rpath); - vnet_mpls_tunnel_add (rpaths, label_stack, - mp->mt_l2_only, &tunnel_sw_if_index); - vec_free (rpaths); - vec_free (label_stack); + tunnel_sw_if_index = ntohl (mp->mt_sw_if_index); + + if (mp->mt_is_add) + { + if (~0 == tunnel_sw_if_index) + tunnel_sw_if_index = vnet_mpls_tunnel_create (mp->mt_l2_only, + mp->mt_is_multicast); + vnet_mpls_tunnel_path_add (tunnel_sw_if_index, rpaths); } else { tunnel_sw_if_index = ntohl (mp->mt_sw_if_index); - vnet_mpls_tunnel_del (tunnel_sw_if_index); + if (!vnet_mpls_tunnel_path_remove (tunnel_sw_if_index, rpaths)) + vnet_mpls_tunnel_del (tunnel_sw_if_index); } + vec_free (rpaths); + stats_dsunlock (); /* *INDENT-OFF* */ @@ -289,10 +302,12 @@ typedef struct mpls_tunnel_send_walk_ctx_t_ static void send_mpls_tunnel_entry (u32 mti, void *arg) { + fib_route_path_encode_t *api_rpaths, *api_rpath; mpls_tunnel_send_walk_ctx_t *ctx; vl_api_mpls_tunnel_details_t *mp; const mpls_tunnel_t *mt; - u32 nlabels; + vl_api_fib_path2_t *fp; + u32 n; ctx = arg; @@ -300,18 +315,34 @@ send_mpls_tunnel_entry (u32 mti, void *arg) return; mt = mpls_tunnel_get (mti); - nlabels = vec_len (mt->mt_label_stack); + n = fib_path_list_get_n_paths (mt->mt_path_list); + + mp = vl_msg_api_alloc (sizeof (*mp) + n * sizeof (vl_api_fib_path2_t)); + memset (mp, 0, sizeof (*mp) + n * sizeof (vl_api_fib_path2_t)); - mp = vl_msg_api_alloc (sizeof (*mp) + nlabels * sizeof (u32)); - memset (mp, 0, sizeof (*mp)); mp->_vl_msg_id = ntohs (VL_API_MPLS_TUNNEL_DETAILS); mp->context = ctx->context; - mp->tunnel_index = ntohl (mti); - memcpy (mp->mt_next_hop_out_labels, - mt->mt_label_stack, nlabels * sizeof (u32)); + mp->mt_tunnel_index = ntohl (mti); + mp->mt_count = ntohl (n); + + fib_path_list_walk (mt->mt_path_list, fib_path_encode, &api_rpaths); + + fp = mp->mt_paths; + vec_foreach (api_rpath, api_rpaths) + { + memset (fp, 0, sizeof (*fp)); + + fp->weight = htonl (api_rpath->rpath.frp_weight); + fp->sw_if_index = htonl (api_rpath->rpath.frp_sw_if_index); + copy_fib_next_hop (api_rpath, fp); + fp++; + } // FIXME + // memcpy (mp->mt_next_hop_out_labels, + // mt->mt_label_stack, nlabels * sizeof (u32)); + vl_msg_api_send_shmem (ctx->q, (u8 *) & mp); } diff --git a/src/vnet/mpls/mpls_input.c b/src/vnet/mpls/mpls_input.c index 1b9bdd05..86ad8bba 100644 --- a/src/vnet/mpls/mpls_input.c +++ b/src/vnet/mpls/mpls_input.c @@ -291,7 +291,7 @@ mpls_setup_nodes (vlib_main_t * vm) rt->last_outer_fib_index = 0; rt->mpls_main = &mpls_main; - ethernet_register_input_type (vm, ETHERNET_TYPE_MPLS_UNICAST, + ethernet_register_input_type (vm, ETHERNET_TYPE_MPLS, mpls_input_node.index); } diff --git a/src/vnet/mpls/mpls_lookup.c b/src/vnet/mpls/mpls_lookup.c index ace6a70f..3c6be7e8 100644 --- a/src/vnet/mpls/mpls_lookup.c +++ b/src/vnet/mpls/mpls_lookup.c @@ -20,8 +20,17 @@ #include #include #include +#include -vlib_node_registration_t mpls_lookup_node; +/** + * Static MPLS VLIB forwarding node + */ +static vlib_node_registration_t mpls_lookup_node; + +/** + * The arc/edge from the MPLS lookup node to the MPLS replicate node + */ +static u32 mpls_lookup_to_replicate_edge; typedef struct { u32 next_index; @@ -156,81 +165,123 @@ mpls_lookup (vlib_main_t * vm, lbi2 = mpls_fib_table_forwarding_lookup (lfib_index2, h2); lbi3 = mpls_fib_table_forwarding_lookup (lfib_index3, h3); - lb0 = load_balance_get(lbi0); - lb1 = load_balance_get(lbi1); - lb2 = load_balance_get(lbi2); - lb3 = load_balance_get(lbi3); - hash_c0 = vnet_buffer(b0)->ip.flow_hash = 0; hash_c1 = vnet_buffer(b1)->ip.flow_hash = 0; hash_c2 = vnet_buffer(b2)->ip.flow_hash = 0; hash_c3 = vnet_buffer(b3)->ip.flow_hash = 0; - if (PREDICT_FALSE(lb0->lb_n_buckets > 1)) + if (MPLS_IS_REPLICATE & lbi0) { - hash_c0 = vnet_buffer (b0)->ip.flow_hash = - mpls_compute_flow_hash(h0, lb0->lb_hash_config); + next0 = mpls_lookup_to_replicate_edge; + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = + (lbi0 & ~MPLS_IS_REPLICATE); } - if (PREDICT_FALSE(lb1->lb_n_buckets > 1)) + else { - hash_c1 = vnet_buffer (b1)->ip.flow_hash = - mpls_compute_flow_hash(h1, lb1->lb_hash_config); + lb0 = load_balance_get(lbi0); + + if (PREDICT_FALSE(lb0->lb_n_buckets > 1)) + { + hash_c0 = vnet_buffer (b0)->ip.flow_hash = + mpls_compute_flow_hash(h0, lb0->lb_hash_config); + } + ASSERT (lb0->lb_n_buckets > 0); + ASSERT (is_pow2 (lb0->lb_n_buckets)); + dpo0 = load_balance_get_bucket_i(lb0, + (hash_c0 & + (lb0->lb_n_buckets_minus_1))); + next0 = dpo0->dpoi_next_node; + + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; + + vlib_increment_combined_counter + (cm, thread_index, lbi0, 1, + vlib_buffer_length_in_chain (vm, b0)); } - if (PREDICT_FALSE(lb2->lb_n_buckets > 1)) + if (MPLS_IS_REPLICATE & lbi1) { - hash_c2 = vnet_buffer (b2)->ip.flow_hash = - mpls_compute_flow_hash(h2, lb2->lb_hash_config); + next1 = mpls_lookup_to_replicate_edge; + vnet_buffer (b1)->ip.adj_index[VLIB_TX] = + (lbi1 & ~MPLS_IS_REPLICATE); } - if (PREDICT_FALSE(lb3->lb_n_buckets > 1)) + else { - hash_c3 = vnet_buffer (b3)->ip.flow_hash = - mpls_compute_flow_hash(h3, lb3->lb_hash_config); - } - - ASSERT (lb0->lb_n_buckets > 0); - ASSERT (is_pow2 (lb0->lb_n_buckets)); - ASSERT (lb1->lb_n_buckets > 0); - ASSERT (is_pow2 (lb1->lb_n_buckets)); - ASSERT (lb2->lb_n_buckets > 0); - ASSERT (is_pow2 (lb2->lb_n_buckets)); - ASSERT (lb3->lb_n_buckets > 0); - ASSERT (is_pow2 (lb3->lb_n_buckets)); - - dpo0 = load_balance_get_bucket_i(lb0, - (hash_c0 & - (lb0->lb_n_buckets_minus_1))); - dpo1 = load_balance_get_bucket_i(lb1, - (hash_c1 & - (lb1->lb_n_buckets_minus_1))); - dpo2 = load_balance_get_bucket_i(lb2, - (hash_c2 & - (lb2->lb_n_buckets_minus_1))); - dpo3 = load_balance_get_bucket_i(lb3, - (hash_c3 & - (lb3->lb_n_buckets_minus_1))); + lb1 = load_balance_get(lbi1); - next0 = dpo0->dpoi_next_node; - next1 = dpo1->dpoi_next_node; - next2 = dpo2->dpoi_next_node; - next3 = dpo3->dpoi_next_node; + if (PREDICT_FALSE(lb1->lb_n_buckets > 1)) + { + hash_c1 = vnet_buffer (b1)->ip.flow_hash = + mpls_compute_flow_hash(h1, lb1->lb_hash_config); + } + ASSERT (lb1->lb_n_buckets > 0); + ASSERT (is_pow2 (lb1->lb_n_buckets)); + dpo1 = load_balance_get_bucket_i(lb1, + (hash_c1 & + (lb1->lb_n_buckets_minus_1))); + next1 = dpo1->dpoi_next_node; + + vnet_buffer (b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; + + vlib_increment_combined_counter + (cm, thread_index, lbi1, 1, + vlib_buffer_length_in_chain (vm, b1)); + } + if (MPLS_IS_REPLICATE & lbi2) + { + next2 = mpls_lookup_to_replicate_edge; + vnet_buffer (b2)->ip.adj_index[VLIB_TX] = + (lbi2 & ~MPLS_IS_REPLICATE); + } + else + { + lb2 = load_balance_get(lbi2); - vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - vnet_buffer (b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; - vnet_buffer (b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index; - vnet_buffer (b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index; + if (PREDICT_FALSE(lb2->lb_n_buckets > 1)) + { + hash_c2 = vnet_buffer (b2)->ip.flow_hash = + mpls_compute_flow_hash(h2, lb2->lb_hash_config); + } + ASSERT (lb2->lb_n_buckets > 0); + ASSERT (is_pow2 (lb2->lb_n_buckets)); + dpo2 = load_balance_get_bucket_i(lb2, + (hash_c2 & + (lb2->lb_n_buckets_minus_1))); + next2 = dpo2->dpoi_next_node; + + vnet_buffer (b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index; + + vlib_increment_combined_counter + (cm, thread_index, lbi2, 1, + vlib_buffer_length_in_chain (vm, b2)); + } + if (MPLS_IS_REPLICATE & lbi3) + { + next3 = mpls_lookup_to_replicate_edge; + vnet_buffer (b3)->ip.adj_index[VLIB_TX] = + (lbi3 & ~MPLS_IS_REPLICATE); + } + else + { + lb3 = load_balance_get(lbi3); - vlib_increment_combined_counter - (cm, thread_index, lbi0, 1, - vlib_buffer_length_in_chain (vm, b0)); - vlib_increment_combined_counter - (cm, thread_index, lbi1, 1, - vlib_buffer_length_in_chain (vm, b1)); - vlib_increment_combined_counter - (cm, thread_index, lbi2, 1, - vlib_buffer_length_in_chain (vm, b2)); - vlib_increment_combined_counter - (cm, thread_index, lbi3, 1, - vlib_buffer_length_in_chain (vm, b3)); + if (PREDICT_FALSE(lb3->lb_n_buckets > 1)) + { + hash_c3 = vnet_buffer (b3)->ip.flow_hash = + mpls_compute_flow_hash(h3, lb3->lb_hash_config); + } + ASSERT (lb3->lb_n_buckets > 0); + ASSERT (is_pow2 (lb3->lb_n_buckets)); + dpo3 = load_balance_get_bucket_i(lb3, + (hash_c3 & + (lb3->lb_n_buckets_minus_1))); + next3 = dpo3->dpoi_next_node; + + vnet_buffer (b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index; + + vlib_increment_combined_counter + (cm, thread_index, lbi3, 1, + vlib_buffer_length_in_chain (vm, b3)); + } /* * before we pop the label copy th values we need to maintain. @@ -331,31 +382,41 @@ mpls_lookup (vlib_main_t * vm, vnet_buffer(b0)->sw_if_index[VLIB_RX]); lbi0 = mpls_fib_table_forwarding_lookup(lfib_index0, h0); - lb0 = load_balance_get(lbi0); - hash_c0 = vnet_buffer(b0)->ip.flow_hash = 0; - if (PREDICT_FALSE(lb0->lb_n_buckets > 1)) + + if (MPLS_IS_REPLICATE & lbi0) { - hash_c0 = vnet_buffer (b0)->ip.flow_hash = - mpls_compute_flow_hash(h0, lb0->lb_hash_config); + next0 = mpls_lookup_to_replicate_edge; + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = + (lbi0 & ~MPLS_IS_REPLICATE); } + else + { + lb0 = load_balance_get(lbi0); - ASSERT (lb0->lb_n_buckets > 0); - ASSERT (is_pow2 (lb0->lb_n_buckets)); + if (PREDICT_FALSE(lb0->lb_n_buckets > 1)) + { + hash_c0 = vnet_buffer (b0)->ip.flow_hash = + mpls_compute_flow_hash(h0, lb0->lb_hash_config); + } - dpo0 = load_balance_get_bucket_i(lb0, - (hash_c0 & - (lb0->lb_n_buckets_minus_1))); + ASSERT (lb0->lb_n_buckets > 0); + ASSERT (is_pow2 (lb0->lb_n_buckets)); - next0 = dpo0->dpoi_next_node; - vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; + dpo0 = load_balance_get_bucket_i(lb0, + (hash_c0 & + (lb0->lb_n_buckets_minus_1))); - vlib_increment_combined_counter - (cm, thread_index, lbi0, 1, - vlib_buffer_length_in_chain (vm, b0)); + next0 = dpo0->dpoi_next_node; + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; + + vlib_increment_combined_counter + (cm, thread_index, lbi0, 1, + vlib_buffer_length_in_chain (vm, b0)); + } /* - * before we pop the label copy th values we need to maintain. + * before we pop the label copy, values we need to maintain. * The label header is in network byte order. * last byte is the TTL. * bits 2 to 4 inclusive are the EXP bits @@ -398,7 +459,7 @@ static char * mpls_error_strings[] = { #undef mpls_error }; -VLIB_REGISTER_NODE (mpls_lookup_node) = { +VLIB_REGISTER_NODE (mpls_lookup_node, static) = { .function = mpls_lookup, .name = "mpls-lookup", /* Takes a vector of packets. */ @@ -621,3 +682,22 @@ VLIB_REGISTER_NODE (mpls_load_balance_node) = { }; VLIB_NODE_FUNCTION_MULTIARCH (mpls_load_balance_node, mpls_load_balance) + + +static clib_error_t * +mpls_lookup_init (vlib_main_t * vm) +{ + clib_error_t * error; + + if ((error = vlib_call_init_function (vm, mpls_init))) + return error; + + mpls_lookup_to_replicate_edge = + vlib_node_add_named_next(vm, + mpls_lookup_node.index, + "mpls-replicate"); + + return (NULL); +} + +VLIB_INIT_FUNCTION (mpls_lookup_init); diff --git a/src/vnet/mpls/mpls_tunnel.c b/src/vnet/mpls/mpls_tunnel.c index ac6fdcdf..1254dd9d 100644 --- a/src/vnet/mpls/mpls_tunnel.c +++ b/src/vnet/mpls/mpls_tunnel.c @@ -18,9 +18,12 @@ #include #include #include +#include #include #include #include +#include +#include /** * @brief pool of tunnel instances @@ -37,6 +40,11 @@ static u32 * mpls_tunnel_free_hw_if_indices; */ static u32 *mpls_tunnel_db; +/** + * @brief MPLS tunnel flags strings + */ +static const char *mpls_tunnel_attribute_names[] = MPLS_TUNNEL_ATTRIBUTES; + /** * @brief Get a tunnel object from a SW interface index */ @@ -44,103 +52,178 @@ static mpls_tunnel_t* mpls_tunnel_get_from_sw_if_index (u32 sw_if_index) { if ((vec_len(mpls_tunnel_db) < sw_if_index) || - (~0 == mpls_tunnel_db[sw_if_index])) - return (NULL); + (~0 == mpls_tunnel_db[sw_if_index])) + return (NULL); return (pool_elt_at_index(mpls_tunnel_pool, - mpls_tunnel_db[sw_if_index])); + mpls_tunnel_db[sw_if_index])); } /** - * @brief Return true if the label stack is imp-null only + * @brief Build a rewrite string for the MPLS tunnel. */ -static fib_forward_chain_type_t -mpls_tunnel_get_fwd_chain_type (const mpls_tunnel_t *mt) +static u8* +mpls_tunnel_build_rewrite_i (void) { - if ((1 == vec_len(mt->mt_label_stack)) && - (mt->mt_label_stack[0] == MPLS_IETF_IMPLICIT_NULL_LABEL)) - { - /* - * the only label in the label stack is implicit null - * we need to build an IP chain. - */ - if (FIB_PROTOCOL_IP4 == fib_path_list_get_proto(mt->mt_path_list)) - { - return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4); - } - else - { - return (FIB_FORW_CHAIN_TYPE_UNICAST_IP6); - } - } - else - { - return (FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS); - } + /* + * passing the adj code a NULL rewirte means 'i don't have one cos + * t'other end is unresolved'. That's not the case here. For the mpls + * tunnel there are just no bytes of encap to apply in the adj. We'll impose + * the label stack once we choose a path. So return a zero length rewrite. + */ + u8 *rewrite = NULL; + + vec_validate(rewrite, 0); + vec_reset_length(rewrite); + + return (rewrite); } /** * @brief Build a rewrite string for the MPLS tunnel. - * - * We have choices here; - * 1 - have an Adjacency with a zero length string and stack it on - * MPLS label objects - * 2 - put the label header rewrites in the adjacency string. - * - * We choose 2 since it results in fewer graph nodes in the egress path */ static u8* mpls_tunnel_build_rewrite (vnet_main_t * vnm, - u32 sw_if_index, - vnet_link_t link_type, - const void *dst_address) + u32 sw_if_index, + vnet_link_t link_type, + const void *dst_address) { - mpls_unicast_header_t *muh; - mpls_tunnel_t *mt; - u8 *rewrite; - u32 mti, ii; + return (mpls_tunnel_build_rewrite_i()); +} - rewrite = NULL; - mti = mpls_tunnel_db[sw_if_index]; - mt = pool_elt_at_index(mpls_tunnel_pool, mti); +typedef struct mpls_tunnel_collect_forwarding_ctx_t_ +{ + load_balance_path_t * next_hops; + const mpls_tunnel_t *mt; + fib_forward_chain_type_t fct; +} mpls_tunnel_collect_forwarding_ctx_t; + +static int +mpls_tunnel_collect_forwarding (fib_node_index_t pl_index, + fib_node_index_t path_index, + void *arg) +{ + mpls_tunnel_collect_forwarding_ctx_t *ctx; + fib_path_ext_t *path_ext; + int have_path_ext; + + ctx = arg; /* - * The vector must be allocated as u8 so the length is correct + * if the path is not resolved, don't include it. */ - ASSERT(0 < vec_len(mt->mt_label_stack)); - vec_validate(rewrite, (sizeof(*muh) * vec_len(mt->mt_label_stack)) - 1); - ASSERT(rewrite); - muh = (mpls_unicast_header_t *)rewrite; + if (!fib_path_is_resolved(path_index)) + { + return (!0); + } /* - * The last (inner most) label in the stack may be EOS, all the rest Non-EOS + * get the matching path-extension for the path being visited. */ - for (ii = 0; ii < vec_len(mt->mt_label_stack)-1; ii++) + have_path_ext = 0; + vec_foreach(path_ext, ctx->mt->mt_path_exts) { - vnet_mpls_uc_set_label(&muh[ii].label_exp_s_ttl, mt->mt_label_stack[ii]); - vnet_mpls_uc_set_ttl(&muh[ii].label_exp_s_ttl, 255); - vnet_mpls_uc_set_exp(&muh[ii].label_exp_s_ttl, 0); - vnet_mpls_uc_set_s(&muh[ii].label_exp_s_ttl, MPLS_NON_EOS); - muh[ii].label_exp_s_ttl = clib_host_to_net_u32(muh[ii].label_exp_s_ttl); + if (path_ext->fpe_path_index == path_index) + { + have_path_ext = 1; + break; + } } - vnet_mpls_uc_set_label(&muh[ii].label_exp_s_ttl, mt->mt_label_stack[ii]); - vnet_mpls_uc_set_ttl(&muh[ii].label_exp_s_ttl, 255); - vnet_mpls_uc_set_exp(&muh[ii].label_exp_s_ttl, 0); - - if ((VNET_LINK_MPLS == link_type) && - (mt->mt_label_stack[ii] != MPLS_IETF_IMPLICIT_NULL_LABEL)) + if (have_path_ext) { - vnet_mpls_uc_set_s(&muh[ii].label_exp_s_ttl, MPLS_NON_EOS); + /* + * found a matching extension. stack it to obtain the forwarding + * info for this path. + */ + ctx->next_hops = fib_path_ext_stack(path_ext, + ctx->fct, + ctx->fct, + ctx->next_hops); } else + ASSERT(0); + /* + * else + * There should be a path-extenios associated with each path + */ + + return (!0); +} + +static void +mpls_tunnel_mk_lb (mpls_tunnel_t *mt, + vnet_link_t linkt, + fib_forward_chain_type_t fct, + dpo_id_t *dpo_lb) +{ + dpo_proto_t lb_proto; + + /* + * If the entry has path extensions then we construct a load-balance + * by stacking the extensions on the forwarding chains of the paths. + * Otherwise we use the load-balance of the path-list + */ + mpls_tunnel_collect_forwarding_ctx_t ctx = { + .mt = mt, + .next_hops = NULL, + .fct = fct, + }; + + /* + * As an optimisation we allocate the vector of next-hops to be sized + * equal to the maximum nuber of paths we will need, which is also the + * most likely number we will need, since in most cases the paths are 'up'. + */ + vec_validate(ctx.next_hops, fib_path_list_get_n_paths(mt->mt_path_list)); + vec_reset_length(ctx.next_hops); + + lb_proto = vnet_link_to_dpo_proto(linkt); + + fib_path_list_walk(mt->mt_path_list, + mpls_tunnel_collect_forwarding, + &ctx); + + if (!dpo_id_is_valid(dpo_lb)) { - vnet_mpls_uc_set_s(&muh[ii].label_exp_s_ttl, MPLS_EOS); + /* + * first time create + */ + if (mt->mt_flags & MPLS_TUNNEL_FLAG_MCAST) + { + dpo_set(dpo_lb, + DPO_REPLICATE, + lb_proto, + replicate_create(0, lb_proto)); + } + else + { + flow_hash_config_t fhc; + + fhc = 0; // FIXME + /* fhc = fib_table_get_flow_hash_config(fib_entry->fe_fib_index, */ + /* dpo_proto_to_fib(lb_proto)); */ + dpo_set(dpo_lb, + DPO_LOAD_BALANCE, + lb_proto, + load_balance_create(0, lb_proto, fhc)); + } } - muh[ii].label_exp_s_ttl = clib_host_to_net_u32(muh[ii].label_exp_s_ttl); - - return (rewrite); + if (mt->mt_flags & MPLS_TUNNEL_FLAG_MCAST) + { + /* + * MPLS multicast + */ + replicate_multipath_update(dpo_lb, ctx.next_hops); + } + else + { + load_balance_multipath_update(dpo_lb, + ctx.next_hops, + LOAD_BALANCE_FLAG_NONE); + vec_free(ctx.next_hops); + } } /** @@ -161,45 +244,47 @@ mpls_tunnel_stack (adj_index_t ai) mt = mpls_tunnel_get_from_sw_if_index(sw_if_index); if (NULL == mt) - return; + return; /* - * find the adjacency that is contributed by the FIB path-list - * that this tunnel resovles via, and use it as the next adj - * in the midchain + * while we're stacking the adj, remove the tunnel from the child list + * of the path list. this breaks a circular dependency of walk updates + * where the create of adjacencies in the children can lead to walks + * that get back here. */ - if (vnet_hw_interface_get_flags(vnet_get_main(), - mt->mt_hw_if_index) & - VNET_HW_INTERFACE_FLAG_LINK_UP) - { - dpo_id_t dpo = DPO_INVALID; + fib_path_list_lock(mt->mt_path_list); - fib_path_list_contribute_forwarding(mt->mt_path_list, - mpls_tunnel_get_fwd_chain_type(mt), - &dpo); - - if (DPO_LOAD_BALANCE == dpo.dpoi_type) - { - /* - * we don't support multiple paths, so no need to load-balance. - * pull the first and only choice and stack directly on that. - */ - load_balance_t *lb; - - lb = load_balance_get (dpo.dpoi_index); + fib_path_list_child_remove(mt->mt_path_list, + mt->mt_sibling_index); - ASSERT(1 == lb->lb_n_buckets); + /* + * Construct the DPO (load-balance or replicate) that we can stack + * the tunnel's midchain on + */ + if (vnet_hw_interface_get_flags(vnet_get_main(), + mt->mt_hw_if_index) & + VNET_HW_INTERFACE_FLAG_LINK_UP) + { + dpo_id_t dpo = DPO_INVALID; - dpo_copy(&dpo, load_balance_get_bucket_i (lb, 0)); - } + mpls_tunnel_mk_lb(mt, + adj->ia_link, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + &dpo); - adj_nbr_midchain_stack(ai, &dpo); - dpo_reset(&dpo); + adj_nbr_midchain_stack(ai, &dpo); + dpo_reset(&dpo); } else { - adj_nbr_midchain_unstack(ai); + adj_nbr_midchain_unstack(ai); } + + mt->mt_sibling_index = fib_path_list_child_add(mt->mt_path_list, + FIB_NODE_TYPE_MPLS_TUNNEL, + mt - mpls_tunnel_pool); + + fib_path_list_lock(mt->mt_path_list); } /** @@ -207,7 +292,7 @@ mpls_tunnel_stack (adj_index_t ai) */ static adj_walk_rc_t mpls_adj_walk_cb (adj_index_t ai, - void *ctx) + void *ctx) { mpls_tunnel_stack(ai); @@ -224,17 +309,17 @@ mpls_tunnel_restack (mpls_tunnel_t *mt) */ FOR_EACH_FIB_PROTOCOL(proto) { - adj_nbr_walk(mt->mt_sw_if_index, - proto, - mpls_adj_walk_cb, - NULL); + adj_nbr_walk(mt->mt_sw_if_index, + proto, + mpls_adj_walk_cb, + NULL); } } static clib_error_t * mpls_tunnel_admin_up_down (vnet_main_t * vnm, - u32 hw_if_index, - u32 flags) + u32 hw_if_index, + u32 flags) { vnet_hw_interface_t * hi; mpls_tunnel_t *mt; @@ -244,13 +329,13 @@ mpls_tunnel_admin_up_down (vnet_main_t * vnm, mt = mpls_tunnel_get_from_sw_if_index(hi->sw_if_index); if (NULL == mt) - return (NULL); + return (NULL); if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) - vnet_hw_interface_set_flags (vnm, hw_if_index, - VNET_HW_INTERFACE_FLAG_LINK_UP); + vnet_hw_interface_set_flags (vnm, hw_if_index, + VNET_HW_INTERFACE_FLAG_LINK_UP); else - vnet_hw_interface_set_flags (vnm, hw_if_index, 0 /* down */); + vnet_hw_interface_set_flags (vnm, hw_if_index, 0 /* down */); mpls_tunnel_restack(mt); @@ -263,22 +348,58 @@ mpls_tunnel_admin_up_down (vnet_main_t * vnm, */ static void mpls_tunnel_fixup (vlib_main_t *vm, - ip_adjacency_t *adj, - vlib_buffer_t *b0) + ip_adjacency_t *adj, + vlib_buffer_t *b0) { + /* + * A no-op w.r.t. the header. but reset the 'have we pushed any + * MPLS labels onto the packet' flag. That way when we enter the + * tunnel we'll get a TTL set to 255 + */ + vnet_buffer(b0)->mpls.first = 0; } static void mpls_tunnel_update_adj (vnet_main_t * vnm, - u32 sw_if_index, - adj_index_t ai) + u32 sw_if_index, + adj_index_t ai) { - adj_nbr_midchain_update_rewrite( - ai, mpls_tunnel_fixup, - ADJ_FLAG_NONE, - mpls_tunnel_build_rewrite(vnm, sw_if_index, - adj_get_link_type(ai), - NULL)); + ip_adjacency_t *adj; + + ASSERT(ADJ_INDEX_INVALID != ai); + + adj = adj_get(ai); + + switch (adj->lookup_next_index) + { + case IP_LOOKUP_NEXT_ARP: + case IP_LOOKUP_NEXT_GLEAN: + adj_nbr_midchain_update_rewrite(ai, mpls_tunnel_fixup, + ADJ_FLAG_NONE, + mpls_tunnel_build_rewrite_i()); + break; + case IP_LOOKUP_NEXT_MCAST: + /* + * Construct a partial rewrite from the known ethernet mcast dest MAC + * There's no MAC fixup, so the last 2 parameters are 0 + */ + adj_mcast_midchain_update_rewrite(ai, mpls_tunnel_fixup, + ADJ_FLAG_NONE, + mpls_tunnel_build_rewrite_i(), + 0, 0); + break; + + case IP_LOOKUP_NEXT_DROP: + case IP_LOOKUP_NEXT_PUNT: + case IP_LOOKUP_NEXT_LOCAL: + case IP_LOOKUP_NEXT_REWRITE: + case IP_LOOKUP_NEXT_MIDCHAIN: + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: + case IP_LOOKUP_NEXT_ICMP_ERROR: + case IP_LOOKUP_N_NEXT: + ASSERT (0); + break; + } mpls_tunnel_stack(ai); } @@ -312,7 +433,7 @@ typedef struct mpls_tunnel_trace_t_ static u8 * format_mpls_tunnel_tx_trace (u8 * s, - va_list * args) + va_list * args) { CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); @@ -327,8 +448,8 @@ format_mpls_tunnel_tx_trace (u8 * s, */ static uword mpls_tunnel_tx (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) + vlib_node_runtime_t * node, + vlib_frame_t * frame) { u32 next_index; u32 * from, * to_next, n_left_from, n_left_to_next; @@ -355,32 +476,32 @@ mpls_tunnel_tx (vlib_main_t * vm, * FIXME DUAL LOOP */ while (n_left_from > 0 && n_left_to_next > 0) - { - vlib_buffer_t * b0; - u32 bi0; + { + vlib_buffer_t * b0; + u32 bi0; - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; - b0 = vlib_get_buffer(vm, bi0); + b0 = vlib_get_buffer(vm, bi0); - vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mt->mt_l2_adj; + vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mt->mt_l2_adj; - if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) - { - mpls_tunnel_trace_t *tr = vlib_add_trace (vm, node, - b0, sizeof (*tr)); - tr->tunnel_id = rd->dev_instance; - } + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_tunnel_trace_t *tr = vlib_add_trace (vm, node, + b0, sizeof (*tr)); + tr->tunnel_id = rd->dev_instance; + } - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, mt->mt_l2_tx_arc); - } + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, mt->mt_l2_tx_arc); + } vlib_put_next_frame (vm, node, next_index, n_left_to_next); } @@ -417,13 +538,13 @@ mpls_tunnel_get (u32 mti) */ void mpls_tunnel_walk (mpls_tunnel_walk_cb_t cb, - void *ctx) + void *ctx) { u32 mti; pool_foreach_index(mti, mpls_tunnel_pool, ({ - cb(mti, ctx); + cb(mti, ctx); })); } @@ -435,25 +556,22 @@ vnet_mpls_tunnel_del (u32 sw_if_index) mt = mpls_tunnel_get_from_sw_if_index(sw_if_index); if (NULL == mt) - return; - - fib_path_list_child_remove(mt->mt_path_list, - mt->mt_sibling_index); - if (ADJ_INDEX_INVALID != mt->mt_l2_adj) - adj_unlock(mt->mt_l2_adj); + return; - vec_free(mt->mt_label_stack); + if (FIB_NODE_INDEX_INVALID != mt->mt_path_list) + fib_path_list_child_remove(mt->mt_path_list, + mt->mt_sibling_index); + if (ADJ_INDEX_INVALID != mt->mt_l2_adj) + adj_unlock(mt->mt_l2_adj); vec_add1 (mpls_tunnel_free_hw_if_indices, mt->mt_hw_if_index); pool_put(mpls_tunnel_pool, mt); mpls_tunnel_db[sw_if_index] = ~0; } -void -vnet_mpls_tunnel_add (fib_route_path_t *rpaths, - mpls_label_t *label_stack, - u8 l2_only, - u32 *sw_if_index) +u32 +vnet_mpls_tunnel_create (u8 l2_only, + u8 is_multicast) { vnet_hw_interface_t * hi; mpls_tunnel_t *mt; @@ -466,28 +584,33 @@ vnet_mpls_tunnel_add (fib_route_path_t *rpaths, mti = mt - mpls_tunnel_pool; fib_node_init(&mt->mt_node, FIB_NODE_TYPE_MPLS_TUNNEL); mt->mt_l2_adj = ADJ_INDEX_INVALID; + mt->mt_path_list = FIB_NODE_INDEX_INVALID; + mt->mt_sibling_index = FIB_NODE_INDEX_INVALID; + + if (is_multicast) + mt->mt_flags |= MPLS_TUNNEL_FLAG_MCAST; /* * Create a new, or re=use and old, tunnel HW interface */ if (vec_len (mpls_tunnel_free_hw_if_indices) > 0) { - mt->mt_hw_if_index = - mpls_tunnel_free_hw_if_indices[vec_len(mpls_tunnel_free_hw_if_indices)-1]; - _vec_len (mpls_tunnel_free_hw_if_indices) -= 1; - hi = vnet_get_hw_interface (vnm, mt->mt_hw_if_index); - hi->hw_instance = mti; - hi->dev_instance = mti; + mt->mt_hw_if_index = + mpls_tunnel_free_hw_if_indices[vec_len(mpls_tunnel_free_hw_if_indices)-1]; + _vec_len (mpls_tunnel_free_hw_if_indices) -= 1; + hi = vnet_get_hw_interface (vnm, mt->mt_hw_if_index); + hi->hw_instance = mti; + hi->dev_instance = mti; } - else + else { - mt->mt_hw_if_index = vnet_register_interface( - vnm, - mpls_tunnel_class.index, - mti, - mpls_tunnel_hw_interface_class.index, - mti); - hi = vnet_get_hw_interface(vnm, mt->mt_hw_if_index); + mt->mt_hw_if_index = vnet_register_interface( + vnm, + mpls_tunnel_class.index, + mti, + mpls_tunnel_hw_interface_class.index, + mti); + hi = vnet_get_hw_interface(vnm, mt->mt_hw_if_index); } /* @@ -497,43 +620,218 @@ vnet_mpls_tunnel_add (fib_route_path_t *rpaths, vec_validate_init_empty(mpls_tunnel_db, mt->mt_sw_if_index, ~0); mpls_tunnel_db[mt->mt_sw_if_index] = mti; + if (l2_only) + { + mt->mt_l2_adj = + adj_nbr_add_or_lock(fib_path_list_get_proto(mt->mt_path_list), + VNET_LINK_ETHERNET, + &zero_addr, + mt->mt_sw_if_index); + + mt->mt_l2_tx_arc = vlib_node_add_named_next(vlib_get_main(), + hi->tx_node_index, + "adj-l2-midchain"); + } + + return (mt->mt_sw_if_index); +} + +/* + * mpls_tunnel_path_ext_add + * + * append a path extension to the entry's list + */ +static void +mpls_tunnel_path_ext_append (mpls_tunnel_t *mt, + const fib_route_path_t *rpath) +{ + if (NULL != rpath->frp_label_stack) + { + fib_path_ext_t *path_ext; + + vec_add2(mt->mt_path_exts, path_ext, 1); + + fib_path_ext_init(path_ext, mt->mt_path_list, rpath); + } +} + +/* + * mpls_tunnel_path_ext_insert + * + * insert, sorted, a path extension to the entry's list. + * It's not strictly necessary in sort the path extensions, since each + * extension has the path index to which it resolves. However, by being + * sorted the load-balance produced has a deterministic order, not an order + * based on the sequence of extension additions. this is a considerable benefit. + */ +static void +mpls_tunnel_path_ext_insert (mpls_tunnel_t *mt, + const fib_route_path_t *rpath) +{ + if (0 == vec_len(mt->mt_path_exts)) + return (mpls_tunnel_path_ext_append(mt, rpath)); + + if (NULL != rpath->frp_label_stack) + { + fib_path_ext_t path_ext; + int i = 0; + + fib_path_ext_init(&path_ext, mt->mt_path_list, rpath); + + while (i < vec_len(mt->mt_path_exts) && + (fib_path_ext_cmp(&mt->mt_path_exts[i], rpath) < 0)) + { + i++; + } + + vec_insert_elts(mt->mt_path_exts, &path_ext, 1, i); + } +} + +void +vnet_mpls_tunnel_path_add (u32 sw_if_index, + fib_route_path_t *rpaths) +{ + mpls_tunnel_t *mt; + u32 mti; + + mt = mpls_tunnel_get_from_sw_if_index(sw_if_index); + + if (NULL == mt) + return; + + mti = mt - mpls_tunnel_pool; + /* * construct a path-list from the path provided */ - mt->mt_path_list = fib_path_list_create(FIB_PATH_LIST_FLAG_SHARED, rpaths); - mt->mt_sibling_index = fib_path_list_child_add(mt->mt_path_list, - FIB_NODE_TYPE_MPLS_TUNNEL, - mti); + if (FIB_NODE_INDEX_INVALID == mt->mt_path_list) + { + mt->mt_path_list = fib_path_list_create(FIB_PATH_LIST_FLAG_SHARED, rpaths); + mt->mt_sibling_index = fib_path_list_child_add(mt->mt_path_list, + FIB_NODE_TYPE_MPLS_TUNNEL, + mti); + } + else + { + fib_node_index_t old_pl_index; + fib_path_ext_t *path_ext; + + old_pl_index = mt->mt_path_list; + + mt->mt_path_list = + fib_path_list_copy_and_path_add(old_pl_index, + FIB_PATH_LIST_FLAG_SHARED, + rpaths); + + fib_path_list_child_remove(old_pl_index, + mt->mt_sibling_index); + mt->mt_sibling_index = fib_path_list_child_add(mt->mt_path_list, + FIB_NODE_TYPE_MPLS_TUNNEL, + mti); + /* + * re-resolve all the path-extensions with the new path-list + */ + vec_foreach(path_ext, mt->mt_path_exts) + { + fib_path_ext_resolve(path_ext, mt->mt_path_list); + } + } + mpls_tunnel_path_ext_insert(mt, rpaths); + mpls_tunnel_restack(mt); +} + +int +vnet_mpls_tunnel_path_remove (u32 sw_if_index, + fib_route_path_t *rpaths) +{ + mpls_tunnel_t *mt; + u32 mti; - mt->mt_label_stack = vec_dup(label_stack); + mt = mpls_tunnel_get_from_sw_if_index(sw_if_index); - if (l2_only) + if (NULL == mt) + return (0); + + mti = mt - mpls_tunnel_pool; + + /* + * construct a path-list from the path provided + */ + if (FIB_NODE_INDEX_INVALID == mt->mt_path_list) { - mt->mt_l2_adj = - adj_nbr_add_or_lock(fib_path_list_get_proto(mt->mt_path_list), - VNET_LINK_ETHERNET, - &zero_addr, - mt->mt_sw_if_index); - - mt->mt_l2_tx_arc = vlib_node_add_named_next(vlib_get_main(), - hi->tx_node_index, - "adj-l2-midchain"); + /* can't remove a path if we have onoe */ + return (0); } - - *sw_if_index = mt->mt_sw_if_index; + else + { + fib_node_index_t old_pl_index; + fib_path_ext_t *path_ext; + + old_pl_index = mt->mt_path_list; + + mt->mt_path_list = + fib_path_list_copy_and_path_remove(old_pl_index, + FIB_PATH_LIST_FLAG_SHARED, + rpaths); + + fib_path_list_child_remove(old_pl_index, + mt->mt_sibling_index); + + if (FIB_NODE_INDEX_INVALID == mt->mt_path_list) + { + /* no paths left */ + return (0); + } + else + { + mt->mt_sibling_index = + fib_path_list_child_add(mt->mt_path_list, + FIB_NODE_TYPE_MPLS_TUNNEL, + mti); + } + /* + * find the matching path extension and remove it + */ + vec_foreach(path_ext, mt->mt_path_exts) + { + if (!fib_path_ext_cmp(path_ext, rpaths)) + { + /* + * delete the element moving the remaining elements down 1 position. + * this preserves the sorted order. + */ + vec_free(path_ext->fpe_label_stack); + vec_delete(mt->mt_path_exts, 1, + (path_ext - mt->mt_path_exts)); + break; + } + } + /* + * re-resolve all the path-extensions with the new path-list + */ + vec_foreach(path_ext, mt->mt_path_exts) + { + fib_path_ext_resolve(path_ext, mt->mt_path_list); + } + + mpls_tunnel_restack(mt); + } + + return (fib_path_list_get_n_paths(mt->mt_path_list)); } + static clib_error_t * vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) + unformat_input_t * input, + vlib_cli_command_t * cmd) { unformat_input_t _line_input, * line_input = &_line_input; vnet_main_t * vnm = vnet_get_main(); - u8 is_del = 0; - u8 l2_only = 0; + u8 is_del = 0, l2_only = 0, is_multicast =0; fib_route_path_t rpath, *rpaths = NULL; - mpls_label_t out_label = MPLS_LABEL_INVALID, *labels = NULL; + mpls_label_t out_label = MPLS_LABEL_INVALID; u32 sw_if_index; clib_error_t *error = NULL; @@ -541,87 +839,89 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm, /* Get a line of input. */ if (! unformat_user (input, unformat_line_input, line_input)) - return 0; + return 0; while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) { - if (unformat (line_input, "del %U", - unformat_vnet_sw_interface, vnm, - &sw_if_index)) - is_del = 1; - else if (unformat (line_input, "add")) - is_del = 0; - else if (unformat (line_input, "out-label %U", - unformat_mpls_unicast_label, &out_label)) - { - vec_add1(labels, out_label); - } - else if (unformat (line_input, "via %U %U", - unformat_ip4_address, - &rpath.frp_addr.ip4, - unformat_vnet_sw_interface, vnm, - &rpath.frp_sw_if_index)) - { - rpath.frp_weight = 1; - rpath.frp_proto = FIB_PROTOCOL_IP4; - } - - else if (unformat (line_input, "via %U %U", - unformat_ip6_address, - &rpath.frp_addr.ip6, - unformat_vnet_sw_interface, vnm, - &rpath.frp_sw_if_index)) - { - rpath.frp_weight = 1; - rpath.frp_proto = FIB_PROTOCOL_IP6; - } - else if (unformat (line_input, "via %U", - unformat_ip6_address, - &rpath.frp_addr.ip6)) - { - rpath.frp_fib_index = 0; - rpath.frp_weight = 1; - rpath.frp_sw_if_index = ~0; - rpath.frp_proto = FIB_PROTOCOL_IP6; - } - else if (unformat (line_input, "via %U", - unformat_ip4_address, - &rpath.frp_addr.ip4)) - { - rpath.frp_fib_index = 0; - rpath.frp_weight = 1; - rpath.frp_sw_if_index = ~0; - rpath.frp_proto = FIB_PROTOCOL_IP4; - } - else if (unformat (line_input, "l2-only")) - l2_only = 1; - else - { - error = clib_error_return (0, "unknown input '%U'", - format_unformat_error, line_input); - goto done; - } + if (unformat (line_input, "del %U", + unformat_vnet_sw_interface, vnm, + &sw_if_index)) + is_del = 1; + else if (unformat (line_input, "add")) + is_del = 0; + else if (unformat (line_input, "out-label %U", + unformat_mpls_unicast_label, &out_label)) + { + vec_add1(rpath.frp_label_stack, out_label); + } + else if (unformat (line_input, "via %U %U", + unformat_ip4_address, + &rpath.frp_addr.ip4, + unformat_vnet_sw_interface, vnm, + &rpath.frp_sw_if_index)) + { + rpath.frp_weight = 1; + rpath.frp_proto = FIB_PROTOCOL_IP4; + } + + else if (unformat (line_input, "via %U %U", + unformat_ip6_address, + &rpath.frp_addr.ip6, + unformat_vnet_sw_interface, vnm, + &rpath.frp_sw_if_index)) + { + rpath.frp_weight = 1; + rpath.frp_proto = FIB_PROTOCOL_IP6; + } + else if (unformat (line_input, "via %U", + unformat_ip6_address, + &rpath.frp_addr.ip6)) + { + rpath.frp_fib_index = 0; + rpath.frp_weight = 1; + rpath.frp_sw_if_index = ~0; + rpath.frp_proto = FIB_PROTOCOL_IP6; + } + else if (unformat (line_input, "via %U", + unformat_ip4_address, + &rpath.frp_addr.ip4)) + { + rpath.frp_fib_index = 0; + rpath.frp_weight = 1; + rpath.frp_sw_if_index = ~0; + rpath.frp_proto = FIB_PROTOCOL_IP4; + } + else if (unformat (line_input, "l2-only")) + l2_only = 1; + else if (unformat (line_input, "multicast")) + is_multicast = 1; + else + { + error = clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + goto done; + } } if (is_del) { - vnet_mpls_tunnel_del(sw_if_index); + vnet_mpls_tunnel_del(sw_if_index); } else { - if (0 == vec_len(labels)) - { - error = clib_error_return (0, "No Output Labels '%U'", - format_unformat_error, line_input); - goto done; - } - - vec_add1(rpaths, rpath); - vnet_mpls_tunnel_add(rpaths, labels, l2_only, &sw_if_index); + if (0 == vec_len(rpath.frp_label_stack)) + { + error = clib_error_return (0, "No Output Labels '%U'", + format_unformat_error, line_input); + goto done; + } + + vec_add1(rpaths, rpath); + sw_if_index = vnet_mpls_tunnel_create(l2_only, is_multicast); + vnet_mpls_tunnel_path_add(sw_if_index, rpaths); } done: - vec_free(labels); vec_free(rpaths); unformat_free (line_input); @@ -638,7 +938,7 @@ done: ?*/ VLIB_CLI_COMMAND (create_mpls_tunnel_command, static) = { .path = "mpls tunnel", - .short_help = + .short_help = "mpls tunnel via [addr] [interface] [out-labels]", .function = vnet_create_mpls_tunnel_command_fn, }; @@ -647,19 +947,28 @@ static u8 * format_mpls_tunnel (u8 * s, va_list * args) { mpls_tunnel_t *mt = va_arg (*args, mpls_tunnel_t *); - int ii; + mpls_tunnel_attribute_t attr; + fib_path_ext_t *path_ext; s = format(s, "mpls_tunnel%d: sw_if_index:%d hw_if_index:%d", - mt - mpls_tunnel_pool, - mt->mt_sw_if_index, - mt->mt_hw_if_index); - s = format(s, "\n label-stack:\n "); - for (ii = 0; ii < vec_len(mt->mt_label_stack); ii++) - { - s = format(s, "%d, ", mt->mt_label_stack[ii]); + mt - mpls_tunnel_pool, + mt->mt_sw_if_index, + mt->mt_hw_if_index); + if (MPLS_TUNNEL_FLAG_NONE != mt->mt_flags) { + s = format(s, " \n flags:"); + FOR_EACH_MPLS_TUNNEL_ATTRIBUTE(attr) { + if ((1<mt_flags) { + s = format (s, "%s,", mpls_tunnel_attribute_names[attr]); + } + } } s = format(s, "\n via:\n"); s = fib_path_list_format(mt->mt_path_list, s); + s = format(s, " Extensions:"); + vec_foreach(path_ext, mt->mt_path_exts) + { + s = format(s, "\n %U", format_fib_path_ext, path_ext); + } s = format(s, "\n"); return (s); @@ -667,42 +976,42 @@ format_mpls_tunnel (u8 * s, va_list * args) static clib_error_t * show_mpls_tunnel_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) + unformat_input_t * input, + vlib_cli_command_t * cmd) { mpls_tunnel_t * mt; u32 mti = ~0; if (pool_elts (mpls_tunnel_pool) == 0) - vlib_cli_output (vm, "No MPLS tunnels configured..."); + vlib_cli_output (vm, "No MPLS tunnels configured..."); while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { - if (unformat (input, "%d", &mti)) - ; - else - break; + if (unformat (input, "%d", &mti)) + ; + else + break; } if (~0 == mti) { - pool_foreach (mt, mpls_tunnel_pool, - ({ - vlib_cli_output (vm, "[@%d] %U", - mt - mpls_tunnel_pool, - format_mpls_tunnel, mt); - })); + pool_foreach (mt, mpls_tunnel_pool, + ({ + vlib_cli_output (vm, "[@%d] %U", + mt - mpls_tunnel_pool, + format_mpls_tunnel, mt); + })); } else { - if (pool_is_free_index(mpls_tunnel_pool, mti)) - return clib_error_return (0, "Not atunnel index %d", mti); + if (pool_is_free_index(mpls_tunnel_pool, mti)) + return clib_error_return (0, "Not atunnel index %d", mti); - mt = pool_elt_at_index(mpls_tunnel_pool, mti); + mt = pool_elt_at_index(mpls_tunnel_pool, mti); - vlib_cli_output (vm, "[@%d] %U", - mt - mpls_tunnel_pool, - format_mpls_tunnel, mt); + vlib_cli_output (vm, "[@%d] %U", + mt - mpls_tunnel_pool, + format_mpls_tunnel, mt); } return 0; @@ -715,7 +1024,7 @@ show_mpls_tunnel_command_fn (vlib_main_t * vm, * @cliexstart{sh mpls tunnel 2} * [@2] mpls_tunnel2: sw_if_index:5 hw_if_index:5 * label-stack: - * 3, + * 3, * via: * index:26 locks:1 proto:ipv4 uPRF-list:26 len:1 itfs:[2, ] * index:26 pl-index:26 ipv4 weight=1 attached-nexthop: oper-flags:resolved, @@ -743,7 +1052,7 @@ mpls_tunnel_from_fib_node (fib_node_t *node) */ static fib_node_back_walk_rc_t mpls_tunnel_back_walk (fib_node_t *node, - fib_node_back_walk_ctx_t *ctx) + fib_node_back_walk_ctx_t *ctx) { mpls_tunnel_restack(mpls_tunnel_from_fib_node(node)); diff --git a/src/vnet/mpls/mpls_tunnel.h b/src/vnet/mpls/mpls_tunnel.h index ee56c0fc..0b55d0db 100644 --- a/src/vnet/mpls/mpls_tunnel.h +++ b/src/vnet/mpls/mpls_tunnel.h @@ -17,6 +17,31 @@ #define __MPLS_TUNNEL_H__ #include +#include + +typedef enum mpls_tunnel_attribute_t_ +{ + MPLS_TUNNEL_ATTRIBUTE_FIRST = 0, + /** + * @brief The tunnel has an underlying multicast LSP + */ + MPLS_TUNNEL_ATTRIBUTE_MCAST = MPLS_TUNNEL_ATTRIBUTE_FIRST, + MPLS_TUNNEL_ATTRIBUTE_LAST = MPLS_TUNNEL_ATTRIBUTE_MCAST, +} mpls_tunnel_attribute_t; + +#define MPLS_TUNNEL_ATTRIBUTES { \ + [MPLS_TUNNEL_ATTRIBUTE_MCAST] = "multicast", \ +} +#define FOR_EACH_MPLS_TUNNEL_ATTRIBUTE(_item) \ + for (_item = MPLS_TUNNEL_ATTRIBUTE_FIRST; \ + _item < MPLS_TUNNEL_ATTRIBUTE_LAST; \ + _item++) + +typedef enum mpls_tunnel_flag_t_ { + MPLS_TUNNEL_FLAG_NONE = 0, + MPLS_TUNNEL_FLAG_MCAST = (1 << MPLS_TUNNEL_ATTRIBUTE_MCAST), +} __attribute__ ((packed)) mpls_tunnel_flags_t; + /** * @brief A uni-directional MPLS tunnel @@ -28,6 +53,11 @@ typedef struct mpls_tunnel_t_ */ fib_node_t mt_node; + /** + * @brief Tunnel flags + */ + mpls_tunnel_flags_t mt_flags; + /** * @brief If the tunnel is an L2 tunnel, this is the link type ETHERNET * adjacency @@ -50,9 +80,9 @@ typedef struct mpls_tunnel_t_ u32 mt_sibling_index; /** - * @brief The Label stack to apply to egress packets + * A vector of path extensions o hold the label stack for each path */ - mpls_label_t *mt_label_stack; + fib_path_ext_t *mt_path_exts; /** * @brief Flag to indicate the tunnel is only for L2 traffic, that is @@ -74,12 +104,27 @@ typedef struct mpls_tunnel_t_ /** * @brief Create a new MPLS tunnel + * @return the SW Interface index of the newly created tuneel */ -extern void vnet_mpls_tunnel_add (fib_route_path_t *rpath, - mpls_label_t *label_stack, - u8 l2_only, - u32 *sw_if_index); +extern u32 vnet_mpls_tunnel_create (u8 l2_only, + u8 is_multicast); +/** + * @brief Add a path to an MPLS tunnel + */ +extern void vnet_mpls_tunnel_path_add (u32 sw_if_index, + fib_route_path_t *rpath); + +/** + * @brief remove a path from a tunnel. + * @return the number of remaining paths. 0 implies the tunnel can be deleted + */ +extern int vnet_mpls_tunnel_path_remove (u32 sw_if_index, + fib_route_path_t *rpath); + +/** + * @brief Delete an MPLS tunnel + */ extern void vnet_mpls_tunnel_del (u32 sw_if_index); extern const mpls_tunnel_t *mpls_tunnel_get(u32 index); diff --git a/src/vnet/mpls/mpls_types.h b/src/vnet/mpls/mpls_types.h index d7c629df..b1075cdd 100644 --- a/src/vnet/mpls/mpls_types.h +++ b/src/vnet/mpls/mpls_types.h @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef __MPLS_TYPES_H__ #define __MPLS_TYPES_H__ @@ -36,4 +50,10 @@ (((_lbl) > MPLS_IETF_MIN_UNRES_LABEL) && \ ((_lbl) <= MPLS_IETF_MAX_UNRES_LABEL)) +/** + * The top bit of the index, which is the result of the MPLS lookup + * is used to determine if the DPO is a load-balance or a replicate + */ +#define MPLS_IS_REPLICATE 0x80000000 + #endif diff --git a/src/vnet/srp/interface.c b/src/vnet/srp/interface.c index d427cc3c..44e2b0d6 100644 --- a/src/vnet/srp/interface.c +++ b/src/vnet/srp/interface.c @@ -58,7 +58,7 @@ srp_build_rewrite (vnet_main_t * vnm, #define _(a,b) case VNET_LINK_##a: type = ETHERNET_TYPE_##b; break _ (IP4, IP4); _ (IP6, IP6); - _ (MPLS, MPLS_UNICAST); + _ (MPLS, MPLS); _ (ARP, ARP); #undef _ default: diff --git a/test/test_ip_mcast.py b/test/test_ip_mcast.py index 36d597a7..c1397d70 100644 --- a/test/test_ip_mcast.py +++ b/test/test_ip_mcast.py @@ -622,6 +622,7 @@ class TestIPMcast(VppTestCase): (MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT | MRouteItfFlags.MFIB_ITF_FLAG_NEGATE_SIGNAL)) + self.vapi.cli("clear trace") tx = self._mcast_connected_send_stream("232.1.1.1") signals = self.vapi.mfib_signal_dump() diff --git a/test/test_mpls.py b/test/test_mpls.py index fc832644..700b7091 100644 --- a/test/test_mpls.py +++ b/test/test_mpls.py @@ -5,7 +5,9 @@ import socket from framework import VppTestCase, VppTestRunner from vpp_ip_route import VppIpRoute, VppRoutePath, VppMplsRoute, \ - VppMplsIpBind + VppMplsIpBind, VppIpMRoute, VppMRoutePath, \ + MRouteItfFlags, MRouteEntryFlags +from vpp_mpls_tunnel_interface import VppMPLSTunnelInterface from scapy.packet import Raw from scapy.layers.l2 import Ether @@ -21,7 +23,7 @@ class TestMPLS(VppTestCase): super(TestMPLS, self).setUp() # create 2 pg interfaces - self.create_pg_interfaces(range(2)) + self.create_pg_interfaces(range(4)) # setup both interfaces # assign them different tables. @@ -53,10 +55,12 @@ class TestMPLS(VppTestCase): mpls_labels, mpls_ttl=255, ping=0, - ip_itf=None): + ip_itf=None, + dst_ip=None, + n=257): self.reset_packet_infos() pkts = [] - for i in range(0, 257): + for i in range(0, n): info = self.create_packet_info(src_if, src_if) payload = self.info_to_payload(info) p = Ether(dst=src_if.local_mac, src=src_if.remote_mac) @@ -67,9 +71,14 @@ class TestMPLS(VppTestCase): else: p = p / MPLS(label=mpls_labels[ii], ttl=mpls_ttl, s=0) if not ping: - p = (p / IP(src=src_if.local_ip4, dst=src_if.remote_ip4) / - UDP(sport=1234, dport=1234) / - Raw(payload)) + if not dst_ip: + p = (p / IP(src=src_if.local_ip4, dst=src_if.remote_ip4) / + UDP(sport=1234, dport=1234) / + Raw(payload)) + else: + p = (p / IP(src=src_if.local_ip4, dst=dst_ip) / + UDP(sport=1234, dport=1234) / + Raw(payload)) else: p = (p / IP(src=ip_itf.remote_ip4, dst=ip_itf.local_ip4) / @@ -254,6 +263,13 @@ class TestMPLS(VppTestCase): except: raise + def send_and_assert_no_replies(self, intf, pkts, remark): + intf.add_stream(pkts) + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + for i in self.pg_interfaces: + i.assert_nothing_captured(remark=remark) + def test_swap(self): """ MPLS label swap tests """ @@ -278,7 +294,7 @@ class TestMPLS(VppTestCase): self.pg_start() rx = self.pg0.get_capture() - self.verify_capture_labelled_ip4(self.pg0, rx, tx, [33]) + self.verify_capture_labelled(self.pg0, rx, tx, [33]) # # A simple MPLS xconnect - non-eos label in label out @@ -358,7 +374,7 @@ class TestMPLS(VppTestCase): self.pg_start() rx = self.pg0.get_capture() - self.verify_capture_labelled_ip4(self.pg0, rx, tx, [33, 44, 45]) + self.verify_capture_labelled(self.pg0, rx, tx, [33, 44, 45], num=2) # # A recursive non-EOS x-connect, which resolves through another @@ -576,25 +592,19 @@ class TestMPLS(VppTestCase): # # Create a tunnel with a single out label # - nh_addr = socket.inet_pton(socket.AF_INET, self.pg0.remote_ip4) - - reply = self.vapi.mpls_tunnel_add_del( - 0xffffffff, # don't know the if index yet - 1, # IPv4 next-hop - nh_addr, - self.pg0.sw_if_index, - 0, # next-hop-table-id - 1, # next-hop-weight - 2, # num-out-labels, - [44, 46]) - self.vapi.sw_interface_set_flags(reply.sw_if_index, admin_up_down=1) + mpls_tun = VppMPLSTunnelInterface(self, + [VppRoutePath(self.pg0.remote_ip4, + self.pg0.sw_if_index, + labels=[44, 46])]) + mpls_tun.add_vpp_config() + mpls_tun.admin_up() # # add an unlabelled route through the new tunnel # route_10_0_0_3 = VppIpRoute(self, "10.0.0.3", 32, [VppRoutePath("0.0.0.0", - reply.sw_if_index)]) + mpls_tun._sw_if_index)]) route_10_0_0_3.add_vpp_config() self.vapi.cli("clear trace") @@ -738,6 +748,229 @@ class TestMPLS(VppTestCase): route_35_eos.remove_vpp_config() route_34_eos.remove_vpp_config() + def test_interface_rx(self): + """ MPLS Interface Receive """ + + # + # Add a non-recursive route that will forward the traffic + # post-interface-rx + # + route_10_0_0_1 = VppIpRoute(self, "10.0.0.1", 32, + table_id=1, + paths=[VppRoutePath(self.pg1.remote_ip4, + self.pg1.sw_if_index)]) + route_10_0_0_1.add_vpp_config() + + # + # An interface receive label that maps traffic to RX on interface + # pg1 + # by injecting the packet in on pg0, which is in table 0 + # doing an interface-rx on pg1 and matching a route in table 1 + # if the packet egresses, then we must have swapped to pg1 + # so as to have matched the route in table 1 + # + route_34_eos = VppMplsRoute(self, 34, 1, + [VppRoutePath("0.0.0.0", + self.pg1.sw_if_index, + is_interface_rx=1)]) + route_34_eos.add_vpp_config() + + # + # ping an interface in the default table + # PG0 is in the default table + # + self.vapi.cli("clear trace") + tx = self.create_stream_labelled_ip4(self.pg0, [34], n=257, + dst_ip="10.0.0.1") + self.pg0.add_stream(tx) + + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + + rx = self.pg1.get_capture(257) + self.verify_capture_ip4(self.pg1, rx, tx) + + def test_mcast_mid_point(self): + """ MPLS Multicast Mid Point """ + + # + # Add a non-recursive route that will forward the traffic + # post-interface-rx + # + route_10_0_0_1 = VppIpRoute(self, "10.0.0.1", 32, + table_id=1, + paths=[VppRoutePath(self.pg1.remote_ip4, + self.pg1.sw_if_index)]) + route_10_0_0_1.add_vpp_config() + + # + # Add a mcast entry that replicate to pg2 and pg3 + # and replicate to a interface-rx (like a bud node would) + # + route_3400_eos = VppMplsRoute(self, 3400, 1, + [VppRoutePath(self.pg2.remote_ip4, + self.pg2.sw_if_index, + labels=[3401]), + VppRoutePath(self.pg3.remote_ip4, + self.pg3.sw_if_index, + labels=[3402]), + VppRoutePath("0.0.0.0", + self.pg1.sw_if_index, + is_interface_rx=1)], + is_multicast=1) + route_3400_eos.add_vpp_config() + + # + # ping an interface in the default table + # PG0 is in the default table + # + self.vapi.cli("clear trace") + tx = self.create_stream_labelled_ip4(self.pg0, [3400], n=257, + dst_ip="10.0.0.1") + self.pg0.add_stream(tx) + + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + + rx = self.pg1.get_capture(257) + self.verify_capture_ip4(self.pg1, rx, tx) + + rx = self.pg2.get_capture(257) + self.verify_capture_labelled(self.pg2, rx, tx, [3401]) + rx = self.pg3.get_capture(257) + self.verify_capture_labelled(self.pg3, rx, tx, [3402]) + + def test_mcast_head(self): + """ MPLS Multicast Head-end """ + + # + # Create a multicast tunnel with two replications + # + mpls_tun = VppMPLSTunnelInterface(self, + [VppRoutePath(self.pg2.remote_ip4, + self.pg2.sw_if_index, + labels=[42]), + VppRoutePath(self.pg3.remote_ip4, + self.pg3.sw_if_index, + labels=[43])], + is_multicast=1) + mpls_tun.add_vpp_config() + mpls_tun.admin_up() + + # + # add an unlabelled route through the new tunnel + # + route_10_0_0_3 = VppIpRoute(self, "10.0.0.3", 32, + [VppRoutePath("0.0.0.0", + mpls_tun._sw_if_index)]) + route_10_0_0_3.add_vpp_config() + + self.vapi.cli("clear trace") + tx = self.create_stream_ip4(self.pg0, "10.0.0.3") + self.pg0.add_stream(tx) + + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + + rx = self.pg2.get_capture(257) + self.verify_capture_tunneled_ip4(self.pg0, rx, tx, [42]) + rx = self.pg3.get_capture(257) + self.verify_capture_tunneled_ip4(self.pg0, rx, tx, [43]) + + # + # An an IP multicast route via the tunnel + # A (*,G). + # one accepting interface, pg0, 1 forwarding interface via the tunnel + # + route_232_1_1_1 = VppIpMRoute( + self, + "0.0.0.0", + "232.1.1.1", 32, + MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE, + [VppMRoutePath(self.pg0.sw_if_index, + MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT), + VppMRoutePath(mpls_tun._sw_if_index, + MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)]) + route_232_1_1_1.add_vpp_config() + + self.vapi.cli("clear trace") + tx = self.create_stream_ip4(self.pg0, "232.1.1.1") + self.pg0.add_stream(tx) + + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + + rx = self.pg2.get_capture(257) + self.verify_capture_tunneled_ip4(self.pg0, rx, tx, [42]) + rx = self.pg3.get_capture(257) + self.verify_capture_tunneled_ip4(self.pg0, rx, tx, [43]) + + def test_mcast_tail(self): + """ MPLS Multicast Tail """ + + # + # Add a multicast route that will forward the traffic + # post-disposition + # + route_232_1_1_1 = VppIpMRoute( + self, + "0.0.0.0", + "232.1.1.1", 32, + MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE, + table_id=1, + paths=[VppMRoutePath(self.pg1.sw_if_index, + MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)]) + route_232_1_1_1.add_vpp_config() + + # + # An interface receive label that maps traffic to RX on interface + # pg1 + # by injecting the packet in on pg0, which is in table 0 + # doing an rpf-id and matching a route in table 1 + # if the packet egresses, then we must have matched the route in + # table 1 + # + route_34_eos = VppMplsRoute(self, 34, 1, + [VppRoutePath("0.0.0.0", + self.pg1.sw_if_index, + nh_table_id=1, + rpf_id=55)], + is_multicast=1) + + route_34_eos.add_vpp_config() + + # + # Drop due to interface lookup miss + # + self.vapi.cli("clear trace") + tx = self.create_stream_labelled_ip4(self.pg0, [34], + dst_ip="232.1.1.1", n=1) + self.send_and_assert_no_replies(self.pg0, tx, "RPF-ID drop none") + + # + # set the RPF-ID of the enrtry to match the input packet's + # + route_232_1_1_1.update_rpf_id(55) + + self.vapi.cli("clear trace") + tx = self.create_stream_labelled_ip4(self.pg0, [34], + dst_ip="232.1.1.1", n=257) + self.pg0.add_stream(tx) + + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + + rx = self.pg1.get_capture(257) + self.verify_capture_ip4(self.pg1, rx, tx) + + # + # set the RPF-ID of the enrtry to not match the input packet's + # + route_232_1_1_1.update_rpf_id(56) + tx = self.create_stream_labelled_ip4(self.pg0, [34], + dst_ip="232.1.1.1") + self.send_and_assert_no_replies(self.pg0, tx, "RPF-ID drop 56") + class TestMPLSDisabled(VppTestCase): """ MPLS disabled """ diff --git a/test/vpp_ip_route.py b/test/vpp_ip_route.py index faf5f801..d6146f28 100644 --- a/test/vpp_ip_route.py +++ b/test/vpp_ip_route.py @@ -55,15 +55,24 @@ class VppRoutePath(object): nh_table_id=0, labels=[], nh_via_label=MPLS_LABEL_INVALID, - is_ip6=0): + is_ip6=0, + rpf_id=0, + is_interface_rx=0): self.nh_itf = nh_sw_if_index self.nh_table_id = nh_table_id self.nh_via_label = nh_via_label self.nh_labels = labels + self.weight = 1 + self.rpf_id = rpf_id if is_ip6: self.nh_addr = inet_pton(AF_INET6, nh_addr) else: self.nh_addr = inet_pton(AF_INET, nh_addr) + self.is_interface_rx = is_interface_rx + self.is_rpf_id = 0 + if rpf_id != 0: + self.is_rpf_id = 1 + self.nh_itf = rpf_id class VppMRoutePath(VppRoutePath): @@ -176,13 +185,15 @@ class VppIpMRoute(VppObject): """ def __init__(self, test, src_addr, grp_addr, - grp_addr_len, e_flags, paths, table_id=0, is_ip6=0): + grp_addr_len, e_flags, paths, table_id=0, + rpf_id=0, is_ip6=0): self._test = test self.paths = paths self.grp_addr_len = grp_addr_len self.table_id = table_id self.e_flags = e_flags self.is_ip6 = is_ip6 + self.rpf_id = rpf_id if is_ip6: self.grp_addr = inet_pton(AF_INET6, grp_addr) @@ -199,6 +210,7 @@ class VppIpMRoute(VppObject): self.e_flags, path.nh_itf, path.nh_i_flags, + rpf_id=self.rpf_id, table_id=self.table_id, is_ipv6=self.is_ip6) self._test.registry.register(self, self._test.logger) @@ -226,6 +238,18 @@ class VppIpMRoute(VppObject): table_id=self.table_id, is_ipv6=self.is_ip6) + def update_rpf_id(self, rpf_id): + self.rpf_id = rpf_id + self._test.vapi.ip_mroute_add_del(self.src_addr, + self.grp_addr, + self.grp_addr_len, + self.e_flags, + 0xffffffff, + 0, + rpf_id=self.rpf_id, + table_id=self.table_id, + is_ipv6=self.is_ip6) + def update_path_flags(self, itf, flags): for path in self.paths: if path.nh_itf == itf: @@ -342,14 +366,17 @@ class VppMplsRoute(VppObject): MPLS Route/LSP """ - def __init__(self, test, local_label, eos_bit, paths, table_id=0): + def __init__(self, test, local_label, eos_bit, paths, table_id=0, + is_multicast=0): self._test = test self.paths = paths self.local_label = local_label self.eos_bit = eos_bit self.table_id = table_id + self.is_multicast = is_multicast def add_vpp_config(self): + is_multipath = len(self.paths) > 1 for path in self.paths: self._test.vapi.mpls_route_add_del( self.local_label, @@ -357,7 +384,11 @@ class VppMplsRoute(VppObject): 1, path.nh_addr, path.nh_itf, + is_multicast=self.is_multicast, + is_multipath=is_multipath, table_id=self.table_id, + is_interface_rx=path.is_interface_rx, + is_rpf_id=path.is_rpf_id, next_hop_out_label_stack=path.nh_labels, next_hop_n_out_labels=len( path.nh_labels), @@ -372,6 +403,7 @@ class VppMplsRoute(VppObject): 1, path.nh_addr, path.nh_itf, + is_rpf_id=path.is_rpf_id, table_id=self.table_id, is_add=0) diff --git a/test/vpp_mpls_tunnel_interface.py b/test/vpp_mpls_tunnel_interface.py new file mode 100644 index 00000000..f2001574 --- /dev/null +++ b/test/vpp_mpls_tunnel_interface.py @@ -0,0 +1,46 @@ + +from vpp_interface import VppInterface +from vpp_ip_route import VppRoutePath +import socket + + +class VppMPLSTunnelInterface(VppInterface): + """ + VPP MPLS Tunnel interface + """ + + def __init__(self, test, paths, is_multicast=0): + """ Create MPLS Tunnel interface """ + self._sw_if_index = 0 + super(VppMPLSTunnelInterface, self).__init__(test) + self._test = test + self.t_paths = paths + self.is_multicast = is_multicast + + def add_vpp_config(self): + self._sw_if_index = 0xffffffff + for path in self.t_paths: + reply = self.test.vapi.mpls_tunnel_add_del( + self._sw_if_index, + 1, # IPv4 next-hop + path.nh_addr, + path.nh_itf, + path.nh_table_id, + path.weight, + next_hop_out_label_stack=path.nh_labels, + next_hop_n_out_labels=len(path.nh_labels), + is_multicast=self.is_multicast) + self._sw_if_index = reply.sw_if_index + + def remove_vpp_config(self): + for path in self.t_paths: + reply = self.test.vapi.mpls_tunnel_add_del( + self.sw_if_index, + 1, # IPv4 next-hop + path.nh_addr, + path.nh_itf, + path.nh_table_id, + path.weight, + next_hop_out_label_stack=path.nh_labels, + next_hop_n_out_labels=len(path.nh_labels), + is_add=0) diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py index e8025dff..ceb684b7 100644 --- a/test/vpp_papi_provider.py +++ b/test/vpp_papi_provider.py @@ -849,6 +849,9 @@ class VppPapiProvider(object): create_vrf_if_needed=0, is_resolve_host=0, is_resolve_attached=0, + is_interface_rx=0, + is_rpf_id=0, + is_multicast=0, is_add=1, is_drop=0, is_multipath=0, @@ -872,6 +875,7 @@ class VppPapiProvider(object): :param is_local: (Default value = 0) :param is_classify: (Default value = 0) :param is_multipath: (Default value = 0) + :param is_multicast: (Default value = 0) :param is_resolve_host: (Default value = 0) :param is_resolve_attached: (Default value = 0) :param not_last: (Default value = 0) @@ -889,8 +893,11 @@ class VppPapiProvider(object): 'mr_is_add': is_add, 'mr_is_classify': is_classify, 'mr_is_multipath': is_multipath, + 'mr_is_multicast': is_multicast, 'mr_is_resolve_host': is_resolve_host, 'mr_is_resolve_attached': is_resolve_attached, + 'mr_is_interface_rx': is_interface_rx, + 'mr_is_rpf_id': is_rpf_id, 'mr_next_hop_proto_is_ip4': next_hop_proto_is_ip4, 'mr_next_hop_weight': next_hop_weight, 'mr_next_hop': next_hop_address, @@ -936,7 +943,8 @@ class VppPapiProvider(object): next_hop_via_label=MPLS_LABEL_INVALID, create_vrf_if_needed=0, is_add=1, - l2_only=0): + l2_only=0, + is_multicast=0): """ :param dst_address_length: @@ -956,8 +964,8 @@ class VppPapiProvider(object): :param is_multipath: (Default value = 0) :param is_resolve_host: (Default value = 0) :param is_resolve_attached: (Default value = 0) - :param not_last: (Default value = 0) :param next_hop_weight: (Default value = 1) + :param is_multicast: (Default value = 0) """ return self.api( @@ -965,6 +973,7 @@ class VppPapiProvider(object): {'mt_sw_if_index': tun_sw_if_index, 'mt_is_add': is_add, 'mt_l2_only': l2_only, + 'mt_is_multicast': is_multicast, 'mt_next_hop_proto_is_ip4': next_hop_proto_is_ip4, 'mt_next_hop_weight': next_hop_weight, 'mt_next_hop': next_hop_address, @@ -1469,6 +1478,7 @@ class VppPapiProvider(object): e_flags, next_hop_sw_if_index, i_flags, + rpf_id=0, table_id=0, create_vrf_if_needed=0, is_add=1, @@ -1481,6 +1491,8 @@ class VppPapiProvider(object): {'next_hop_sw_if_index': next_hop_sw_if_index, 'entry_flags': e_flags, 'itf_flags': i_flags, + 'table_id': table_id, + 'rpf_id': rpf_id, 'create_vrf_if_needed': create_vrf_if_needed, 'is_add': is_add, 'is_ipv6': is_ipv6, -- cgit 1.2.3-korg