diff options
Diffstat (limited to 'src/vnet/sr/sr_policy_rewrite.c')
| -rwxr-xr-x | src/vnet/sr/sr_policy_rewrite.c | 3253 |
1 files changed, 3253 insertions, 0 deletions
diff --git a/src/vnet/sr/sr_policy_rewrite.c b/src/vnet/sr/sr_policy_rewrite.c new file mode 100755 index 00000000000..1f8bdca5143 --- /dev/null +++ b/src/vnet/sr/sr_policy_rewrite.c @@ -0,0 +1,3253 @@ +/* + * sr_policy_rewrite.c: ipv6 sr policy creation + * + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file + * @brief SR policy creation and application + * + * Create an SR policy. + * An SR policy can be either of 'default' type or 'spray' type + * An SR policy has attached a list of SID lists. + * In case the SR policy is a default one it will load balance among them. + * An SR policy has associated a BindingSID. + * In case any packet arrives with IPv6 DA == BindingSID then the SR policy + * associated to such bindingSID will be applied to such packet. + * + * SR policies can be applied either by using IPv6 encapsulation or + * SRH insertion. Both methods can be found on this file. + * + * Traffic input usually is IPv6 packets. However it is possible to have + * IPv4 packets or L2 frames. (that are encapsulated into IPv6 with SRH) + * + * This file provides the appropiates VPP graph nodes to do any of these + * methods. + * + */ + +#include <vlib/vlib.h> +#include <vnet/vnet.h> +#include <vnet/sr/sr.h> +#include <vnet/ip/ip.h> +#include <vnet/sr/sr_packet.h> +#include <vnet/ip/ip6_packet.h> +#include <vnet/fib/ip6_fib.h> +#include <vnet/dpo/dpo.h> +#include <vnet/dpo/replicate_dpo.h> + +#include <vppinfra/error.h> +#include <vppinfra/elog.h> + +/** + * @brief SR policy rewrite trace + */ +typedef struct +{ + ip6_address_t src, dst; +} sr_policy_rewrite_trace_t; + +/* Graph arcs */ +#define foreach_sr_policy_rewrite_next \ +_(IP6_LOOKUP, "ip6-lookup") \ +_(ERROR, "error-drop") + +typedef enum +{ +#define _(s,n) SR_POLICY_REWRITE_NEXT_##s, + foreach_sr_policy_rewrite_next +#undef _ + SR_POLICY_REWRITE_N_NEXT, +} sr_policy_rewrite_next_t; + +/* SR rewrite errors */ +#define foreach_sr_policy_rewrite_error \ +_(INTERNAL_ERROR, "Segment Routing undefined error") \ +_(BSID_ZERO, "BSID with SL = 0") \ +_(COUNTER_TOTAL, "SR steered IPv6 packets") \ +_(COUNTER_ENCAP, "SR: Encaps packets") \ +_(COUNTER_INSERT, "SR: SRH inserted packets") \ +_(COUNTER_BSID, "SR: BindingSID steered packets") + +typedef enum +{ +#define _(sym,str) SR_POLICY_REWRITE_ERROR_##sym, + foreach_sr_policy_rewrite_error +#undef _ + SR_POLICY_REWRITE_N_ERROR, +} sr_policy_rewrite_error_t; + +static char *sr_policy_rewrite_error_strings[] = { +#define _(sym,string) string, + foreach_sr_policy_rewrite_error +#undef _ +}; + +/** + * @brief Dynamically added SR SL DPO type + */ +static dpo_type_t sr_pr_encaps_dpo_type; +static dpo_type_t sr_pr_insert_dpo_type; +static dpo_type_t sr_pr_bsid_encaps_dpo_type; +static dpo_type_t sr_pr_bsid_insert_dpo_type; + +/** + * @brief IPv6 SA for encapsulated packets + */ +static ip6_address_t sr_pr_encaps_src; + +/******************* SR rewrite set encaps IPv6 source addr *******************/ +/* Note: This is temporal. We don't know whether to follow this path or + take the ip address of a loopback interface or even the OIF */ + +static clib_error_t * +set_sr_src_command_fn (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat + (input, "addr %U", unformat_ip6_address, &sr_pr_encaps_src)) + return 0; + else + return clib_error_return (0, "No address specified"); + } + return clib_error_return (0, "No address specified"); +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (set_sr_src_command, static) = { + .path = "set sr encaps source", + .short_help = "set sr encaps source addr <ip6_addr>", + .function = set_sr_src_command_fn, +}; +/* *INDENT-ON* */ + +/*********************** SR rewrite string computation ************************/ +/** + * @brief SR rewrite string computation for IPv6 encapsulation (inline) + * + * @param sl is a vector of IPv6 addresses composing the Segment List + * + * @return precomputed rewrite string for encapsulation + */ +static inline u8 * +compute_rewrite_encaps (ip6_address_t * sl) +{ + ip6_header_t *iph; + ip6_sr_header_t *srh; + ip6_address_t *addrp, *this_address; + u32 header_length = 0; + u8 *rs = NULL; + + header_length = 0; + header_length += IPv6_DEFAULT_HEADER_LENGTH; + if (vec_len (sl) > 1) + { + header_length += sizeof (ip6_sr_header_t); + header_length += vec_len (sl) * sizeof (ip6_address_t); + } + + vec_validate (rs, header_length - 1); + + iph = (ip6_header_t *) rs; + iph->ip_version_traffic_class_and_flow_label = + clib_host_to_net_u32 (0 | ((6 & 0xF) << 28)); + iph->src_address.as_u64[0] = sr_pr_encaps_src.as_u64[0]; + iph->src_address.as_u64[1] = sr_pr_encaps_src.as_u64[1]; + iph->payload_length = header_length - IPv6_DEFAULT_HEADER_LENGTH; + iph->protocol = IP_PROTOCOL_IPV6; + iph->hop_limit = IPv6_DEFAULT_HOP_LIMIT; + + srh = (ip6_sr_header_t *) (iph + 1); + iph->protocol = IP_PROTOCOL_IPV6_ROUTE; + srh->protocol = IP_PROTOCOL_IPV6; + srh->type = ROUTING_HEADER_TYPE_SR; + srh->segments_left = vec_len (sl) - 1; + srh->first_segment = vec_len (sl) - 1; + srh->length = ((sizeof (ip6_sr_header_t) + + (vec_len (sl) * sizeof (ip6_address_t))) / 8) - 1; + srh->flags = 0x00; + srh->reserved = 0x00; + addrp = srh->segments + vec_len (sl) - 1; + vec_foreach (this_address, sl) + { + clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t)); + addrp--; + } + iph->dst_address.as_u64[0] = sl->as_u64[0]; + iph->dst_address.as_u64[1] = sl->as_u64[1]; + return rs; +} + +/** + * @brief SR rewrite string computation for SRH insertion (inline) + * + * @param sl is a vector of IPv6 addresses composing the Segment List + * + * @return precomputed rewrite string for SRH insertion + */ +static inline u8 * +compute_rewrite_insert (ip6_address_t * sl) +{ + ip6_sr_header_t *srh; + ip6_address_t *addrp, *this_address; + u32 header_length = 0; + u8 *rs = NULL; + + header_length = 0; + header_length += sizeof (ip6_sr_header_t); + header_length += (vec_len (sl) + 1) * sizeof (ip6_address_t); + + vec_validate (rs, header_length - 1); + + srh = (ip6_sr_header_t *) rs; + srh->type = ROUTING_HEADER_TYPE_SR; + srh->segments_left = vec_len (sl); + srh->first_segment = vec_len (sl); + srh->length = ((sizeof (ip6_sr_header_t) + + ((vec_len (sl) + 1) * sizeof (ip6_address_t))) / 8) - 1; + srh->flags = 0x00; + srh->reserved = 0x0000; + addrp = srh->segments + vec_len (sl); + vec_foreach (this_address, sl) + { + clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t)); + addrp--; + } + return rs; +} + +/** + * @brief SR rewrite string computation for SRH insertion with BSID (inline) + * + * @param sl is a vector of IPv6 addresses composing the Segment List + * + * @return precomputed rewrite string for SRH insertion with BSID + */ +static inline u8 * +compute_rewrite_bsid (ip6_address_t * sl) +{ + ip6_sr_header_t *srh; + ip6_address_t *addrp, *this_address; + u32 header_length = 0; + u8 *rs = NULL; + + header_length = 0; + header_length += sizeof (ip6_sr_header_t); + header_length += vec_len (sl) * sizeof (ip6_address_t); + + vec_validate (rs, header_length - 1); + + srh = (ip6_sr_header_t *) rs; + srh->type = ROUTING_HEADER_TYPE_SR; + srh->segments_left = vec_len (sl) - 1; + srh->first_segment = vec_len (sl) - 1; + srh->length = ((sizeof (ip6_sr_header_t) + + (vec_len (sl) * sizeof (ip6_address_t))) / 8) - 1; + srh->flags = 0x00; + srh->reserved = 0x0000; + addrp = srh->segments + vec_len (sl) - 1; + vec_foreach (this_address, sl) + { + clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t)); + addrp--; + } + return rs; +} + +/*************************** SR LB helper functions **************************/ +/** + * @brief Creates a Segment List and adds it to an SR policy + * + * Creates a Segment List and adds it to the SR policy. Notice that the SL are + * not necessarily unique. Hence there might be two Segment List within the + * same SR Policy with exactly the same segments and same weight. + * + * @param sr_policy is the SR policy where the SL will be added + * @param sl is a vector of IPv6 addresses composing the Segment List + * @param weight is the weight of the SegmentList (for load-balancing purposes) + * @param is_encap represents the mode (SRH insertion vs Encapsulation) + * + * @return pointer to the just created segment list + */ +static inline ip6_sr_sl_t * +create_sl (ip6_sr_policy_t * sr_policy, ip6_address_t * sl, u32 weight, + u8 is_encap) +{ + ip6_sr_main_t *sm = &sr_main; + ip6_sr_sl_t *segment_list; + + pool_get (sm->sid_lists, segment_list); + memset (segment_list, 0, sizeof (*segment_list)); + + vec_add1 (sr_policy->segments_lists, segment_list - sm->sid_lists); + + /* Fill in segment list */ + segment_list->weight = + (weight != (u32) ~ 0 ? weight : SR_SEGMENT_LIST_WEIGHT_DEFAULT); + segment_list->segments = vec_dup (sl); + + if (is_encap) + { + segment_list->rewrite = compute_rewrite_encaps (sl); + segment_list->rewrite_bsid = segment_list->rewrite; + } + else + { + segment_list->rewrite = compute_rewrite_insert (sl); + segment_list->rewrite_bsid = compute_rewrite_bsid (sl); + } + + /* Create DPO */ + dpo_reset (&segment_list->bsid_dpo); + dpo_reset (&segment_list->ip6_dpo); + dpo_reset (&segment_list->ip4_dpo); + + if (is_encap) + { + dpo_set (&segment_list->ip6_dpo, sr_pr_encaps_dpo_type, DPO_PROTO_IP6, + segment_list - sm->sid_lists); + dpo_set (&segment_list->ip4_dpo, sr_pr_encaps_dpo_type, DPO_PROTO_IP4, + segment_list - sm->sid_lists); + dpo_set (&segment_list->bsid_dpo, sr_pr_bsid_encaps_dpo_type, + DPO_PROTO_IP6, segment_list - sm->sid_lists); + } + else + { + dpo_set (&segment_list->ip6_dpo, sr_pr_insert_dpo_type, DPO_PROTO_IP6, + segment_list - sm->sid_lists); + dpo_set (&segment_list->bsid_dpo, sr_pr_bsid_insert_dpo_type, + DPO_PROTO_IP6, segment_list - sm->sid_lists); + } + + return segment_list; +} + +/** + * @brief Updates the Load Balancer after an SR Policy change + * + * @param sr_policy is the modified SR Policy + */ +static inline void +update_lb (ip6_sr_policy_t * sr_policy) +{ + flow_hash_config_t fhc; + u32 *sl_index; + ip6_sr_sl_t *segment_list; + ip6_sr_main_t *sm = &sr_main; + load_balance_path_t path; + load_balance_path_t *ip4_path_vector = 0; + load_balance_path_t *ip6_path_vector = 0; + load_balance_path_t *b_path_vector = 0; + + /* In case LB does not exist, create it */ + if (!dpo_id_is_valid (&sr_policy->bsid_dpo)) + { + fib_prefix_t pfx = { + .fp_proto = FIB_PROTOCOL_IP6, + .fp_len = 128, + .fp_addr = { + .ip6 = sr_policy->bsid, + } + }; + + /* Add FIB entry for BSID */ + fhc = fib_table_get_flow_hash_config (sr_policy->fib_table, + dpo_proto_to_fib (DPO_PROTO_IP6)); + + dpo_set (&sr_policy->bsid_dpo, DPO_LOAD_BALANCE, DPO_PROTO_IP6, + load_balance_create (0, DPO_PROTO_IP6, fhc)); + + dpo_set (&sr_policy->ip6_dpo, DPO_LOAD_BALANCE, DPO_PROTO_IP6, + load_balance_create (0, DPO_PROTO_IP6, fhc)); + + /* Update FIB entry's to point to the LB DPO in the main FIB and hidden one */ + fib_table_entry_special_dpo_update (fib_table_id_find_fib_index + (FIB_PROTOCOL_IP6, + sr_policy->fib_table), &pfx, + FIB_SOURCE_SR, + FIB_ENTRY_FLAG_EXCLUSIVE, + &sr_policy->bsid_dpo); + + fib_table_entry_special_dpo_update (sm->fib_table_ip6, + &pfx, + FIB_SOURCE_SR, + FIB_ENTRY_FLAG_EXCLUSIVE, + &sr_policy->ip6_dpo); + + if (sr_policy->is_encap) + { + dpo_set (&sr_policy->ip4_dpo, DPO_LOAD_BALANCE, DPO_PROTO_IP4, + load_balance_create (0, DPO_PROTO_IP4, fhc)); + + fib_table_entry_special_dpo_update (sm->fib_table_ip4, + &pfx, + FIB_SOURCE_SR, + FIB_ENTRY_FLAG_EXCLUSIVE, + &sr_policy->ip4_dpo); + } + + } + + /* Create the LB path vector */ + //path_vector = vec_new(load_balance_path_t, vec_len(sr_policy->segments_lists)); + vec_foreach (sl_index, sr_policy->segments_lists) + { + segment_list = pool_elt_at_index (sm->sid_lists, *sl_index); + path.path_dpo = segment_list->bsid_dpo; + path.path_weight = segment_list->weight; + vec_add1 (b_path_vector, path); + path.path_dpo = segment_list->ip6_dpo; + vec_add1 (ip6_path_vector, path); + if (sr_policy->is_encap) + { + path.path_dpo = segment_list->ip4_dpo; + vec_add1 (ip4_path_vector, path); + } + } + + /* Update LB multipath */ + load_balance_multipath_update (&sr_policy->bsid_dpo, b_path_vector, + LOAD_BALANCE_FLAG_NONE); + load_balance_multipath_update (&sr_policy->ip6_dpo, ip6_path_vector, + LOAD_BALANCE_FLAG_NONE); + if (sr_policy->is_encap) + load_balance_multipath_update (&sr_policy->ip4_dpo, ip4_path_vector, + LOAD_BALANCE_FLAG_NONE); + + /* Cleanup */ + vec_free (b_path_vector); + vec_free (ip6_path_vector); + vec_free (ip4_path_vector); + +} + +/** + * @brief Updates the Replicate DPO after an SR Policy change + * + * @param sr_policy is the modified SR Policy (type spray) + */ +static inline void +update_replicate (ip6_sr_policy_t * sr_policy) +{ + u32 *sl_index; + ip6_sr_sl_t *segment_list; + ip6_sr_main_t *sm = &sr_main; + load_balance_path_t path; + load_balance_path_t *b_path_vector = 0; + load_balance_path_t *ip6_path_vector = 0; + load_balance_path_t *ip4_path_vector = 0; + + /* In case LB does not exist, create it */ + if (!dpo_id_is_valid (&sr_policy->bsid_dpo)) + { + dpo_set (&sr_policy->bsid_dpo, DPO_REPLICATE, + DPO_PROTO_IP6, replicate_create (0, DPO_PROTO_IP6)); + + dpo_set (&sr_policy->ip6_dpo, DPO_REPLICATE, + DPO_PROTO_IP6, replicate_create (0, DPO_PROTO_IP6)); + + /* Update FIB entry's DPO to point to SR without LB */ + fib_prefix_t pfx = { + .fp_proto = FIB_PROTOCOL_IP6, + .fp_len = 128, + .fp_addr = { + .ip6 = sr_policy->bsid, + } + }; + fib_table_entry_special_dpo_update (fib_table_id_find_fib_index + (FIB_PROTOCOL_IP6, + sr_policy->fib_table), &pfx, + FIB_SOURCE_SR, + FIB_ENTRY_FLAG_EXCLUSIVE, + &sr_policy->bsid_dpo); + + fib_table_entry_special_dpo_update (sm->fib_table_ip6, + &pfx, + FIB_SOURCE_SR, + FIB_ENTRY_FLAG_EXCLUSIVE, + &sr_policy->ip6_dpo); + + if (sr_policy->is_encap) + { + dpo_set (&sr_policy->ip4_dpo, DPO_REPLICATE, DPO_PROTO_IP4, + replicate_create (0, DPO_PROTO_IP4)); + + fib_table_entry_special_dpo_update (sm->fib_table_ip4, + &pfx, + FIB_SOURCE_SR, + FIB_ENTRY_FLAG_EXCLUSIVE, + &sr_policy->ip4_dpo); + } + + } + + /* Create the replicate path vector */ + path.path_weight = 1; + vec_foreach (sl_index, sr_policy->segments_lists) + { + segment_list = pool_elt_at_index (sm->sid_lists, *sl_index); + path.path_dpo = segment_list->bsid_dpo; + vec_add1 (b_path_vector, path); + path.path_dpo = segment_list->ip6_dpo; + vec_add1 (ip6_path_vector, path); + if (sr_policy->is_encap) + { + path.path_dpo = segment_list->ip4_dpo; + vec_add1 (ip4_path_vector, path); + } + } + + /* Update replicate multipath */ + replicate_multipath_update (&sr_policy->bsid_dpo, b_path_vector); + replicate_multipath_update (&sr_policy->ip6_dpo, ip6_path_vector); + if (sr_policy->is_encap) + replicate_multipath_update (&sr_policy->ip4_dpo, ip4_path_vector); + + /* Cleanup */ + vec_free (b_path_vector); + vec_free (ip6_path_vector); + vec_free (ip4_path_vector); +} + +/******************************* SR rewrite API *******************************/ +/* Three functions for handling sr policies: + * -> sr_policy_add + * -> sr_policy_del + * -> sr_policy_mod + * All of them are API. CLI function on sr_policy_command_fn */ + +/** + * @brief Create a new SR policy + * + * @param bsid is the bindingSID of the SR Policy + * @param segments is a vector of IPv6 address composing the segment list + * @param weight is the weight of the sid list. optional. + * @param behavior is the behavior of the SR policy. (default//spray) + * @param fib_table is the VRF where to install the FIB entry for the BSID + * @param is_encap (bool) whether SR policy should behave as Encap/SRH Insertion + * + * @return 0 if correct, else error + */ +int +sr_policy_add (ip6_address_t * bsid, ip6_address_t * segments, + u32 weight, u8 behavior, u32 fib_table, u8 is_encap) +{ + ip6_sr_main_t *sm = &sr_main; + ip6_sr_policy_t *sr_policy = 0; + ip6_address_t *key_copy; + uword *p; + + /* Search for existing keys (BSID) */ + p = hash_get_mem (sm->sr_policy_index_by_key, bsid); + if (p) + { + /* Add SR policy that already exists; complain */ + return -12; + } + + /* Search collision in FIB entries */ + /* Explanation: It might be possible that some other entity has already + * created a route for the BSID. This in theory is impossible, but in + * practise we could see it. Assert it and scream if needed */ + fib_prefix_t pfx = { + .fp_proto = FIB_PROTOCOL_IP6, + .fp_len = 128, + .fp_addr = { + .ip6 = *bsid, + } + }; + + /* Lookup the FIB index associated to the table selected */ + u32 fib_index = fib_table_id_find_fib_index (FIB_PROTOCOL_IP6, + (fib_table != + (u32) ~ 0 ? fib_table : 0)); + if (fib_index == ~0) + return -13; + + /* Lookup whether there exists an entry for the BSID */ + fib_node_index_t fei = fib_table_lookup_exact_match (fib_index, &pfx); + if (FIB_NODE_INDEX_INVALID != fei) + return -12; //There is an entry for such lookup + + /* Add an SR policy object */ + pool_get (sm->sr_policies, sr_policy); + memset (sr_policy, 0, sizeof (*sr_policy)); + clib_memcpy (&sr_policy->bsid, bsid, sizeof (ip6_address_t)); + sr_policy->type = behavior; + sr_policy->fib_table = (fib_table != (u32) ~ 0 ? fib_table : 0); //Is default FIB 0 ? + sr_policy->is_encap = is_encap; + + /* Copy the key */ + key_copy = vec_new (ip6_address_t, 1); + clib_memcpy (key_copy, bsid, sizeof (ip6_address_t)); + hash_set_mem (sm->sr_policy_index_by_key, key_copy, + sr_policy - sm->sr_policies); + + /* Create a segment list and add the index to the SR policy */ + create_sl (sr_policy, segments, weight, is_encap); + + /* If FIB doesnt exist, create them */ + if (sm->fib_table_ip6 == (u32) ~ 0) + { + sm->fib_table_ip6 = fib_table_create_and_lock (FIB_PROTOCOL_IP6, + "SRv6 steering of IP6 prefixes through BSIDs"); + sm->fib_table_ip4 = fib_table_create_and_lock (FIB_PROTOCOL_IP6, + "SRv6 steering of IP4 prefixes through BSIDs"); + fib_table_flush (sm->fib_table_ip6, FIB_PROTOCOL_IP6, + FIB_SOURCE_SPECIAL); + fib_table_flush (sm->fib_table_ip4, FIB_PROTOCOL_IP6, + FIB_SOURCE_SPECIAL); + } + + /* Create IPv6 FIB for the BindingSID attached to the DPO of the only SL */ + if (sr_policy->type == SR_POLICY_TYPE_DEFAULT) + update_lb (sr_policy); + else if (sr_policy->type == SR_POLICY_TYPE_SPRAY) + update_replicate (sr_policy); + return 0; +} + +/** + * @brief Delete a SR policy + * + * @param bsid is the bindingSID of the SR Policy + * @param index is the index of the SR policy + * + * @return 0 if correct, else error + */ +int +sr_policy_del (ip6_address_t * bsid, u32 index) +{ + ip6_sr_main_t *sm = &sr_main; + ip6_sr_policy_t *sr_policy = 0; + ip6_sr_sl_t *segment_list; + ip6_address_t *key_copy; + u32 *sl_index; + uword *p; + + hash_pair_t *hp; + if (bsid) + { + p = hash_get_mem (sm->sr_policy_index_by_key, bsid); + if (p) + sr_policy = pool_elt_at_index (sm->sr_policies, p[0]); + else + return -1; + } + else + { + sr_policy = pool_elt_at_index (sm->sr_policies, index); + if (!sr_policy) + return -1; + } + + /* Remove BindingSID FIB entry */ + fib_prefix_t pfx = { + .fp_proto = FIB_PROTOCOL_IP6, + .fp_len = 128, + .fp_addr = { + .ip6 = sr_policy->bsid, + } + , + }; + + fib_table_entry_special_remove (fib_table_id_find_fib_index + (FIB_PROTOCOL_IP6, sr_policy->fib_table), + &pfx, FIB_SOURCE_SR); + + fib_table_entry_special_remove (sm->fib_table_ip6, &pfx, FIB_SOURCE_SR); + + if (sr_policy->is_encap) + fib_table_entry_special_remove (sm->fib_table_ip4, &pfx, FIB_SOURCE_SR); + + if (dpo_id_is_valid (&sr_policy->bsid_dpo)) + { + dpo_reset (&sr_policy->bsid_dpo); + dpo_reset (&sr_policy->ip4_dpo); + dpo_reset (&sr_policy->ip6_dpo); + } + + /* Clean SID Lists */ + vec_foreach (sl_index, sr_policy->segments_lists) + { + segment_list = pool_elt_at_index (sm->sid_lists, *sl_index); + vec_free (segment_list->segments); + vec_free (segment_list->rewrite); + vec_free (segment_list->rewrite_bsid); + pool_put_index (sm->sid_lists, *sl_index); + } + + /* Remove SR policy entry */ + hp = hash_get_pair (sm->sr_policy_index_by_key, &sr_policy->bsid); + key_copy = (void *) (hp->key); + hash_unset_mem (sm->sr_policy_index_by_key, &sr_policy->bsid); + vec_free (key_copy); + pool_put (sm->sr_policies, sr_policy); + + /* If FIB empty unlock it */ + if (!pool_elts (sm->sr_policies)) + { + fib_table_unlock (sm->fib_table_ip6, FIB_PROTOCOL_IP6); + fib_table_unlock (sm->fib_table_ip4, FIB_PROTOCOL_IP6); + sm->fib_table_ip6 = (u32) ~ 0; + sm->fib_table_ip4 = (u32) ~ 0; + } + + return 0; +} + +/** + * @brief Modify an existing SR policy + * + * The possible modifications are adding a new Segment List, modifying an + * existing Segment List (modify the weight only) and delete a given + * Segment List from the SR Policy. + * + * @param bsid is the bindingSID of the SR Policy + * @param index is the index of the SR policy + * @param fib_table is the VRF where to install the FIB entry for the BSID + * @param operation is the operation to perform (among the top ones) + * @param segments is a vector of IPv6 address composing the segment list + * @param sl_index is the index of the Segment List to modify/delete + * @param weight is the weight of the sid list. optional. + * @param is_encap Mode. Encapsulation or SRH insertion. + * + * @return 0 if correct, else error + */ +int +sr_policy_mod (ip6_address_t * bsid, u32 index, u32 fib_table, + u8 operation, ip6_address_t * segments, u32 sl_index, + u32 weight) +{ + ip6_sr_main_t *sm = &sr_main; + ip6_sr_policy_t *sr_policy = 0; + ip6_sr_sl_t *segment_list; + u32 *sl_index_iterate; + uword *p; + + if (bsid) + { + p = hash_get_mem (sm->sr_policy_index_by_key, bsid); + if (p) + sr_policy = pool_elt_at_index (sm->sr_policies, p[0]); + else + return -1; + } + else + { + sr_policy = pool_elt_at_index (sm->sr_policies, index); + if (!sr_policy) + return -1; + } + + if (operation == 1) /* Add SR List to an existing SR policy */ + { + /* Create the new SL */ + segment_list = + create_sl (sr_policy, segments, weight, sr_policy->is_encap); + + /* Create a new LB DPO */ + if (sr_policy->type == SR_POLICY_TYPE_DEFAULT) + update_lb (sr_policy); + else if (sr_policy->type == SR_POLICY_TYPE_SPRAY) + update_replicate (sr_policy); + } + else if (operation == 2) /* Delete SR List from an existing SR policy */ + { + /* Check that currently there are more than one SID list */ + if (vec_len (sr_policy->segments_lists) == 1) + return -21; + + /* Check that the SR list does exist and is assigned to the sr policy */ + vec_foreach (sl_index_iterate, sr_policy->segments_lists) + if (*sl_index_iterate == sl_index) + break; + + if (*sl_index_iterate != sl_index) + return -22; + + /* Remove the lucky SR list that is being kicked out */ + segment_list = pool_elt_at_index (sm->sid_lists, sl_index); + vec_free (segment_list->segments); + vec_free (segment_list->rewrite); + vec_free (segment_list->rewrite_bsid); + pool_put_index (sm->sid_lists, sl_index); + vec_del1 (sr_policy->segments_lists, + sl_index_iterate - sr_policy->segments_lists); + + /* Create a new LB DPO */ + if (sr_policy->type == SR_POLICY_TYPE_DEFAULT) + update_lb (sr_policy); + else if (sr_policy->type == SR_POLICY_TYPE_SPRAY) + update_replicate (sr_policy); + } + else if (operation == 3) /* Modify the weight of an existing SR List */ + { + /* Find the corresponding SL */ + vec_foreach (sl_index_iterate, sr_policy->segments_lists) + if (*sl_index_iterate == sl_index) + break; + + if (*sl_index_iterate != sl_index) + return -32; + + /* Change the weight */ + segment_list = pool_elt_at_index (sm->sid_lists, sl_index); + segment_list->weight = weight; + + /* Update LB */ + if (sr_policy->type == SR_POLICY_TYPE_DEFAULT) + update_lb (sr_policy); + } + else /* Incorrect op. */ + return -1; + + return 0; +} + +/** + * @brief CLI for 'sr policies' command family + */ +static clib_error_t * +sr_policy_command_fn (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + int rv = -1; + char is_del = 0, is_add = 0, is_mod = 0; + char policy_set = 0; + ip6_address_t bsid, next_address; + u32 sr_policy_index = (u32) ~ 0, sl_index = (u32) ~ 0; + u32 weight = (u32) ~ 0, fib_table = (u32) ~ 0; + ip6_address_t *segments = 0, *this_seg; + u8 operation = 0; + char is_encap = 1; + char is_spray = 0; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (!is_add && !is_mod && !is_del && unformat (input, "add")) + is_add = 1; + else if (!is_add && !is_mod && !is_del && unformat (input, "del")) + is_del = 1; + else if (!is_add && !is_mod && !is_del && unformat (input, "mod")) + is_mod = 1; + else if (!policy_set + && unformat (input, "bsid %U", unformat_ip6_address, &bsid)) + policy_set = 1; + else if (!is_add && !policy_set + && unformat (input, "index %d", &sr_policy_index)) + policy_set = 1; + else if (unformat (input, "weight %d", &weight)); + else + if (unformat (input, "next %U", unformat_ip6_address, &next_address)) + { + vec_add2 (segments, this_seg, 1); + clib_memcpy (this_seg->as_u8, next_address.as_u8, + sizeof (*this_seg)); + } + else if (unformat (input, "add sl")) + operation = 1; + else if (unformat (input, "del sl index %d", &sl_index)) + operation = 2; + else if (unformat (input, "mod sl index %d", &sl_index)) + operation = 3; + else if (fib_table == (u32) ~ 0 + && unformat (input, "fib-table %d", &fib_table)); + else if (unformat (input, "encap")) + is_encap = 1; + else if (unformat (input, "insert")) + is_encap = 0; + else if (unformat (input, "spray")) + is_spray = 1; + else + break; + } + + if (!is_add && !is_mod && !is_del) + return clib_error_return (0, "Incorrect CLI"); + + if (!policy_set) + return clib_error_return (0, "No SR policy BSID or index specified"); + + if (is_add) + { + if (vec_len (segments) == 0) + return clib_error_return (0, "No Segment List specified"); + rv = sr_policy_add (&bsid, segments, weight, + (is_spray ? SR_POLICY_TYPE_SPRAY : + SR_POLICY_TYPE_DEFAULT), fib_table, is_encap); + } + else if (is_del) + rv = sr_policy_del ((sr_policy_index != (u32) ~ 0 ? NULL : &bsid), + sr_policy_index); + else if (is_mod) + { + if (!operation) + return clib_error_return (0, "No SL modification specified"); + if (operation != 1 && sl_index == (u32) ~ 0) + return clib_error_return (0, "No Segment List index specified"); + if (operation == 1 && vec_len (segments) == 0) + return clib_error_return (0, "No Segment List specified"); + if (operation == 3 && weight == (u32) ~ 0) + return clib_error_return (0, "No new weight for the SL specified"); + rv = sr_policy_mod ((sr_policy_index != (u32) ~ 0 ? NULL : &bsid), + sr_policy_index, fib_table, operation, segments, + sl_index, weight); + } + + switch (rv) + { + case 0: + break; + case 1: + return 0; + case -12: + return clib_error_return (0, + "There is already a FIB entry for the BindingSID address.\n" + "The SR policy could not be created."); + case -13: + return clib_error_return (0, "The specified FIB table does not exist."); + case -21: + return clib_error_return (0, + "The selected SR policy only contains ONE segment list. " + "Please remove the SR policy instead"); + case -22: + return clib_error_return (0, + "Could not delete the segment list. " + "It is not associated with that SR policy."); + case -32: + return clib_error_return (0, + "Could not modify the segment list. " + "The given SL is not associated with such SR policy."); + default: + return clib_error_return (0, "BUG: sr policy returns %d", rv); + } + return 0; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (sr_policy_command, static) = { + .path = "sr policy", + .short_help = "sr policy [add||del||mod] [bsid 2001::1||index 5] " + "next A:: next B:: next C:: (weight 1) (fib-table 2) (encap|insert)", + .long_help = + "Manipulation of SR policies.\n" + "A Segment Routing policy may contain several SID lists. Each SID list has\n" + "an associated weight (default 1), which will result in wECMP (uECMP).\n" + "Segment Routing policies might be of type encapsulation or srh insertion\n" + "Each SR policy will be associated with a unique BindingSID.\n" + "A BindingSID is a locally allocated SegmentID. For every packet that arrives\n" + "with IPv6_DA:BSID such traffic will be steered into the SR policy.\n" + "The add command will create a SR policy with its first segment list (sl)\n" + "The mod command allows you to add, remove, or modify the existing segment lists\n" + "within an SR policy.\n" + "The del command allows you to delete a SR policy along with all its associated\n" + "SID lists.\n", + .function = sr_policy_command_fn, +}; +/* *INDENT-ON* */ + +/** + * @brief CLI to display onscreen all the SR policies + */ +static clib_error_t * +show_sr_policies_command_fn (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + ip6_sr_main_t *sm = &sr_main; + u32 *sl_index; + ip6_sr_sl_t *segment_list = 0; + ip6_sr_policy_t *sr_policy = 0; + ip6_sr_policy_t **vec_policies = 0; + ip6_address_t *addr; + u8 *s; + int i = 0; + + vlib_cli_output (vm, "SR policies:"); + + /* *INDENT-OFF* */ + pool_foreach (sr_policy, sm->sr_policies, + {vec_add1 (vec_policies, sr_policy); } ); + /* *INDENT-ON* */ + + vec_foreach_index (i, vec_policies) + { + sr_policy = vec_policies[i]; + vlib_cli_output (vm, "[%u].-\tBSID: %U", + (u32) (sr_policy - sm->sr_policies), + format_ip6_address, &sr_policy->bsid); + vlib_cli_output (vm, "\tBehavior: %s", + (sr_policy->is_encap ? "Encapsulation" : + "SRH insertion")); + vlib_cli_output (vm, "\tType: %s", + (sr_policy->type == + SR_POLICY_TYPE_DEFAULT ? "Default" : "Spray")); + vlib_cli_output (vm, "\tFIB table: %u", + (sr_policy->fib_table != + (u32) ~ 0 ? sr_policy->fib_table : 0)); + vlib_cli_output (vm, "\tSegment Lists:"); + vec_foreach (sl_index, sr_policy->segments_lists) + { + s = NULL; + s = format (s, "\t[%u].- ", *sl_index); + segment_list = pool_elt_at_index (sm->sid_lists, *sl_index); + s = format (s, "< "); + vec_foreach (addr, segment_list->segments) + { + s = format (s, "%U, ", format_ip6_address, addr); + } + s = format (s, "\b\b > "); + s = format (s, "weight: %u", segment_list->weight); + vlib_cli_output (vm, " %s", s); + } + vlib_cli_output (vm, "-----------"); + } + return 0; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (show_sr_policies_command, static) = { + .path = "show sr policies", + .short_help = "show sr policies", + .function = show_sr_policies_command_fn, +}; +/* *INDENT-ON* */ + +/*************************** SR rewrite graph node ****************************/ +/** + * @brief Trace for the SR Policy Rewrite graph node + */ +static u8 * +format_sr_policy_rewrite_trace (u8 * s, va_list * args) +{ + //TODO + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + sr_policy_rewrite_trace_t *t = va_arg (*args, sr_policy_rewrite_trace_t *); + + s = format + (s, "SR-policy-rewrite: src %U dst %U", + format_ip6_address, &t->src, format_ip6_address, &t->dst); + + return s; +} + +/** + * @brief IPv6 encapsulation processing as per RFC2473 + */ +static_always_inline void +encaps_processing_v6 (vlib_node_runtime_t * node, + vlib_buffer_t * b0, + ip6_header_t * ip0, ip6_header_t * ip0_encap) +{ + u32 new_l0; + + ip0_encap->hop_limit -= 1; + new_l0 = + ip0->payload_length + sizeof (ip6_header_t) + + clib_net_to_host_u16 (ip0_encap->payload_length); + ip0->payload_length = clib_host_to_net_u16 (new_l0); + ip0->ip_version_traffic_class_and_flow_label = + ip0_encap->ip_version_traffic_class_and_flow_label; +} + +/** + * @brief Graph node for applying a SR policy into an IPv6 packet. Encapsulation + */ +static uword +sr_policy_rewrite_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + ip6_sr_main_t *sm = &sr_main; + u32 n_left_from, next_index, *from, *to_next; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + int encap_pkts = 0, bsid_pkts = 0; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + /* Quad - Loop */ + while (n_left_from >= 8 && n_left_to_next >= 4) + { + u32 bi0, bi1, bi2, bi3; + vlib_buffer_t *b0, *b1, *b2, *b3; + u32 next0, next1, next2, next3; + next0 = next1 = next2 = next3 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP; + ip6_header_t *ip0, *ip1, *ip2, *ip3; + ip6_header_t *ip0_encap, *ip1_encap, *ip2_encap, *ip3_encap; + ip6_sr_sl_t *sl0, *sl1, *sl2, *sl3; + + /* Prefetch next iteration. */ + { + vlib_buffer_t *p4, *p5, *p6, *p7; + + p4 = vlib_get_buffer (vm, from[4]); + p5 = vlib_get_buffer (vm, from[5]); + p6 = vlib_get_buffer (vm, from[6]); + p7 = vlib_get_buffer (vm, from[7]); + + /* Prefetch the buffer header and packet for the N+2 loop iteration */ + vlib_prefetch_buffer_header (p4, LOAD); + vlib_prefetch_buffer_header (p5, LOAD); + vlib_prefetch_buffer_header (p6, LOAD); + vlib_prefetch_buffer_header (p7, LOAD); + + CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + } + + to_next[0] = bi0 = from[0]; + to_next[1] = bi1 = from[1]; + to_next[2] = bi2 = from[2]; + to_next[3] = bi3 = from[3]; + from += 4; + to_next += 4; + n_left_from -= 4; + n_left_to_next -= 4; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + b2 = vlib_get_buffer (vm, bi2); + b3 = vlib_get_buffer (vm, bi3); + + sl0 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b0)->ip.adj_index[VLIB_TX]); + sl1 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b1)->ip.adj_index[VLIB_TX]); + sl2 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b2)->ip.adj_index[VLIB_TX]); + sl3 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b3)->ip.adj_index[VLIB_TX]); + + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl0->rewrite) + b0->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl1->rewrite) + b1->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl2->rewrite) + b2->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl3->rewrite) + b3->current_data)); + + ip0_encap = vlib_buffer_get_current (b0); + ip1_encap = vlib_buffer_get_current (b1); + ip2_encap = vlib_buffer_get_current (b2); + ip3_encap = vlib_buffer_get_current (b3); + + clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy (((u8 *) ip1_encap) - vec_len (sl1->rewrite), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy (((u8 *) ip2_encap) - vec_len (sl2->rewrite), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy (((u8 *) ip3_encap) - vec_len (sl3->rewrite), + sl3->rewrite, vec_len (sl3->rewrite)); + + vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); + vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); + vlib_buffer_advance (b2, -(word) vec_len (sl2->rewrite)); + vlib_buffer_advance (b3, -(word) vec_len (sl3->rewrite)); + + ip0 = vlib_buffer_get_current (b0); + ip1 = vlib_buffer_get_current (b1); + ip2 = vlib_buffer_get_current (b2); + ip3 = vlib_buffer_get_current (b3); + + encaps_processing_v6 (node, b0, ip0, ip0_encap); + encaps_processing_v6 (node, b1, ip1, ip1_encap); + encaps_processing_v6 (node, b2, ip2, ip2_encap); + encaps_processing_v6 (node, b3, ip3, ip3_encap); + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE))) + { + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b1, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b2, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b3, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + } + + encap_pkts += 4; + vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, + n_left_to_next, bi0, bi1, bi2, bi3, + next0, next1, next2, next3); + } + + /* Single loop for potentially the last three packets */ + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + ip6_header_t *ip0 = 0, *ip0_encap = 0; + ip6_sr_sl_t *sl0; + u32 next0 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + b0 = vlib_get_buffer (vm, bi0); + + sl0 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b0)->ip.adj_index[VLIB_TX]); + + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl0->rewrite) + b0->current_data)); + + ip0_encap = vlib_buffer_get_current (b0); + + clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); + + ip0 = vlib_buffer_get_current (b0); + + encaps_processing_v6 (node, b0, ip0, ip0_encap); + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) && + PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + encap_pkts++; + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + /* Update counters */ + vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index, + SR_POLICY_REWRITE_ERROR_COUNTER_TOTAL, + encap_pkts); + vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index, + SR_POLICY_REWRITE_ERROR_COUNTER_BSID, + bsid_pkts); + + return from_frame->n_vectors; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (sr_policy_rewrite_encaps_node) = { + .function = sr_policy_rewrite_encaps, + .name = "sr-pl-rewrite-encaps", + .vector_size = sizeof (u32), + .format_trace = format_sr_policy_rewrite_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = SR_POLICY_REWRITE_N_ERROR, + .error_strings = sr_policy_rewrite_error_strings, + .n_next_nodes = SR_POLICY_REWRITE_N_NEXT, + .next_nodes = { +#define _(s,n) [SR_POLICY_REWRITE_NEXT_##s] = n, + foreach_sr_policy_rewrite_next +#undef _ + }, +}; +/* *INDENT-ON* */ + +/** + * @brief IPv4 encapsulation processing as per RFC2473 + */ +static_always_inline void +encaps_processing_v4 (vlib_node_runtime_t * node, + vlib_buffer_t * b0, + ip6_header_t * ip0, ip4_header_t * ip0_encap) +{ + u32 new_l0; + ip6_sr_header_t *sr0; + + u32 checksum0; + + /* Inner IPv4: Decrement TTL & update checksum */ + ip0_encap->ttl -= 1; + checksum0 = ip0_encap->checksum + clib_host_to_net_u16 (0x0100); + checksum0 += checksum0 >= 0xffff; + ip0_encap->checksum = checksum0; + + /* Outer IPv6: Update length, FL, proto */ + new_l0 = ip0->payload_length + clib_net_to_host_u16 (ip0_encap->length); + ip0->payload_length = clib_host_to_net_u16 (new_l0); + ip0->ip_version_traffic_class_and_flow_label = + clib_host_to_net_u32 (0 | ((6 & 0xF) << 28) | + ((ip0_encap->tos & 0xFF) << 20)); + sr0 = (void *) (ip0 + 1); + sr0->protocol = IP_PROTOCOL_IP_IN_IP; +} + +/** + * @brief Graph node for applying a SR policy into an IPv4 packet. Encapsulation + */ +static uword +sr_policy_rewrite_encaps_v4 (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + ip6_sr_main_t *sm = &sr_main; + u32 n_left_from, next_index, *from, *to_next; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + int encap_pkts = 0, bsid_pkts = 0; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + /* Quad - Loop */ + while (n_left_from >= 8 && n_left_to_next >= 4) + { + u32 bi0, bi1, bi2, bi3; + vlib_buffer_t *b0, *b1, *b2, *b3; + u32 next0, next1, next2, next3; + next0 = next1 = next2 = next3 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP; + ip6_header_t *ip0, *ip1, *ip2, *ip3; + ip4_header_t *ip0_encap, *ip1_encap, *ip2_encap, *ip3_encap; + ip6_sr_sl_t *sl0, *sl1, *sl2, *sl3; + + /* Prefetch next iteration. */ + { + vlib_buffer_t *p4, *p5, *p6, *p7; + + p4 = vlib_get_buffer (vm, from[4]); + p5 = vlib_get_buffer (vm, from[5]); + p6 = vlib_get_buffer (vm, from[6]); + p7 = vlib_get_buffer (vm, from[7]); + + /* Prefetch the buffer header and packet for the N+2 loop iteration */ + vlib_prefetch_buffer_header (p4, LOAD); + vlib_prefetch_buffer_header (p5, LOAD); + vlib_prefetch_buffer_header (p6, LOAD); + vlib_prefetch_buffer_header (p7, LOAD); + + CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + } + + to_next[0] = bi0 = from[0]; + to_next[1] = bi1 = from[1]; + to_next[2] = bi2 = from[2]; + to_next[3] = bi3 = from[3]; + from += 4; + to_next += 4; + n_left_from -= 4; + n_left_to_next -= 4; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + b2 = vlib_get_buffer (vm, bi2); + b3 = vlib_get_buffer (vm, bi3); + + sl0 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b0)->ip.adj_index[VLIB_TX]); + sl1 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b1)->ip.adj_index[VLIB_TX]); + sl2 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b2)->ip.adj_index[VLIB_TX]); + sl3 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b3)->ip.adj_index[VLIB_TX]); + + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl0->rewrite) + b0->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl1->rewrite) + b1->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl2->rewrite) + b2->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl3->rewrite) + b3->current_data)); + + ip0_encap = vlib_buffer_get_current (b0); + ip1_encap = vlib_buffer_get_current (b1); + ip2_encap = vlib_buffer_get_current (b2); + ip3_encap = vlib_buffer_get_current (b3); + + clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy (((u8 *) ip1_encap) - vec_len (sl1->rewrite), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy (((u8 *) ip2_encap) - vec_len (sl2->rewrite), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy (((u8 *) ip3_encap) - vec_len (sl3->rewrite), + sl3->rewrite, vec_len (sl3->rewrite)); + + vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); + vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); + vlib_buffer_advance (b2, -(word) vec_len (sl2->rewrite)); + vlib_buffer_advance (b3, -(word) vec_len (sl3->rewrite)); + + ip0 = vlib_buffer_get_current (b0); + ip1 = vlib_buffer_get_current (b1); + ip2 = vlib_buffer_get_current (b2); + ip3 = vlib_buffer_get_current (b3); + + encaps_processing_v4 (node, b0, ip0, ip0_encap); + encaps_processing_v4 (node, b1, ip1, ip1_encap); + encaps_processing_v4 (node, b2, ip2, ip2_encap); + encaps_processing_v4 (node, b3, ip3, ip3_encap); + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE))) + { + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b1, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b2, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b3, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + } + + encap_pkts += 4; + vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, + n_left_to_next, bi0, bi1, bi2, bi3, + next0, next1, next2, next3); + } + + /* Single loop for potentially the last three packets */ + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + ip6_header_t *ip0 = 0; + ip4_header_t *ip0_encap = 0; + ip6_sr_sl_t *sl0; + u32 next0 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + b0 = vlib_get_buffer (vm, bi0); + + sl0 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b0)->ip.adj_index[VLIB_TX]); + + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl0->rewrite) + b0->current_data)); + + ip0_encap = vlib_buffer_get_current (b0); + + clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); + + ip0 = vlib_buffer_get_current (b0); + + encaps_processing_v4 (node, b0, ip0, ip0_encap); + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) && + PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + encap_pkts++; + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + /* Update counters */ + vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index, + SR_POLICY_REWRITE_ERROR_COUNTER_TOTAL, + encap_pkts); + vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index, + SR_POLICY_REWRITE_ERROR_COUNTER_BSID, + bsid_pkts); + + return from_frame->n_vectors; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (sr_policy_rewrite_encaps_v4_node) = { + .function = sr_policy_rewrite_encaps_v4, + .name = "sr-pl-rewrite-encaps-v4", + .vector_size = sizeof (u32), + .format_trace = format_sr_policy_rewrite_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = SR_POLICY_REWRITE_N_ERROR, + .error_strings = sr_policy_rewrite_error_strings, + .n_next_nodes = SR_POLICY_REWRITE_N_NEXT, + .next_nodes = { +#define _(s,n) [SR_POLICY_REWRITE_NEXT_##s] = n, + foreach_sr_policy_rewrite_next +#undef _ + }, +}; +/* *INDENT-ON* */ + +always_inline u32 +ip_flow_hash (void *data) +{ + ip4_header_t *iph = (ip4_header_t *) data; + + if ((iph->ip_version_and_header_length & 0xF0) == 0x40) + return ip4_compute_flow_hash (iph, IP_FLOW_HASH_DEFAULT); + else + return ip6_compute_flow_hash ((ip6_header_t *) iph, IP_FLOW_HASH_DEFAULT); +} + +always_inline u64 +mac_to_u64 (u8 * m) +{ + return (*((u64 *) m) & 0xffffffffffff); +} + +always_inline u32 +l2_flow_hash (vlib_buffer_t * b0) +{ + ethernet_header_t *eh; + u64 a, b, c; + uword is_ip, eh_size; + u16 eh_type; + + eh = vlib_buffer_get_current (b0); + eh_type = clib_net_to_host_u16 (eh->type); + eh_size = ethernet_buffer_header_size (b0); + + is_ip = (eh_type == ETHERNET_TYPE_IP4 || eh_type == ETHERNET_TYPE_IP6); + + /* since we have 2 cache lines, use them */ + if (is_ip) + a = ip_flow_hash ((u8 *) vlib_buffer_get_current (b0) + eh_size); + else + a = eh->type; + + b = mac_to_u64 ((u8 *) eh->dst_address); + c = mac_to_u64 ((u8 *) eh->src_address); + hash_mix64 (a, b, c); + + return (u32) c; +} + +/** + * @brief Graph node for applying a SR policy into a L2 frame + */ +static uword +sr_policy_rewrite_encaps_l2 (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + ip6_sr_main_t *sm = &sr_main; + u32 n_left_from, next_index, *from, *to_next; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + int encap_pkts = 0, bsid_pkts = 0; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + /* Quad - Loop */ + while (n_left_from >= 8 && n_left_to_next >= 4) + { + u32 bi0, bi1, bi2, bi3; + vlib_buffer_t *b0, *b1, *b2, *b3; + u32 next0, next1, next2, next3; + next0 = next1 = next2 = next3 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP; + ethernet_header_t *en0, *en1, *en2, *en3; + ip6_header_t *ip0, *ip1, *ip2, *ip3; + ip6_sr_header_t *sr0, *sr1, *sr2, *sr3; + ip6_sr_policy_t *sp0, *sp1, *sp2, *sp3; + ip6_sr_sl_t *sl0, *sl1, *sl2, *sl3; + + /* Prefetch next iteration. */ + { + vlib_buffer_t *p4, *p5, *p6, *p7; + + p4 = vlib_get_buffer (vm, from[4]); + p5 = vlib_get_buffer (vm, from[5]); + p6 = vlib_get_buffer (vm, from[6]); + p7 = vlib_get_buffer (vm, from[7]); + + /* Prefetch the buffer header and packet for the N+2 loop iteration */ + vlib_prefetch_buffer_header (p4, LOAD); + vlib_prefetch_buffer_header (p5, LOAD); + vlib_prefetch_buffer_header (p6, LOAD); + vlib_prefetch_buffer_header (p7, LOAD); + + CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + } + + to_next[0] = bi0 = from[0]; + to_next[1] = bi1 = from[1]; + to_next[2] = bi2 = from[2]; + to_next[3] = bi3 = from[3]; + from += 4; + to_next += 4; + n_left_from -= 4; + n_left_to_next -= 4; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + b2 = vlib_get_buffer (vm, bi2); + b3 = vlib_get_buffer (vm, bi3); + + sp0 = pool_elt_at_index (sm->sr_policies, + sm->sw_iface_sr_policies[vnet_buffer + (b0)->sw_if_index + [VLIB_RX]]); + + sp1 = pool_elt_at_index (sm->sr_policies, + sm->sw_iface_sr_policies[vnet_buffer + (b1)->sw_if_index + [VLIB_RX]]); + + sp2 = pool_elt_at_index (sm->sr_policies, + sm->sw_iface_sr_policies[vnet_buffer + (b2)->sw_if_index + [VLIB_RX]]); + + sp3 = pool_elt_at_index (sm->sr_policies, + sm->sw_iface_sr_policies[vnet_buffer + (b3)->sw_if_index + [VLIB_RX]]); + + if (vec_len (sp0->segments_lists) == 1) + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = sp0->segments_lists[0]; + else + { + vnet_buffer (b0)->ip.flow_hash = l2_flow_hash (b0); + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = + sp0->segments_lists[(vnet_buffer (b0)->ip.flow_hash & + (vec_len (sp0->segments_lists) - 1))]; + } + + if (vec_len (sp1->segments_lists) == 1) + vnet_buffer (b1)->ip.adj_index[VLIB_TX] = sp1->segments_lists[1]; + else + { + vnet_buffer (b1)->ip.flow_hash = l2_flow_hash (b1); + vnet_buffer (b1)->ip.adj_index[VLIB_TX] = + sp1->segments_lists[(vnet_buffer (b1)->ip.flow_hash & + (vec_len (sp1->segments_lists) - 1))]; + } + + if (vec_len (sp2->segments_lists) == 1) + vnet_buffer (b2)->ip.adj_index[VLIB_TX] = sp2->segments_lists[2]; + else + { + vnet_buffer (b2)->ip.flow_hash = l2_flow_hash (b2); + vnet_buffer (b2)->ip.adj_index[VLIB_TX] = + sp2->segments_lists[(vnet_buffer (b2)->ip.flow_hash & + (vec_len (sp2->segments_lists) - 1))]; + } + + if (vec_len (sp3->segments_lists) == 1) + vnet_buffer (b3)->ip.adj_index[VLIB_TX] = sp3->segments_lists[3]; + else + { + vnet_buffer (b3)->ip.flow_hash = l2_flow_hash (b3); + vnet_buffer (b3)->ip.adj_index[VLIB_TX] = + sp3->segments_lists[(vnet_buffer (b3)->ip.flow_hash & + (vec_len (sp3->segments_lists) - 1))]; + } + + sl0 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b0)->ip.adj_index[VLIB_TX]); + sl1 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b1)->ip.adj_index[VLIB_TX]); + sl2 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b2)->ip.adj_index[VLIB_TX]); + sl3 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b3)->ip.adj_index[VLIB_TX]); + + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl0->rewrite) + b0->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl1->rewrite) + b1->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl2->rewrite) + b2->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl3->rewrite) + b3->current_data)); + + en0 = vlib_buffer_get_current (b0); + en1 = vlib_buffer_get_current (b1); + en2 = vlib_buffer_get_current (b2); + en3 = vlib_buffer_get_current (b3); + + clib_memcpy (((u8 *) en0) - vec_len (sl0->rewrite), sl0->rewrite, + vec_len (sl0->rewrite)); + clib_memcpy (((u8 *) en1) - vec_len (sl1->rewrite), sl1->rewrite, + vec_len (sl1->rewrite)); + clib_memcpy (((u8 *) en2) - vec_len (sl2->rewrite), sl2->rewrite, + vec_len (sl2->rewrite)); + clib_memcpy (((u8 *) en3) - vec_len (sl3->rewrite), sl3->rewrite, + vec_len (sl3->rewrite)); + + vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); + vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); + vlib_buffer_advance (b2, -(word) vec_len (sl2->rewrite)); + vlib_buffer_advance (b3, -(word) vec_len (sl3->rewrite)); + + ip0 = vlib_buffer_get_current (b0); + ip1 = vlib_buffer_get_current (b1); + ip2 = vlib_buffer_get_current (b2); + ip3 = vlib_buffer_get_current (b3); + + ip0->payload_length = + clib_host_to_net_u16 (b0->current_length - sizeof (ip6_header_t)); + ip1->payload_length = + clib_host_to_net_u16 (b1->current_length - sizeof (ip6_header_t)); + ip2->payload_length = + clib_host_to_net_u16 (b2->current_length - sizeof (ip6_header_t)); + ip3->payload_length = + clib_host_to_net_u16 (b3->current_length - sizeof (ip6_header_t)); + + sr0 = (void *) (ip0 + 1); + sr1 = (void *) (ip1 + 1); + sr2 = (void *) (ip2 + 1); + sr3 = (void *) (ip3 + 1); + + sr0->protocol = sr1->protocol = sr2->protocol = sr3->protocol = + IP_PROTOCOL_IP6_NONXT; + + /* Which Traffic class and flow label do I set ? */ + //ip0->ip_version_traffic_class_and_flow_label = clib_host_to_net_u32(0|((6&0xF)<<28)|((ip0_encap->tos&0xFF)<<20)); + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE))) + { + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b1, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b2, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b3, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + } + + encap_pkts += 4; + vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, + n_left_to_next, bi0, bi1, bi2, bi3, + next0, next1, next2, next3); + } + + /* Single loop for potentially the last three packets */ + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + ip6_header_t *ip0 = 0; + ip6_sr_header_t *sr0; + ethernet_header_t *en0; + ip6_sr_policy_t *sp0; + ip6_sr_sl_t *sl0; + u32 next0 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + b0 = vlib_get_buffer (vm, bi0); + + /* Find the SR policy */ + sp0 = pool_elt_at_index (sm->sr_policies, + sm->sw_iface_sr_policies[vnet_buffer + (b0)->sw_if_index + [VLIB_RX]]); + + /* In case there is more than one SL, LB among them */ + if (vec_len (sp0->segments_lists) == 1) + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = sp0->segments_lists[0]; + else + { + vnet_buffer (b0)->ip.flow_hash = l2_flow_hash (b0); + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = + sp0->segments_lists[(vnet_buffer (b0)->ip.flow_hash & + (vec_len (sp0->segments_lists) - 1))]; + } + sl0 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b0)->ip.adj_index[VLIB_TX]); + + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl0->rewrite) + b0->current_data)); + + en0 = vlib_buffer_get_current (b0); + + clib_memcpy (((u8 *) en0) - vec_len (sl0->rewrite), sl0->rewrite, + vec_len (sl0->rewrite)); + + vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); + + ip0 = vlib_buffer_get_current (b0); + + ip0->payload_length = + clib_host_to_net_u16 (b0->current_length - sizeof (ip6_header_t)); + + sr0 = (void *) (ip0 + 1); + sr0->protocol = IP_PROTOCOL_IP6_NONXT; + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) && + PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + encap_pkts++; + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + /* Update counters */ + vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index, + SR_POLICY_REWRITE_ERROR_COUNTER_TOTAL, + encap_pkts); + vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index, + SR_POLICY_REWRITE_ERROR_COUNTER_BSID, + bsid_pkts); + + return from_frame->n_vectors; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (sr_policy_rewrite_encaps_l2_node) = { + .function = sr_policy_rewrite_encaps_l2, + .name = "sr-pl-rewrite-encaps-l2", + .vector_size = sizeof (u32), + .format_trace = format_sr_policy_rewrite_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = SR_POLICY_REWRITE_N_ERROR, + .error_strings = sr_policy_rewrite_error_strings, + .n_next_nodes = SR_POLICY_REWRITE_N_NEXT, + .next_nodes = { +#define _(s,n) [SR_POLICY_REWRITE_NEXT_##s] = n, + foreach_sr_policy_rewrite_next +#undef _ + }, +}; +/* *INDENT-ON* */ + +/** + * @brief Graph node for applying a SR policy into a packet. SRH insertion. + */ +static uword +sr_policy_rewrite_insert (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + ip6_sr_main_t *sm = &sr_main; + u32 n_left_from, next_index, *from, *to_next; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + int insert_pkts = 0, bsid_pkts = 0; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + /* Quad - Loop */ + while (n_left_from >= 8 && n_left_to_next >= 4) + { + u32 bi0, bi1, bi2, bi3; + vlib_buffer_t *b0, *b1, *b2, *b3; + u32 next0, next1, next2, next3; + next0 = next1 = next2 = next3 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP; + ip6_header_t *ip0, *ip1, *ip2, *ip3; + ip6_sr_header_t *sr0, *sr1, *sr2, *sr3; + ip6_sr_sl_t *sl0, *sl1, *sl2, *sl3; + u16 new_l0, new_l1, new_l2, new_l3; + + /* Prefetch next iteration. */ + { + vlib_buffer_t *p4, *p5, *p6, *p7; + + p4 = vlib_get_buffer (vm, from[4]); + p5 = vlib_get_buffer (vm, from[5]); + p6 = vlib_get_buffer (vm, from[6]); + p7 = vlib_get_buffer (vm, from[7]); + + /* Prefetch the buffer header and packet for the N+2 loop iteration */ + vlib_prefetch_buffer_header (p4, LOAD); + vlib_prefetch_buffer_header (p5, LOAD); + vlib_prefetch_buffer_header (p6, LOAD); + vlib_prefetch_buffer_header (p7, LOAD); + + CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + } + + to_next[0] = bi0 = from[0]; + to_next[1] = bi1 = from[1]; + to_next[2] = bi2 = from[2]; + to_next[3] = bi3 = from[3]; + from += 4; + to_next += 4; + n_left_from -= 4; + n_left_to_next -= 4; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + b2 = vlib_get_buffer (vm, bi2); + b3 = vlib_get_buffer (vm, bi3); + + sl0 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b0)->ip.adj_index[VLIB_TX]); + sl1 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b1)->ip.adj_index[VLIB_TX]); + sl2 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b2)->ip.adj_index[VLIB_TX]); + sl3 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b3)->ip.adj_index[VLIB_TX]); + + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl0->rewrite) + b0->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl1->rewrite) + b1->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl2->rewrite) + b2->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl3->rewrite) + b3->current_data)); + + ip0 = vlib_buffer_get_current (b0); + ip1 = vlib_buffer_get_current (b1); + ip2 = vlib_buffer_get_current (b2); + ip3 = vlib_buffer_get_current (b3); + + if (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) + sr0 = + (ip6_sr_header_t *) (((void *) (ip0 + 1)) + + ip6_ext_header_len (ip0 + 1)); + else + sr0 = (ip6_sr_header_t *) (ip0 + 1); + + if (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) + sr1 = + (ip6_sr_header_t *) (((void *) (ip1 + 1)) + + ip6_ext_header_len (ip1 + 1)); + else + sr1 = (ip6_sr_header_t *) (ip1 + 1); + + if (ip2->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) + sr2 = + (ip6_sr_header_t *) (((void *) (ip2 + 1)) + + ip6_ext_header_len (ip2 + 1)); + else + sr2 = (ip6_sr_header_t *) (ip2 + 1); + + if (ip3->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) + sr3 = + (ip6_sr_header_t *) (((void *) (ip3 + 1)) + + ip6_ext_header_len (ip3 + 1)); + else + sr3 = (ip6_sr_header_t *) (ip3 + 1); + + clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite), (u8 *) ip0, + (void *) sr0 - (void *) ip0); + clib_memcpy ((u8 *) ip1 - vec_len (sl1->rewrite), (u8 *) ip1, + (void *) sr1 - (void *) ip1); + clib_memcpy ((u8 *) ip2 - vec_len (sl2->rewrite), (u8 *) ip2, + (void *) sr2 - (void *) ip2); + clib_memcpy ((u8 *) ip3 - vec_len (sl3->rewrite), (u8 *) ip3, + (void *) sr3 - (void *) ip3); + + clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite)), sl0->rewrite, + vec_len (sl0->rewrite)); + clib_memcpy (((u8 *) sr1 - vec_len (sl1->rewrite)), sl1->rewrite, + vec_len (sl1->rewrite)); + clib_memcpy (((u8 *) sr2 - vec_len (sl2->rewrite)), sl2->rewrite, + vec_len (sl2->rewrite)); + clib_memcpy (((u8 *) sr3 - vec_len (sl3->rewrite)), sl3->rewrite, + vec_len (sl3->rewrite)); + + vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); + vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); + vlib_buffer_advance (b2, -(word) vec_len (sl2->rewrite)); + vlib_buffer_advance (b3, -(word) vec_len (sl3->rewrite)); + + ip0 = ((void *) ip0) - vec_len (sl0->rewrite); + ip1 = ((void *) ip1) - vec_len (sl1->rewrite); + ip2 = ((void *) ip2) - vec_len (sl2->rewrite); + ip3 = ((void *) ip3) - vec_len (sl3->rewrite); + + ip0->hop_limit -= 1; + ip1->hop_limit -= 1; + ip2->hop_limit -= 1; + ip3->hop_limit -= 1; + + new_l0 = + clib_net_to_host_u16 (ip0->payload_length) + + vec_len (sl0->rewrite); + new_l1 = + clib_net_to_host_u16 (ip1->payload_length) + + vec_len (sl1->rewrite); + new_l2 = + clib_net_to_host_u16 (ip2->payload_length) + + vec_len (sl2->rewrite); + new_l3 = + clib_net_to_host_u16 (ip3->payload_length) + + vec_len (sl3->rewrite); + + ip0->payload_length = clib_host_to_net_u16 (new_l0); + ip1->payload_length = clib_host_to_net_u16 (new_l1); + ip2->payload_length = clib_host_to_net_u16 (new_l2); + ip3->payload_length = clib_host_to_net_u16 (new_l3); + + sr0 = ((void *) sr0) - vec_len (sl0->rewrite); + sr1 = ((void *) sr1) - vec_len (sl1->rewrite); + sr2 = ((void *) sr2) - vec_len (sl2->rewrite); + sr3 = ((void *) sr3) - vec_len (sl3->rewrite); + + sr0->segments->as_u64[0] = ip0->dst_address.as_u64[0]; + sr0->segments->as_u64[1] = ip0->dst_address.as_u64[1]; + sr1->segments->as_u64[0] = ip1->dst_address.as_u64[0]; + sr1->segments->as_u64[1] = ip1->dst_address.as_u64[1]; + sr2->segments->as_u64[0] = ip2->dst_address.as_u64[0]; + sr2->segments->as_u64[1] = ip2->dst_address.as_u64[1]; + sr3->segments->as_u64[0] = ip3->dst_address.as_u64[0]; + sr3->segments->as_u64[1] = ip3->dst_address.as_u64[1]; + + ip0->dst_address.as_u64[0] = + (sr0->segments + sr0->segments_left)->as_u64[0]; + ip0->dst_address.as_u64[1] = + (sr0->segments + sr0->segments_left)->as_u64[1]; + ip1->dst_address.as_u64[0] = + (sr1->segments + sr1->segments_left)->as_u64[0]; + ip1->dst_address.as_u64[1] = + (sr1->segments + sr1->segments_left)->as_u64[1]; + ip2->dst_address.as_u64[0] = + (sr2->segments + sr2->segments_left)->as_u64[0]; + ip2->dst_address.as_u64[1] = + (sr2->segments + sr2->segments_left)->as_u64[1]; + ip3->dst_address.as_u64[0] = + (sr3->segments + sr3->segments_left)->as_u64[0]; + ip3->dst_address.as_u64[1] = + (sr3->segments + sr3->segments_left)->as_u64[1]; + + ip6_ext_header_t *ip_ext; + if (ip0 + 1 == (void *) sr0) + { + sr0->protocol = ip0->protocol; + ip0->protocol = IP_PROTOCOL_IPV6_ROUTE; + } + else + { + ip_ext = (void *) (ip0 + 1); + sr0->protocol = ip_ext->next_hdr; + ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE; + } + + if (ip1 + 1 == (void *) sr1) + { + sr1->protocol = ip1->protocol; + ip1->protocol = IP_PROTOCOL_IPV6_ROUTE; + } + else + { + ip_ext = (void *) (ip2 + 1); + sr2->protocol = ip_ext->next_hdr; + ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE; + } + + if (ip2 + 1 == (void *) sr2) + { + sr2->protocol = ip2->protocol; + ip2->protocol = IP_PROTOCOL_IPV6_ROUTE; + } + else + { + ip_ext = (void *) (ip2 + 1); + sr2->protocol = ip_ext->next_hdr; + ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE; + } + + if (ip3 + 1 == (void *) sr3) + { + sr3->protocol = ip3->protocol; + ip3->protocol = IP_PROTOCOL_IPV6_ROUTE; + } + else + { + ip_ext = (void *) (ip3 + 1); + sr3->protocol = ip_ext->next_hdr; + ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE; + } + + insert_pkts += 4; + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE))) + { + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b1, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b2, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b3, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + } + + vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, + n_left_to_next, bi0, bi1, bi2, bi3, + next0, next1, next2, next3); + } + + /* Single loop for potentially the last three packets */ + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + ip6_header_t *ip0 = 0; + ip6_sr_header_t *sr0 = 0; + ip6_sr_sl_t *sl0; + u32 next0 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP; + u16 new_l0 = 0; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + sl0 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b0)->ip.adj_index[VLIB_TX]); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl0->rewrite) + b0->current_data)); + + ip0 = vlib_buffer_get_current (b0); + + if (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) + sr0 = + (ip6_sr_header_t *) (((void *) (ip0 + 1)) + + ip6_ext_header_len (ip0 + 1)); + else + sr0 = (ip6_sr_header_t *) (ip0 + 1); + + clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite), (u8 *) ip0, + (void *) sr0 - (void *) ip0); + clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite)), sl0->rewrite, + vec_len (sl0->rewrite)); + + vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); + + ip0 = ((void *) ip0) - vec_len (sl0->rewrite); + ip0->hop_limit -= 1; + new_l0 = + clib_net_to_host_u16 (ip0->payload_length) + + vec_len (sl0->rewrite); + ip0->payload_length = clib_host_to_net_u16 (new_l0); + + sr0 = ((void *) sr0) - vec_len (sl0->rewrite); + sr0->segments->as_u64[0] = ip0->dst_address.as_u64[0]; + sr0->segments->as_u64[1] = ip0->dst_address.as_u64[1]; + + ip0->dst_address.as_u64[0] = + (sr0->segments + sr0->segments_left)->as_u64[0]; + ip0->dst_address.as_u64[1] = + (sr0->segments + sr0->segments_left)->as_u64[1]; + + if (ip0 + 1 == (void *) sr0) + { + sr0->protocol = ip0->protocol; + ip0->protocol = IP_PROTOCOL_IPV6_ROUTE; + } + else + { + ip6_ext_header_t *ip_ext = (void *) (ip0 + 1); + sr0->protocol = ip_ext->next_hdr; + ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE; + } + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) && + PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + insert_pkts++; + + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + /* Update counters */ + vlib_node_increment_counter (vm, sr_policy_rewrite_insert_node.index, + SR_POLICY_REWRITE_ERROR_COUNTER_TOTAL, + insert_pkts); + vlib_node_increment_counter (vm, sr_policy_rewrite_insert_node.index, + SR_POLICY_REWRITE_ERROR_COUNTER_BSID, + bsid_pkts); + return from_frame->n_vectors; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (sr_policy_rewrite_insert_node) = { + .function = sr_policy_rewrite_insert, + .name = "sr-pl-rewrite-insert", + .vector_size = sizeof (u32), + .format_trace = format_sr_policy_rewrite_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = SR_POLICY_REWRITE_N_ERROR, + .error_strings = sr_policy_rewrite_error_strings, + .n_next_nodes = SR_POLICY_REWRITE_N_NEXT, + .next_nodes = { +#define _(s,n) [SR_POLICY_REWRITE_NEXT_##s] = n, + foreach_sr_policy_rewrite_next +#undef _ + }, +}; +/* *INDENT-ON* */ + +/** + * @brief Graph node for applying a SR policy into a packet. BSID - SRH insertion. + */ +static uword +sr_policy_rewrite_b_insert (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + ip6_sr_main_t *sm = &sr_main; + u32 n_left_from, next_index, *from, *to_next; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + int insert_pkts = 0, bsid_pkts = 0; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + /* Quad - Loop */ + while (n_left_from >= 8 && n_left_to_next >= 4) + { + u32 bi0, bi1, bi2, bi3; + vlib_buffer_t *b0, *b1, *b2, *b3; + u32 next0, next1, next2, next3; + next0 = next1 = next2 = next3 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP; + ip6_header_t *ip0, *ip1, *ip2, *ip3; + ip6_sr_header_t *sr0, *sr1, *sr2, *sr3; + ip6_sr_sl_t *sl0, *sl1, *sl2, *sl3; + u16 new_l0, new_l1, new_l2, new_l3; + + /* Prefetch next iteration. */ + { + vlib_buffer_t *p4, *p5, *p6, *p7; + + p4 = vlib_get_buffer (vm, from[4]); + p5 = vlib_get_buffer (vm, from[5]); + p6 = vlib_get_buffer (vm, from[6]); + p7 = vlib_get_buffer (vm, from[7]); + + /* Prefetch the buffer header and packet for the N+2 loop iteration */ + vlib_prefetch_buffer_header (p4, LOAD); + vlib_prefetch_buffer_header (p5, LOAD); + vlib_prefetch_buffer_header (p6, LOAD); + vlib_prefetch_buffer_header (p7, LOAD); + + CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + } + + to_next[0] = bi0 = from[0]; + to_next[1] = bi1 = from[1]; + to_next[2] = bi2 = from[2]; + to_next[3] = bi3 = from[3]; + from += 4; + to_next += 4; + n_left_from -= 4; + n_left_to_next -= 4; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + b2 = vlib_get_buffer (vm, bi2); + b3 = vlib_get_buffer (vm, bi3); + + sl0 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b0)->ip.adj_index[VLIB_TX]); + sl1 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b1)->ip.adj_index[VLIB_TX]); + sl2 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b2)->ip.adj_index[VLIB_TX]); + sl3 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b3)->ip.adj_index[VLIB_TX]); + + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl0->rewrite_bsid) + b0->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl1->rewrite_bsid) + b1->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl2->rewrite_bsid) + b2->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl3->rewrite_bsid) + b3->current_data)); + + ip0 = vlib_buffer_get_current (b0); + ip1 = vlib_buffer_get_current (b1); + ip2 = vlib_buffer_get_current (b2); + ip3 = vlib_buffer_get_current (b3); + + if (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) + sr0 = + (ip6_sr_header_t *) (((void *) (ip0 + 1)) + + ip6_ext_header_len (ip0 + 1)); + else + sr0 = (ip6_sr_header_t *) (ip0 + 1); + + if (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) + sr1 = + (ip6_sr_header_t *) (((void *) (ip1 + 1)) + + ip6_ext_header_len (ip1 + 1)); + else + sr1 = (ip6_sr_header_t *) (ip1 + 1); + + if (ip2->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) + sr2 = + (ip6_sr_header_t *) (((void *) (ip2 + 1)) + + ip6_ext_header_len (ip2 + 1)); + else + sr2 = (ip6_sr_header_t *) (ip2 + 1); + + if (ip3->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) + sr3 = + (ip6_sr_header_t *) (((void *) (ip3 + 1)) + + ip6_ext_header_len (ip3 + 1)); + else + sr3 = (ip6_sr_header_t *) (ip3 + 1); + + clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite_bsid), (u8 *) ip0, + (void *) sr0 - (void *) ip0); + clib_memcpy ((u8 *) ip1 - vec_len (sl1->rewrite_bsid), (u8 *) ip1, + (void *) sr1 - (void *) ip1); + clib_memcpy ((u8 *) ip2 - vec_len (sl2->rewrite_bsid), (u8 *) ip2, + (void *) sr2 - (void *) ip2); + clib_memcpy ((u8 *) ip3 - vec_len (sl3->rewrite_bsid), (u8 *) ip3, + (void *) sr3 - (void *) ip3); + + clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite_bsid)), + sl0->rewrite_bsid, vec_len (sl0->rewrite_bsid)); + clib_memcpy (((u8 *) sr1 - vec_len (sl1->rewrite_bsid)), + sl1->rewrite_bsid, vec_len (sl1->rewrite_bsid)); + clib_memcpy (((u8 *) sr2 - vec_len (sl2->rewrite_bsid)), + sl2->rewrite_bsid, vec_len (sl2->rewrite_bsid)); + clib_memcpy (((u8 *) sr3 - vec_len (sl3->rewrite_bsid)), + sl3->rewrite_bsid, vec_len (sl3->rewrite_bsid)); + + vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite_bsid)); + vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite_bsid)); + vlib_buffer_advance (b2, -(word) vec_len (sl2->rewrite_bsid)); + vlib_buffer_advance (b3, -(word) vec_len (sl3->rewrite_bsid)); + + ip0 = ((void *) ip0) - vec_len (sl0->rewrite_bsid); + ip1 = ((void *) ip1) - vec_len (sl1->rewrite_bsid); + ip2 = ((void *) ip2) - vec_len (sl2->rewrite_bsid); + ip3 = ((void *) ip3) - vec_len (sl3->rewrite_bsid); + + ip0->hop_limit -= 1; + ip1->hop_limit -= 1; + ip2->hop_limit -= 1; + ip3->hop_limit -= 1; + + new_l0 = + clib_net_to_host_u16 (ip0->payload_length) + + vec_len (sl0->rewrite_bsid); + new_l1 = + clib_net_to_host_u16 (ip1->payload_length) + + vec_len (sl1->rewrite_bsid); + new_l2 = + clib_net_to_host_u16 (ip2->payload_length) + + vec_len (sl2->rewrite_bsid); + new_l3 = + clib_net_to_host_u16 (ip3->payload_length) + + vec_len (sl3->rewrite_bsid); + + ip0->payload_length = clib_host_to_net_u16 (new_l0); + ip1->payload_length = clib_host_to_net_u16 (new_l1); + ip2->payload_length = clib_host_to_net_u16 (new_l2); + ip3->payload_length = clib_host_to_net_u16 (new_l3); + + sr0 = ((void *) sr0) - vec_len (sl0->rewrite_bsid); + sr1 = ((void *) sr1) - vec_len (sl1->rewrite_bsid); + sr2 = ((void *) sr2) - vec_len (sl2->rewrite_bsid); + sr3 = ((void *) sr3) - vec_len (sl3->rewrite_bsid); + + ip0->dst_address.as_u64[0] = + (sr0->segments + sr0->segments_left)->as_u64[0]; + ip0->dst_address.as_u64[1] = + (sr0->segments + sr0->segments_left)->as_u64[1]; + ip1->dst_address.as_u64[0] = + (sr1->segments + sr1->segments_left)->as_u64[0]; + ip1->dst_address.as_u64[1] = + (sr1->segments + sr1->segments_left)->as_u64[1]; + ip2->dst_address.as_u64[0] = + (sr2->segments + sr2->segments_left)->as_u64[0]; + ip2->dst_address.as_u64[1] = + (sr2->segments + sr2->segments_left)->as_u64[1]; + ip3->dst_address.as_u64[0] = + (sr3->segments + sr3->segments_left)->as_u64[0]; + ip3->dst_address.as_u64[1] = + (sr3->segments + sr3->segments_left)->as_u64[1]; + + ip6_ext_header_t *ip_ext; + if (ip0 + 1 == (void *) sr0) + { + sr0->protocol = ip0->protocol; + ip0->protocol = IP_PROTOCOL_IPV6_ROUTE; + } + else + { + ip_ext = (void *) (ip0 + 1); + sr0->protocol = ip_ext->next_hdr; + ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE; + } + + if (ip1 + 1 == (void *) sr1) + { + sr1->protocol = ip1->protocol; + ip1->protocol = IP_PROTOCOL_IPV6_ROUTE; + } + else + { + ip_ext = (void *) (ip2 + 1); + sr2->protocol = ip_ext->next_hdr; + ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE; + } + + if (ip2 + 1 == (void *) sr2) + { + sr2->protocol = ip2->protocol; + ip2->protocol = IP_PROTOCOL_IPV6_ROUTE; + } + else + { + ip_ext = (void *) (ip2 + 1); + sr2->protocol = ip_ext->next_hdr; + ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE; + } + + if (ip3 + 1 == (void *) sr3) + { + sr3->protocol = ip3->protocol; + ip3->protocol = IP_PROTOCOL_IPV6_ROUTE; + } + else + { + ip_ext = (void *) (ip3 + 1); + sr3->protocol = ip_ext->next_hdr; + ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE; + } + + insert_pkts += 4; + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE))) + { + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b1, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b2, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b3, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + } + + vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, + n_left_to_next, bi0, bi1, bi2, bi3, + next0, next1, next2, next3); + } + + /* Single loop for potentially the last three packets */ + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + ip6_header_t *ip0 = 0; + ip6_sr_header_t *sr0 = 0; + ip6_sr_sl_t *sl0; + u32 next0 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP; + u16 new_l0 = 0; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + sl0 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b0)->ip.adj_index[VLIB_TX]); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl0->rewrite_bsid) + b0->current_data)); + + ip0 = vlib_buffer_get_current (b0); + + if (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) + sr0 = + (ip6_sr_header_t *) (((void *) (ip0 + 1)) + + ip6_ext_header_len (ip0 + 1)); + else + sr0 = (ip6_sr_header_t *) (ip0 + 1); + + clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite_bsid), (u8 *) ip0, + (void *) sr0 - (void *) ip0); + clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite_bsid)), + sl0->rewrite_bsid, vec_len (sl0->rewrite_bsid)); + + vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite_bsid)); + + ip0 = ((void *) ip0) - vec_len (sl0->rewrite_bsid); + ip0->hop_limit -= 1; + new_l0 = + clib_net_to_host_u16 (ip0->payload_length) + + vec_len (sl0->rewrite_bsid); + ip0->payload_length = clib_host_to_net_u16 (new_l0); + + sr0 = ((void *) sr0) - vec_len (sl0->rewrite_bsid); + + ip0->dst_address.as_u64[0] = + (sr0->segments + sr0->segments_left)->as_u64[0]; + ip0->dst_address.as_u64[1] = + (sr0->segments + sr0->segments_left)->as_u64[1]; + + if (ip0 + 1 == (void *) sr0) + { + sr0->protocol = ip0->protocol; + ip0->protocol = IP_PROTOCOL_IPV6_ROUTE; + } + else + { + ip6_ext_header_t *ip_ext = (void *) (ip0 + 1); + sr0->protocol = ip_ext->next_hdr; + ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE; + } + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) && + PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + insert_pkts++; + + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + /* Update counters */ + vlib_node_increment_counter (vm, sr_policy_rewrite_insert_node.index, + SR_POLICY_REWRITE_ERROR_COUNTER_TOTAL, + insert_pkts); + vlib_node_increment_counter (vm, sr_policy_rewrite_insert_node.index, + SR_POLICY_REWRITE_ERROR_COUNTER_BSID, + bsid_pkts); + return from_frame->n_vectors; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (sr_policy_rewrite_b_insert_node) = { + .function = sr_policy_rewrite_b_insert, + .name = "sr-pl-rewrite-b-insert", + .vector_size = sizeof (u32), + .format_trace = format_sr_policy_rewrite_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = SR_POLICY_REWRITE_N_ERROR, + .error_strings = sr_policy_rewrite_error_strings, + .n_next_nodes = SR_POLICY_REWRITE_N_NEXT, + .next_nodes = { +#define _(s,n) [SR_POLICY_REWRITE_NEXT_##s] = n, + foreach_sr_policy_rewrite_next +#undef _ + }, +}; +/* *INDENT-ON* */ + +/** + * @brief Function BSID encapsulation + */ +static_always_inline void +end_bsid_encaps_srh_processing (vlib_node_runtime_t * node, + vlib_buffer_t * b0, + ip6_header_t * ip0, + ip6_sr_header_t * sr0, u32 * next0) +{ + ip6_address_t *new_dst0; + + if (PREDICT_FALSE (!sr0)) + goto error_bsid_encaps; + + if (PREDICT_TRUE (sr0->type == ROUTING_HEADER_TYPE_SR)) + { + if (PREDICT_TRUE (sr0->segments_left != 0)) + { + sr0->segments_left -= 1; + new_dst0 = (ip6_address_t *) (sr0->segments); + new_dst0 += sr0->segments_left; + ip0->dst_address.as_u64[0] = new_dst0->as_u64[0]; + ip0->dst_address.as_u64[1] = new_dst0->as_u64[1]; + return; + } + } + +error_bsid_encaps: + *next0 = SR_POLICY_REWRITE_NEXT_ERROR; + b0->error = node->errors[SR_POLICY_REWRITE_ERROR_BSID_ZERO]; +} + +/** + * @brief Graph node for applying a SR policy BSID - Encapsulation + */ +static uword +sr_policy_rewrite_b_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + ip6_sr_main_t *sm = &sr_main; + u32 n_left_from, next_index, *from, *to_next; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + int encap_pkts = 0, bsid_pkts = 0; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + /* Quad - Loop */ + while (n_left_from >= 8 && n_left_to_next >= 4) + { + u32 bi0, bi1, bi2, bi3; + vlib_buffer_t *b0, *b1, *b2, *b3; + u32 next0, next1, next2, next3; + next0 = next1 = next2 = next3 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP; + ip6_header_t *ip0, *ip1, *ip2, *ip3; + ip6_header_t *ip0_encap, *ip1_encap, *ip2_encap, *ip3_encap; + ip6_sr_header_t *sr0, *sr1, *sr2, *sr3; + ip6_ext_header_t *prev0, *prev1, *prev2, *prev3; + ip6_sr_sl_t *sl0, *sl1, *sl2, *sl3; + + /* Prefetch next iteration. */ + { + vlib_buffer_t *p4, *p5, *p6, *p7; + + p4 = vlib_get_buffer (vm, from[4]); + p5 = vlib_get_buffer (vm, from[5]); + p6 = vlib_get_buffer (vm, from[6]); + p7 = vlib_get_buffer (vm, from[7]); + + /* Prefetch the buffer header and packet for the N+2 loop iteration */ + vlib_prefetch_buffer_header (p4, LOAD); + vlib_prefetch_buffer_header (p5, LOAD); + vlib_prefetch_buffer_header (p6, LOAD); + vlib_prefetch_buffer_header (p7, LOAD); + + CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + } + + to_next[0] = bi0 = from[0]; + to_next[1] = bi1 = from[1]; + to_next[2] = bi2 = from[2]; + to_next[3] = bi3 = from[3]; + from += 4; + to_next += 4; + n_left_from -= 4; + n_left_to_next -= 4; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + b2 = vlib_get_buffer (vm, bi2); + b3 = vlib_get_buffer (vm, bi3); + + sl0 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b0)->ip.adj_index[VLIB_TX]); + sl1 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b1)->ip.adj_index[VLIB_TX]); + sl2 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b2)->ip.adj_index[VLIB_TX]); + sl3 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b3)->ip.adj_index[VLIB_TX]); + + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl0->rewrite) + b0->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl1->rewrite) + b1->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl2->rewrite) + b2->current_data)); + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl3->rewrite) + b3->current_data)); + + ip0_encap = vlib_buffer_get_current (b0); + ip1_encap = vlib_buffer_get_current (b1); + ip2_encap = vlib_buffer_get_current (b2); + ip3_encap = vlib_buffer_get_current (b3); + + ip6_ext_header_find_t (ip0_encap, prev0, sr0, + IP_PROTOCOL_IPV6_ROUTE); + ip6_ext_header_find_t (ip1_encap, prev1, sr1, + IP_PROTOCOL_IPV6_ROUTE); + ip6_ext_header_find_t (ip2_encap, prev2, sr2, + IP_PROTOCOL_IPV6_ROUTE); + ip6_ext_header_find_t (ip3_encap, prev3, sr3, + IP_PROTOCOL_IPV6_ROUTE); + + end_bsid_encaps_srh_processing (node, b0, ip0_encap, sr0, &next0); + end_bsid_encaps_srh_processing (node, b1, ip1_encap, sr1, &next1); + end_bsid_encaps_srh_processing (node, b2, ip2_encap, sr2, &next2); + end_bsid_encaps_srh_processing (node, b3, ip3_encap, sr3, &next3); + + clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy (((u8 *) ip1_encap) - vec_len (sl1->rewrite), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy (((u8 *) ip2_encap) - vec_len (sl2->rewrite), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy (((u8 *) ip3_encap) - vec_len (sl3->rewrite), + sl3->rewrite, vec_len (sl3->rewrite)); + + vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); + vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); + vlib_buffer_advance (b2, -(word) vec_len (sl2->rewrite)); + vlib_buffer_advance (b3, -(word) vec_len (sl3->rewrite)); + + ip0 = vlib_buffer_get_current (b0); + ip1 = vlib_buffer_get_current (b1); + ip2 = vlib_buffer_get_current (b2); + ip3 = vlib_buffer_get_current (b3); + + encaps_processing_v6 (node, b0, ip0, ip0_encap); + encaps_processing_v6 (node, b1, ip1, ip1_encap); + encaps_processing_v6 (node, b2, ip2, ip2_encap); + encaps_processing_v6 (node, b3, ip3, ip3_encap); + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE))) + { + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b1, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b2, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b3, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + } + + encap_pkts += 4; + vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, + n_left_to_next, bi0, bi1, bi2, bi3, + next0, next1, next2, next3); + } + + /* Single loop for potentially the last three packets */ + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + ip6_header_t *ip0 = 0, *ip0_encap = 0; + ip6_ext_header_t *prev0; + ip6_sr_header_t *sr0; + ip6_sr_sl_t *sl0; + u32 next0 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + b0 = vlib_get_buffer (vm, bi0); + + sl0 = + pool_elt_at_index (sm->sid_lists, + vnet_buffer (b0)->ip.adj_index[VLIB_TX]); + + ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= + (vec_len (sl0->rewrite) + b0->current_data)); + + ip0_encap = vlib_buffer_get_current (b0); + ip6_ext_header_find_t (ip0_encap, prev0, sr0, + IP_PROTOCOL_IPV6_ROUTE); + end_bsid_encaps_srh_processing (node, b0, ip0_encap, sr0, &next0); + + clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); + + ip0 = vlib_buffer_get_current (b0); + + encaps_processing_v6 (node, b0, ip0, ip0_encap); + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) && + PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + sr_policy_rewrite_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); + } + + encap_pkts++; + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + /* Update counters */ + vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index, + SR_POLICY_REWRITE_ERROR_COUNTER_TOTAL, + encap_pkts); + vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index, + SR_POLICY_REWRITE_ERROR_COUNTER_BSID, + bsid_pkts); + + return from_frame->n_vectors; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (sr_policy_rewrite_b_encaps_node) = { + .function = sr_policy_rewrite_b_encaps, + .name = "sr-pl-rewrite-b-encaps", + .vector_size = sizeof (u32), + .format_trace = format_sr_policy_rewrite_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = SR_POLICY_REWRITE_N_ERROR, + .error_strings = sr_policy_rewrite_error_strings, + .n_next_nodes = SR_POLICY_REWRITE_N_NEXT, + .next_nodes = { +#define _(s,n) [SR_POLICY_REWRITE_NEXT_##s] = n, + foreach_sr_policy_rewrite_next +#undef _ + }, +}; +/* *INDENT-ON* */ + +/*************************** SR Segment Lists DPOs ****************************/ +static u8 * +format_sr_segment_list_dpo (u8 * s, va_list * args) +{ + ip6_sr_main_t *sm = &sr_main; + ip6_address_t *addr; + ip6_sr_sl_t *sl; + + index_t index = va_arg (*args, index_t); + CLIB_UNUSED (u32 indent) = va_arg (*args, u32); + s = format (s, "SR: Segment List index:[%d]", index); + s = format (s, "\n\tSegments:"); + + sl = pool_elt_at_index (sm->sid_lists, index); + + s = format (s, "< "); + vec_foreach (addr, sl->segments) + { + s = format (s, "%U, ", format_ip6_address, addr); + } + s = format (s, "\b\b > - "); + s = format (s, "Weight: %u", sl->weight); + + return s; +} + +const static dpo_vft_t sr_policy_rewrite_vft = { + .dv_lock = sr_dpo_lock, + .dv_unlock = sr_dpo_unlock, + .dv_format = format_sr_segment_list_dpo, +}; + +const static char *const sr_pr_encaps_ip6_nodes[] = { + "sr-pl-rewrite-encaps", + NULL, +}; + +const static char *const sr_pr_encaps_ip4_nodes[] = { + "sr-pl-rewrite-encaps-v4", + NULL, +}; + +const static char *const *const sr_pr_encaps_nodes[DPO_PROTO_NUM] = { + [DPO_PROTO_IP6] = sr_pr_encaps_ip6_nodes, + [DPO_PROTO_IP4] = sr_pr_encaps_ip4_nodes, +}; + +const static char *const sr_pr_insert_ip6_nodes[] = { + "sr-pl-rewrite-insert", + NULL, +}; + +const static char *const *const sr_pr_insert_nodes[DPO_PROTO_NUM] = { + [DPO_PROTO_IP6] = sr_pr_insert_ip6_nodes, +}; + +const static char *const sr_pr_bsid_insert_ip6_nodes[] = { + "sr-pl-rewrite-b-insert", + NULL, +}; + +const static char *const *const sr_pr_bsid_insert_nodes[DPO_PROTO_NUM] = { + [DPO_PROTO_IP6] = sr_pr_bsid_insert_ip6_nodes, +}; + +const static char *const sr_pr_bsid_encaps_ip6_nodes[] = { + "sr-pl-rewrite-b-encaps", + NULL, +}; + +const static char *const *const sr_pr_bsid_encaps_nodes[DPO_PROTO_NUM] = { + [DPO_PROTO_IP6] = sr_pr_bsid_encaps_ip6_nodes, +}; + +/********************* SR Policy Rewrite initialization ***********************/ +/** + * @brief SR Policy Rewrite initialization + */ +clib_error_t * +sr_policy_rewrite_init (vlib_main_t * vm) +{ + ip6_sr_main_t *sm = &sr_main; + + /* Init memory for sr policy keys (bsid <-> ip6_address_t) */ + sm->sr_policy_index_by_key = hash_create_mem (0, sizeof (ip6_address_t), + sizeof (uword)); + + /* Init SR VPO DPOs type */ + sr_pr_encaps_dpo_type = + dpo_register_new_type (&sr_policy_rewrite_vft, sr_pr_encaps_nodes); + + sr_pr_insert_dpo_type = + dpo_register_new_type (&sr_policy_rewrite_vft, sr_pr_insert_nodes); + + sr_pr_bsid_encaps_dpo_type = + dpo_register_new_type (&sr_policy_rewrite_vft, sr_pr_bsid_encaps_nodes); + + sr_pr_bsid_insert_dpo_type = + dpo_register_new_type (&sr_policy_rewrite_vft, sr_pr_bsid_insert_nodes); + + /* Register the L2 encaps node used in HW redirect */ + sm->l2_sr_policy_rewrite_index = sr_policy_rewrite_encaps_node.index; + + sm->fib_table_ip6 = (u32) ~ 0; + sm->fib_table_ip4 = (u32) ~ 0; + + return 0; +} + +VLIB_INIT_FUNCTION (sr_policy_rewrite_init); + + +/* +* fd.io coding-style-patch-verification: ON +* +* Local Variables: +* eval: (c-set-style "gnu") +* End: +*/ |