1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189/*
* Copyright (c) 2015 Cisco and/or its affiliates.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
Copyright (c) 2005 Eliot Dresselhaus
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef included_phash_h
#define included_phash_h
#include <vppinfra/hash.h> /* for Bob's mixing functions */
typedef struct
{
/* Maybe either pointer to vector or inline word. */
uword key;
/* Hash code (A, B). */
u32 a, b;
} phash_key_t;
/* Table indexed by B. */
typedef struct
{
/* Vector of key indices with this same value of B. */
u32 *keys;
/* hash=a^tabb[b].val_b */
u32 val_b;
/* High watermark of who has visited this map node. */
u32 water_b;
} phash_tabb_t;
always_inline void
phash_tabb_free (phash_tabb_t * b)
{
vec_free (b->keys);
b->val_b = b->water_b = 0;
}
typedef struct
{
/* b that currently occupies this hash */
u32 b_q;
/* Queue position of parent that could use this hash. */
u32 parent_q;
/* What to change parent tab[b] to use this hash. */
u32 newval_q;
/* Original value of tab[b]. */
u32 oldval_q;
} phash_tabq_t;
typedef struct
{
u8 a_bits, b_bits, s_bits, a_shift;
u32 b_mask;
u32 *tab;
u32 *scramble;
/* Seed value for hash mixer. */
u64 hash_seed;
u32 flags;
/* Key functions want 64 bit keys.
Use hash_mix64 rather than hash_mix32. */
#define PHASH_FLAG_MIX64 (1 << 0)
#define PHASH_FLAG_MIX32 (0 << 0)
/* When b_bits is large enough (>= 12) we scramble. */
#define PHASH_FLAG_USE_SCRAMBLE (1 << 1)
/* Slow mode gives smaller tables but at the expense of more run time. */
#define PHASH_FLAG_SLOW_MODE (0 << 2)
#define PHASH_FLAG_FAST_MODE (1 << 2)
/* Generate minimal perfect hash instead of perfect hash. */
#define PHASH_FLAG_NON_MINIMAL (0 << 3)
#define PHASH_FLAG_MINIMAL (1 << 3)
/* vec_len (keys) for minimal hash;
1 << s_bits for non-minimal hash. */
u32 hash_max;
/* Vector of keys. */
phash_key_t *keys;
/* Used by callbacks to identify keys. */
void *private;
/* Key comparison callback. */
int (*key_is_equal) (void *private, uword key1, uword key2);
/* Callback to reduce single key -> hash seeds. */
void (*key_seed1) (void *private, uword key
@media only all and (prefers-color-scheme: dark) {
.highlight .hll { background-color: #49483e }
.highlight .c { color: #75715e } /* Comment */
.highlight .err { color: #960050; background-color: #1e0010 } /* Error */
.highlight .k { color: #66d9ef } /* Keyword */
.highlight .l { color: #ae81ff } /* Literal */
.highlight .n { color: #f8f8f2 } /* Name */
.highlight .o { color: #f92672 } /* Operator */
.highlight .p { color: #f8f8f2 } /* Punctuation */
.highlight .ch { color: #75715e } /* Comment.Hashbang */
.highlight .cm { color: #75715e } /* Comment.Multiline */
.highlight .cp { color: #75715e } /* Comment.Preproc */
.highlight .cpf { color: #75715e } /* Comment.PreprocFile */
.highlight .c1 { color: #75715e } /* Comment.Single */
.highlight .cs { color: #75715e } /* Comment.Special */
.highlight .gd { color: #f92672 } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gi { color: #a6e22e } /* Generic.Inserted */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #75715e } /* Generic.Subheading */
.highlight .kc { color: #66d9ef } /* Keyword.Constant */
.highlight .kd { color: #66d9ef } /* Keyword.Declaration */
.highlight .kn { color: #f92672 } /* Keyword.Namespace */
.highlight .kp { color: #66d9ef } /* Keyword.Pseudo */
.highlight .kr { color: #66d9ef } /* Keyword.Reserved */
.highlight .kt { color: #66d9ef } /* Keyword.Type */
.highlight .ld { color: #e6db74 } /* Literal.Date */
.highlight .m { color: #ae81ff } /* Literal.Number */
.highlight .s { color: #e6db74 } /* Literal.String */
.highlight .na { color: #a6e22e } /* Name.Attribute */
.highlight .nb { color: #f8f8f2 } /* Name.Builtin */
.highlight .nc { color: #a6e22e } /* Name.Class */
.highlight .no { color: #66d9ef } /* Name.Constant */
.highlight .nd { color: #a6e22e } /* Name.Decorator */
.highlight .ni { color: #f8f8f2 } /* Name.Entity */
.highlight .ne { color: #a6e22e } /* Name.Exception */
.highlight .nf { color: #a6e22e } /* Name.Function */
.highlight .nl { color: #f8f8f2 } /* Name.Label */
.highlight .nn { color: #f8f8f2 } /* Name.Namespace */
.highlight .nx { color: #a6e22e } /* Name.Other */
.highlight .py { color: #f8f8f2 } /* Name.Property */
.highlight .nt { color: #f92672 } /* Name.Tag */
.highlight .nv { color: #f8f8f2 } /* Name.Variable */
.highlight .ow { color: #f92672 } /* Operator.Word */
.highlight .w { color: #f8f8f2 } /* Text.Whitespace */
.highlight .mb { color: #ae81ff } /* Literal.Number.Bin */
.highlight .mf { color: #ae81ff } /* Literal.Number.Float */
.highlight .mh { color: #ae81ff } /* Literal.Number.Hex */
.highlight .mi { color: #ae81ff } /* Literal.Number.Integer */
.highlight .mo { color: #ae81ff } /* Literal.Number.Oct */
.highlight .sa { color: #e6db74 } /* Literal.String.Affix */
.highlight .sb { color: #e6db74 } /* Literal.String.Backtick */
.highlight .sc { color: #e6db74 } /* Literal.String.Char */
.highlight .dl { color: #e6db74 } /* Literal.String.Delimiter */
.highlight .sd { color: #e6db74 } /* Literal.String.Doc */
.highlight .s2 { color: #e6db74 } /* Literal.String.Double */
.highlight .se { color: #ae81ff } /* Literal.String.Escape */
.highlight .sh { color: #e6db74 } /* Literal.String.Heredoc */
.highlight .si { color: #e6db74 } /* Literal.String.Interpol */
.highlight .sx { color: #e6db74 } /* Literal.String.Other */
.highlight .sr { color: #e6db74 } /* Literal.String.Regex */
.highlight .s1 { color: #e6db74 } /* Literal.String.Single */
.highlight .ss { color: #e6db74 } /* Literal.String.Symbol */
.highlight .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #a6e22e } /* Name.Function.Magic */
.highlight .vc { color: #f8f8f2 } /* Name.Variable.Class */
.highlight .vg { color: #f8f8f2 } /* Name.Variable.Global */
.highlight .vi { color: #f8f8f2 } /* Name.Variable.Instance */
.highlight .vm { color: #f8f8f2 } /* Name.Variable.Magic */
.highlight .il { color: #ae81ff } /* Literal.Number.Integer.Long */
}
@media (prefers-color-scheme: light) {
.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight .cpf { color: #888888 } /* Comment.PreprocFile */
.highlight .c1 { color: #888888 } /* Comment.Single */
.highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */
.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gr { color: #aa0000 } /* Generic.Error */
.highlight .gh { color: #333333 } /* Generic.Heading */
.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #555555 } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #666666 } /* Generic.Subheading */
.highlight .gt { color: #aa0000 } /* Generic.Traceback */
.highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008800 } /* Keyword.Pseudo */
.highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */
.highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */
.highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */
.highlight .na { color: #336699 } /* Name.Attribute */
.highlight .nb { color: #003388 } /* Name.Builtin */
.highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */
.highlight .no { color: #003366; font-weight: bold } /* Name.Constant */
.highlight .nd { color: #555555 } /* Name.Decorator */
.highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */
.highlight .nl { color: #336699; font-style: italic } /* Name.Label */
.highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */
.highlight .py { color: #336699; font-weight: bold } /* Name.Property */
.highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #336699 } /* Name.Variable */
.highlight .ow { color: #008800 } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */
.highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */
.highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */
.highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */
.highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */
.highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */
.highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */
.highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */
.highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */
.highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */
.highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */
.highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */
.highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */
.highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */
.highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */
.highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */
.highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */
.highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */
.highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */
.highlight .vc { color: #336699 } /* Name.Variable.Class */
.highlight .vg { color: #dd7700 } /* Name.Variable.Global */
.highlight .vi { color: #3333bb } /* Name.Variable.Instance */
.highlight .vm { color: #336699 } /* Name.Variable.Magic */
.highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */
}
/*
* Copyright (c) 2017 Intel and/or its affiliates.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <vppinfra/error.h>
#include <vppinfra/hash.h>
#include <vnet/vnet.h>
#include <vnet/ip/ip.h>
#include <vnet/ethernet/ethernet.h>
#include <gtpu/gtpu.h>
/* Statistics (not all errors) */
#define foreach_gtpu_encap_error \
_(ENCAPSULATED, "good packets encapsulated")
static char * gtpu_encap_error_strings[] = {
#define _(sym,string) string,
foreach_gtpu_encap_error
#undef _
};
typedef enum {
#define _(sym,str) GTPU_ENCAP_ERROR_##sym,
foreach_gtpu_encap_error
#undef _
GTPU_ENCAP_N_ERROR,
} gtpu_encap_error_t;
#define foreach_gtpu_encap_next \
_(DROP, "error-drop") \
_(IP4_LOOKUP, "ip4-lookup") \
_(IP6_LOOKUP, "ip6-lookup")
typedef enum {
GTPU_ENCAP_NEXT_DROP,
GTPU_ENCAP_NEXT_IP4_LOOKUP,
GTPU_ENCAP_NEXT_IP6_LOOKUP,
GTPU_ENCAP_N_NEXT,
} gtpu_encap_next_t;
typedef struct {
u32 tunnel_index;
u32 teid;
} gtpu_encap_trace_t;
u8 * format_gtpu_encap_trace (u8 * s, va_list * args)
{
CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
gtpu_encap_trace_t * t
= va_arg (*args, gtpu_encap_trace_t *);
s = format (s, "GTPU encap to gtpu_tunnel%d teid %d",
t->tunnel_index, t->teid);
return s;
}
#define foreach_fixed_header4_offset \
_(0) _(1) _(2) _(3)
#define foreach_fixed_header6_offset \
_(0) _(1) _(2) _(3) _(4) _(5) _(6)
always_inline uword
gtpu_encap_inline (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * from_frame,
u32 is_ip4)
{
u32 n_left_from, next_index, * from, * to_next;
gtpu_main_t * gtm = >pu_main;
vnet_main_t * vnm = gtm->vnet_main;
vnet_interface_main_t * im = &vnm->interface_main;
u32 pkts_encapsulated = 0;
u16 old_l0 = 0, old_l1 = 0, old_l2 = 0, old_l3 = 0;
u32 thread_index = vlib_get_thread_index();
u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
u32 sw_if_index0 = 0, sw_if_index1 = 0, sw_if_index2 = 0, sw_if_index3 = 0;
u32 next0 = 0, next1 = 0, next2 = 0, next3 = 0;
vnet_hw_interface_t * hi0, * hi1, * hi2, * hi3;
gtpu_tunnel_t * t0 = NULL, * t1 = NULL, * t2 = NULL, * t3 = NULL;
from = vlib_frame_vector_args (from_frame);
n_left_from = from_frame->n_vectors;
next_index = node->cached_next_index;
stats_sw_if_index = node->runtime_data[0];
stats_n_packets = stats_n_bytes = 0;
while (n_left_from > 0)
{
u32 n_left_to_next;
vlib_get_next_frame (vm, node, next_index,
to_next, n_left_to_next);
while (n_left_from >= 8 && n_left_to_next >= 4)
{
u32 bi0, bi1, bi2, bi3;
vlib_buffer_t * b0, * b1, * b2, * b3;
u32 flow_hash0, flow_hash1, flow_hash2, flow_hash3;
u32 len0, len1, len2, len3;
ip4_header_t * ip4_0, * ip4_1, * ip4_2, * ip4_3;
ip6_header_t * ip6_0, * ip6_1, * ip6_2, * ip6_3;
udp_header_t * udp0, * udp1, * udp2, * udp3;
gtpu_header_t * gtpu0, * gtpu1, * gtpu2, * gtpu3;
u64 * copy_src0, * copy_dst0;
u64 * copy_src1, * copy_dst1;
u64 * copy_src2, * copy_dst2;
u64 * copy_src3, * copy_dst3;
u32 * copy_src_last0, * copy_dst_last0;
u32 * copy_src_last1, * copy_dst_last1;
u32 * copy_src_last2, * copy_dst_last2;
u32 * copy_src_last3, * copy_dst_last3;
u16 new_l0, new_l1, new_l2, new_l3;
ip_csum_t sum0, sum1, sum2, sum3;
/* Prefetch next iteration. */
{
vlib_buffer_t * p4, * p5, * p6, * p7;
p4 = vlib_get_buffer (vm, from[4]);
p5 = vlib_get_buffer (vm, from[5]);
p6 = vlib_get_buffer (vm, from[6]);
p7 = vlib_get_buffer (vm, from[7]);
vlib_prefetch_buffer_header (p4, LOAD);
vlib_prefetch_buffer_header (p5, LOAD);
vlib_prefetch_buffer_header (p6, LOAD);
vlib_prefetch_buffer_header (p7, LOAD);
CLIB_PREFETCH (p4->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
CLIB_PREFETCH (p5->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
CLIB_PREFETCH (p6->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
CLIB_PREFETCH (p7->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
}
bi0 = from[0];
bi1 = from[1];
bi2 = from[2];
bi3 = from[3];
to_next[0] = bi0;
to_next[1] = bi1;
to_next[2] = bi2;
to_next[3] = bi3;
from += 4;
to_next += 4;
n_left_to_next -= 4;
n_left_from -= 4;
b0 = vlib_get_buffer (vm, bi0);
b1 = vlib_get_buffer (vm, bi1);
b2 = vlib_get_buffer (vm, bi2);
b3 = vlib_get_buffer (vm, bi3);
flow_hash0 = vnet_l2_compute_flow_hash (b0);
flow_hash1 = vnet_l2_compute_flow_hash (b1);
flow_hash2 = vnet_l2_compute_flow_hash (b2);
flow_hash3 = vnet_l2_compute_flow_hash (b3);
/* Get next node index and adj index from tunnel next_dpo */
sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_TX];
sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_TX];
sw_if_index2 = vnet_buffer(b2)->sw_if_index[VLIB_TX];
sw_if_index3 = vnet_buffer(b3)->sw_if_index[VLIB_TX];
hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
hi1 = vnet_get_sup_hw_interface (vnm, sw_if_index1);
hi2 = vnet_get_sup_hw_interface (vnm, sw_if_index2);
hi3 = vnet_get_sup_hw_interface (vnm, sw_if_index3);
t0 = >m->tunnels[hi0->dev_instance];
t1 = >m->tunnels[hi1->dev_instance];
t2 = >m->tunnels[hi2->dev_instance];
t3 = >m->tunnels[hi3->dev_instance];
/* Note: change to always set next0 if it may be set to drop */
next0 = t0->next_dpo.dpoi_next_node;
vnet_buffer(b0)->ip.adj_index[VLIB_TX] = t0->next_dpo.dpoi_index;
next1 = t1->next_dpo.dpoi_next_node;
vnet_buffer(b1)->ip.adj_index[VLIB_TX] = t1->next_dpo.dpoi_index;
next2 = t2->next_dpo.dpoi_next_node;
vnet_buffer(b2)->ip.adj_index[VLIB_TX] = t2->next_dpo.dpoi_index;
next3 = t3->next_dpo.dpoi_next_node;
vnet_buffer(b3)->ip.adj_index[VLIB_TX] = t3->next_dpo.dpoi_index;
/* Apply the rewrite string. $$$$ vnet_rewrite? */
vlib_buffer_advance (b0, -(word)_vec_len(t0->rewrite));
vlib_buffer_advance (b1, -(word)_vec_len(t1->rewrite));
vlib_buffer_advance (b2, -(word)_vec_len(t2->rewrite));
vlib_buffer_advance (b3, -(word)_vec_len(t3->rewrite));
if (is_ip4)
{
ip4_0 = vlib_buffer_get_current(b0);
ip4_1 = vlib_buffer_get_current(b1);
ip4_2 = vlib_buffer_get_current(b2);
ip4_3 = vlib_buffer_get_current(b3);
/* Copy the fixed header */
copy_dst0 = (u64 *) ip4_0;
copy_src0 = (u64 *) t0->rewrite;
copy_dst1 = (u64 *) ip4_1;
copy_src1 = (u64 *) t1->rewrite;
copy_dst2 = (u64 *) ip4_2;
copy_src2 = (u64 *) t2->rewrite;
copy_dst3 = (u64 *) ip4_3;
copy_src3 = (u64 *) t3->rewrite;
/* Copy first 32 octets 8-bytes at a time */
#define _(offs) copy_dst0[offs] = copy_src0[offs];
foreach_fixed_header4_offset;
#undef _
#define _(offs) copy_dst1[offs] = copy_src1[offs];
foreach_fixed_header4_offset;
#undef _
#define _(offs) copy_dst2[offs] = copy_src2[offs];
foreach_fixed_header4_offset;
#undef _
#define _(offs) copy_dst3[offs] = copy_src3[offs];
foreach_fixed_header4_offset;
#undef _
/* Last 4 octets. Hopefully gcc will be our friend */
copy_dst_last0 = (u32 *)(©_dst0[4]);
copy_src_last0 = (u32 *)(©_src0[4]);
copy_dst_last0[0] = copy_src_last0[0];
copy_dst_last1 = (u32 *)(©_dst1[4]);
copy_src_last1 = (u32 *)(©_src1[4]);
copy_dst_last1[0] = copy_src_last1[0];
copy_dst_last2 = (u32 *)(©_dst2[4]);
copy_src_last2 = (u32 *)(©_src2[4]);
copy_dst_last2[0] = copy_src_last2[0];
copy_dst_last3 = (u32 *)(©_dst3[4]);
copy_src_last3 = (u32 *)(©_src3[4]);
copy_dst_last3[0] = copy_src_last3[0];
/* Fix the IP4 checksum and length */
sum0 = ip4_0->checksum;
new_l0 = /* old_l0 always 0, see the rewrite setup */
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0));
sum0 = ip_csum_update (sum0, old_l0, new_l0, ip4_header_t,
length /* changed member */);
ip4_0->checksum = ip_csum_fold (sum0);
ip4_0->length = new_l0;
sum1 = ip4_1->checksum;
new_l1 = /* old_l1 always 0, see the rewrite setup */
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b1));
sum1 = ip_csum_update (sum1, old_l1, new_l1, ip4_header_t,
length /* changed member */);
ip4_1->checksum = ip_csum_fold (sum1);
ip4_1->length = new_l1;
sum2 = ip4_2->checksum;
new_l2 = /* old_l0 always 0, see the rewrite setup */
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b2));
sum2 = ip_csum_update (sum2, old_l2, new_l2, ip4_header_t,
length /* changed member */);
ip4_2->checksum = ip_csum_fold (sum2);
ip4_2->length = new_l2;
sum3 = ip4_3->checksum;
new_l3 = /* old_l1 always 0, see the rewrite setup */
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b3));
sum3 = ip_csum_update (sum3, old_l3, new_l3, ip4_header_t,
length /* changed member */);
ip4_3->checksum = ip_csum_fold (sum3);
ip4_3->length = new_l3;
/* Fix UDP length and set source port */
udp0 = (udp_header_t *)(ip4_0+1);
new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
- sizeof (*ip4_0));
udp0->length = new_l0;
udp0->src_port = flow_hash0;
udp1 = (udp_header_t *)(ip4_1+1);
new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b1)
- sizeof (*ip4_1));
udp1->length = new_l1;
udp1->src_port = flow_hash1;
udp2 = (udp_header_t *)(ip4_2+1);
new_l2 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b2)
- sizeof (*ip4_2));
udp2->length = new_l2;
udp2->src_port = flow_hash2;
udp3 = (udp_header_t *)(ip4_3+1);
new_l3 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b3)
- sizeof (*ip4_3));
udp3->length = new_l3;
udp3->src_port = flow_hash3;
/* Fix GTPU length */
gtpu0 = (gtpu_header_t *)(udp0+1);
new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
- sizeof (*ip4_0) - sizeof(*udp0));
gtpu0->length = new_l0;
gtpu1 = (gtpu_header_t *)(udp1+1);
new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b1)
- sizeof (*ip4_1) - sizeof(*udp1));
gtpu1->length = new_l1;
gtpu2 = (gtpu_header_t *)(udp2+1);
new_l2 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b2)
- sizeof (*ip4_2) - sizeof(*udp2));
gtpu2->length = new_l2;
gtpu3 = (gtpu_header_t *)(udp1+3);
new_l3 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b3)
- sizeof (*ip4_3) - sizeof(*udp3));
gtpu3->length = new_l3;
}
else /* ipv6 */
{
int bogus = 0;
ip6_0 = vlib_buffer_get_current(b0);
ip6_1 = vlib_buffer_get_current(b1);
ip6_2 = vlib_buffer_get_current(b2);
ip6_3 = vlib_buffer_get_current(b3);
/* Copy the fixed header */
copy_dst0 = (u64 *) ip6_0;
copy_src0 = (u64 *) t0->rewrite;
copy_dst1 = (u64 *) ip6_1;
copy_src1 = (u64 *) t1->rewrite;
copy_dst2 = (u64 *) ip6_2;
copy_src2 = (u64 *) t2->rewrite;
copy_dst3 = (u64 *) ip6_3;
copy_src3 = (u64 *) t3->rewrite;
/* Copy first 56 (ip6) octets 8-bytes at a time */
#define _(offs) copy_dst0[offs] = copy_src0[offs];
foreach_fixed_header6_offset;
#undef _
#define _(offs) copy_dst1[offs] = copy_src1[offs];
foreach_fixed_header6_offset;
#undef _
#define _(offs) copy_dst2[offs] = copy_src2[offs];
foreach_fixed_header6_offset;
#undef _
#define _(offs) copy_dst3[offs] = copy_src3[offs];
foreach_fixed_header6_offset;
#undef _
/* Fix IP6 payload length */
new_l0 =
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0)
- sizeof(*ip6_0));
ip6_0->payload_length = new_l0;
new_l1 =
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b1)
- sizeof(*ip6_1));
ip6_1->payload_length = new_l1;
new_l2 =
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b2)
- sizeof(*ip6_2));
ip6_2->payload_length = new_l2;
new_l3 =
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b3)
- sizeof(*ip6_3));
ip6_3->payload_length = new_l3;
/* Fix UDP length and set source port */
udp0 = (udp_header_t *)(ip6_0+1);
udp0->length = new_l0;
udp0->src_port = flow_hash0;
udp1 = (udp_header_t *)(ip6_1+1);
udp1->length = new_l1;
udp1->src_port = flow_hash1;
udp2 = (udp_header_t *)(ip6_2+1);
udp2->length = new_l2;
udp2->src_port = flow_hash2;
udp3 = (udp_header_t *)(ip6_3+1);
udp3->length = new_l3;
udp3->src_port = flow_hash3;
/* IPv6 UDP checksum is mandatory */
udp0->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b0,
ip6_0, &bogus);
if (udp0->checksum == 0)
udp0->checksum = 0xffff;
udp1->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b1,
ip6_1, &bogus);
if (udp1->checksum == 0)
udp1->checksum = 0xffff;
udp2->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b2,
ip6_2, &bogus);
if (udp2->checksum == 0)
udp2->checksum = 0xffff;
udp3->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b3,
ip6_3, &bogus);
if (udp3->checksum == 0)
udp3->checksum = 0xffff;
/* Fix GTPU length */
gtpu0 = (gtpu_header_t *)(udp0+1);
new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
- sizeof (*ip4_0) - sizeof(*udp0));
gtpu0->length = new_l0;
gtpu1 = (gtpu_header_t *)(udp1+1);
new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b1)
- sizeof (*ip4_1) - sizeof(*udp1));
gtpu1->length = new_l1;
gtpu2 = (gtpu_header_t *)(udp2+1);
new_l2 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b2)
- sizeof (*ip4_2) - sizeof(*udp2));
gtpu2->length = new_l2;
gtpu3 = (gtpu_header_t *)(udp3+1);
new_l3 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b3)
- sizeof (*ip4_3) - sizeof(*udp3));
gtpu3->length = new_l3;
}
pkts_encapsulated += 4;
len0 = vlib_buffer_length_in_chain (vm, b0);
len1 = vlib_buffer_length_in_chain (vm, b1);
len2 = vlib_buffer_length_in_chain (vm, b2);
len3 = vlib_buffer_length_in_chain (vm, b3);
stats_n_packets += 4;
stats_n_bytes += len0 + len1 + len2 + len3;
/* Batch stats increment on the same gtpu tunnel so counter is not
incremented per packet. Note stats are still incremented for deleted
and admin-down tunnel where packets are dropped. It is not worthwhile
to check for this rare case and affect normal path performance. */
if (PREDICT_FALSE ((sw_if_index0 != stats_sw_if_index) ||
(sw_if_index1 != stats_sw_if_index) ||
(sw_if_index2 != stats_sw_if_index) ||
(sw_if_index3 != stats_sw_if_index) ))
{
stats_n_packets -= 4;
stats_n_bytes -= len0 + len1 + len2 + len3;
if ( (sw_if_index0 == sw_if_index1 ) &&
(sw_if_index1 == sw_if_index2 ) &&
(sw_if_index2 == sw_if_index3 ) )
{
if (stats_n_packets)
vlib_increment_combined_counter
(im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
thread_index, stats_sw_if_index,
stats_n_packets, stats_n_bytes);
stats_sw_if_index = sw_if_index0;
stats_n_packets = 4;
stats_n_bytes = len0 + len1 + len2 + len3;
}
else
{
vlib_increment_combined_counter
(im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
thread_index, sw_if_index0, 1, len0);
vlib_increment_combined_counter
(im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
thread_index, sw_if_index1, 1, len1);
vlib_increment_combined_counter
(im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
thread_index, sw_if_index2, 1, len2);
vlib_increment_combined_counter
(im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
thread_index, sw_if_index3, 1, len3);
}
}
if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
{
gtpu_encap_trace_t *tr =
vlib_add_trace (vm, node, b0, sizeof (*tr));
tr->tunnel_index = t0 - gtm->tunnels;
tr->teid = t0->teid;
}
if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
{
gtpu_encap_trace_t *tr =
vlib_add_trace (vm, node, b1, sizeof (*tr));
tr->tunnel_index = t1 - gtm->tunnels;
tr->teid = t1->teid;
}
vlib_validate_buffer_enqueue_x4 (vm, node, next_index,
to_next, n_left_to_next,
bi0, bi1, bi2, bi3,
next0, next1, next2, next3);
}
while (n_left_from > 0 && n_left_to_next > 0)
{
u32 bi0;
vlib_buffer_t * b0;
u32 flow_hash0;
u32 len0;
ip4_header_t * ip4_0;
ip6_header_t * ip6_0;
udp_header_t * udp0;
gtpu_header_t * gtpu0;
u64 * copy_src0, * copy_dst0;
u32 * copy_src_last0, * copy_dst_last0;
u16 new_l0;
ip_csum_t sum0;
bi0 = from[0];
to_next[0] = bi0;
from += 1;
to_next += 1;
n_left_from -= 1;
n_left_to_next -= 1;
b0 = vlib_get_buffer (vm, bi0);
flow_hash0 = vnet_l2_compute_flow_hash(b0);
/* Get next node index and adj index from tunnel next_dpo */
sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_TX];
hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
t0 = >m->tunnels[hi0->dev_instance];
/* Note: change to always set next0 if it may be set to drop */
next0 = t0->next_dpo.dpoi_next_node;
vnet_buffer(b0)->ip.adj_index[VLIB_TX] = t0->next_dpo.dpoi_index;
/* Apply the rewrite string. $$$$ vnet_rewrite? */
vlib_buffer_advance (b0, -(word)_vec_len(t0->rewrite));
if (is_ip4)
{
ip4_0 = vlib_buffer_get_current(b0);
/* Copy the fixed header */
copy_dst0 = (u64 *) ip4_0;
copy_src0 = (u64 *) t0->rewrite;
/* Copy first 32 octets 8-bytes at a time */
#define _(offs) copy_dst0[offs] = copy_src0[offs];
foreach_fixed_header4_offset;
#undef _
/* Last 4 octets. Hopefully gcc will be our friend */
copy_dst_last0 = (u32 *)(©_dst0[4]);
copy_src_last0 = (u32 *)(©_src0[4]);
copy_dst_last0[0] = copy_src_last0[0];
/* Fix the IP4 checksum and length */
sum0 = ip4_0->checksum;
new_l0 = /* old_l0 always 0, see the rewrite setup */
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0));
sum0 = ip_csum_update (sum0, old_l0, new_l0, ip4_header_t,
length /* changed member */);
ip4_0->checksum = ip_csum_fold (sum0);
ip4_0->length = new_l0;
/* Fix UDP length and set source port */
udp0 = (udp_header_t *)(ip4_0+1);
new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
- sizeof (*ip4_0));
udp0->length = new_l0;
udp0->src_port = flow_hash0;
/* Fix GTPU length */
gtpu0 = (gtpu_header_t *)(udp0+1);
new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
- sizeof (*ip4_0) - sizeof(*udp0));
gtpu0->length = new_l0;
}
else /* ip6 path */
{
int bogus = 0;
ip6_0 = vlib_buffer_get_current(b0);
/* Copy the fixed header */
copy_dst0 = (u64 *) ip6_0;
copy_src0 = (u64 *) t0->rewrite;
/* Copy first 56 (ip6) octets 8-bytes at a time */
#define _(offs) copy_dst0[offs] = copy_src0[offs];
foreach_fixed_header6_offset;
#undef _
/* Fix IP6 payload length */
new_l0 =
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0)
- sizeof(*ip6_0));
ip6_0->payload_length = new_l0;
/* Fix UDP length and set source port */
udp0 = (udp_header_t *)(ip6_0+1);
udp0->length = new_l0;
udp0->src_port = flow_hash0;
/* IPv6 UDP checksum is mandatory */
udp0->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b0,
ip6_0, &bogus);
if (udp0->checksum == 0)
udp0->checksum = 0xffff;
/* Fix GTPU length */
gtpu0 = (gtpu_header_t *)(udp0+1);
new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
- sizeof (*ip4_0) - sizeof(*udp0));
gtpu0->length = new_l0;
}
pkts_encapsulated ++;
len0 = vlib_buffer_length_in_chain (vm, b0);
stats_n_packets += 1;
stats_n_bytes += len0;
/* Batch stats increment on the same gtpu tunnel so counter is not
incremented per packet. Note stats are still incremented for deleted
and admin-down tunnel where packets are dropped. It is not worthwhile
to check for this rare case and affect normal path performance. */
if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
{
stats_n_packets -= 1;
stats_n_bytes -= len0;
if (stats_n_packets)
vlib_increment_combined_counter
(im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
thread_index, stats_sw_if_index,
stats_n_packets, stats_n_bytes);
stats_n_packets = 1;
stats_n_bytes = len0;
stats_sw_if_index = sw_if_index0;
}
if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
{
gtpu_encap_trace_t *tr =
vlib_add_trace (vm, node, b0, sizeof (*tr));
tr->tunnel_index = t0 - gtm->tunnels;
tr->teid = t0->teid;
}
vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
to_next, n_left_to_next,
bi0, next0);
}
vlib_put_next_frame (vm, node, next_index, n_left_to_next);
}
/* Do we still need this now that tunnel tx stats is kept? */
vlib_node_increment_counter (vm, node->node_index,
GTPU_ENCAP_ERROR_ENCAPSULATED,
pkts_encapsulated);
/* Increment any remaining batch stats */
if (stats_n_packets)
{
vlib_increment_combined_counter
(im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
node->runtime_data[0] = stats_sw_if_index;
}
return from_frame->n_vectors;
}
static uword
gtpu4_encap (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * from_frame)
{
return gtpu_encap_inline (vm, node, from_frame, /* is_ip4 */ 1);
}
static uword
gtpu6_encap (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * from_frame)
{
return gtpu_encap_inline (vm, node, from_frame, /* is_ip4 */ 0);
}
VLIB_REGISTER_NODE (gtpu4_encap_node) = {
.function = gtpu4_encap,
.name = "gtpu4-encap",
.vector_size = sizeof (u32),
.format_trace = format_gtpu_encap_trace,
.type = VLIB_NODE_TYPE_INTERNAL,
.n_errors = ARRAY_LEN(gtpu_encap_error_strings),
.error_strings = gtpu_encap_error_strings,
.n_next_nodes = GTPU_ENCAP_N_NEXT,
.next_nodes = {
#define _(s,n) [GTPU_ENCAP_NEXT_##s] = n,
foreach_gtpu_encap_next
#undef _
},
};
VLIB_NODE_FUNCTION_MULTIARCH (gtpu4_encap_node, gtpu4_encap)
VLIB_REGISTER_NODE (gtpu6_encap_node) = {
.function = gtpu6_encap,
.name = "gtpu6-encap",
.vector_size = sizeof (u32),
.format_trace = format_gtpu_encap_trace,
.type = VLIB_NODE_TYPE_INTERNAL,
.n_errors = ARRAY_LEN(gtpu_encap_error_strings),
.error_strings = gtpu_encap_error_strings,
.n_next_nodes = GTPU_ENCAP_N_NEXT,
.next_nodes = {
#define _(s,n) [GTPU_ENCAP_NEXT_##s] = n,
foreach_gtpu_encap_next
#undef _
},
};
VLIB_NODE_FUNCTION_MULTIARCH (gtpu6_encap_node, gtpu6_encap)
|
