summaryrefslogtreecommitdiffstats
path: root/src/vnet
diff options
context:
space:
mode:
authorDave Barach <dave@barachs.net>2017-06-29 09:30:15 -0400
committerFlorin Coras <florin.coras@gmail.com>2017-07-18 17:05:01 +0000
commit2c0a4f407f565d8dd33ff3a9fada346860d30ad2 (patch)
treefeef8e57bc4f7b434715bd04c9be218ce215bdd0 /src/vnet
parent758137a7b70806dc9ec7a28fbe1bc01bc7df2586 (diff)
TCP/UDP checksum offload API
Change-Id: I2cb6ce4e29813f6602b14e6e61713fb381fbcef8 Signed-off-by: Dave Barach <dave@barachs.net>
Diffstat (limited to 'src/vnet')
-rw-r--r--src/vnet/buffer.h7
-rw-r--r--src/vnet/interface.h3
-rw-r--r--src/vnet/interface_output.c95
-rw-r--r--src/vnet/ip/ip4.h4
-rwxr-xr-xsrc/vnet/ip/ip4_forward.c9
-rw-r--r--src/vnet/tcp/tcp_output.c16
6 files changed, 121 insertions, 13 deletions
diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h
index 52dada30ba8..56cb07ae81e 100644
--- a/src/vnet/buffer.h
+++ b/src/vnet/buffer.h
@@ -47,9 +47,14 @@
_( 2, L4_CHECKSUM_CORRECT) \
_( 3, VLAN_2_DEEP) \
_( 4, VLAN_1_DEEP) \
+ _( 8, SPAN_CLONE) \
_( 6, HANDOFF_NEXT_VALID) \
_( 7, LOCALLY_ORIGINATED) \
- _( 8, SPAN_CLONE)
+ _( 8, IS_IP4) \
+ _( 9, IS_IP6) \
+ _(10, OFFLOAD_IP_CKSUM) \
+ _(11, OFFLOAD_TCP_CKSUM) \
+ _(12, OFFLOAD_UDP_CKSUM)
#define VNET_BUFFER_FLAGS_VLAN_BITS \
(VNET_BUFFER_F_VLAN_1_DEEP | VNET_BUFFER_F_VLAN_2_DEEP)
diff --git a/src/vnet/interface.h b/src/vnet/interface.h
index 9d64fc28d91..fb75ff349b9 100644
--- a/src/vnet/interface.h
+++ b/src/vnet/interface.h
@@ -419,6 +419,9 @@ typedef struct vnet_hw_interface_t
/* rx mode flags */
#define VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE (1 << 10)
+ /* tx checksum offload */
+#define VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD (1 << 11)
+
/* Hardware address as vector. Zero (e.g. zero-length vector) if no
address for this class (e.g. PPP). */
u8 *hw_address;
diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c
index cdf18738193..c95f08b1a1e 100644
--- a/src/vnet/interface_output.c
+++ b/src/vnet/interface_output.c
@@ -38,6 +38,10 @@
*/
#include <vnet/vnet.h>
+#include <vnet/ip/icmp46_packet.h>
+#include <vnet/ip/ip4.h>
+#include <vnet/ip/ip6.h>
+#include <vnet/udp/udp_packet.h>
#include <vnet/feature/feature.h>
typedef struct
@@ -153,14 +157,58 @@ vnet_interface_output_trace (vlib_main_t * vm,
}
}
-uword
-vnet_interface_output_node (vlib_main_t * vm,
- vlib_node_runtime_t * node, vlib_frame_t * frame)
+static_always_inline void
+calc_checksums (vlib_main_t * vm, vlib_buffer_t * b)
+{
+ ip4_header_t *ip4;
+ ip6_header_t *ip6;
+ tcp_header_t *th;
+ udp_header_t *uh;
+
+ int is_ip4 = (b->flags & VNET_BUFFER_F_IS_IP4) != 0;
+ int is_ip6 = (b->flags & VNET_BUFFER_F_IS_IP6) != 0;
+
+ ASSERT (!(is_ip4 && is_ip6));
+
+ ip4 = (ip4_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
+ ip6 = (ip6_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
+ th = (tcp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
+ uh = (udp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
+
+ if (is_ip4)
+ {
+ ip4 = (ip4_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
+ if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ ip4->checksum = ip4_header_checksum (ip4);
+ if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
+ if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ uh->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
+ }
+ if (is_ip6)
+ {
+ int bogus;
+ ASSERT (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
+ if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
+ if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ uh->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
+ }
+
+ b->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+}
+
+static_always_inline uword
+vnet_interface_output_node_inline (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame, vnet_main_t * vnm,
+ vnet_hw_interface_t * hi,
+ int do_tx_offloads)
{
- vnet_main_t *vnm = vnet_get_main ();
vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
vnet_sw_interface_t *si;
- vnet_hw_interface_t *hi;
u32 n_left_to_tx, *from, *from_end, *to_tx;
u32 n_bytes, n_buffers, n_packets;
u32 n_bytes_b0, n_bytes_b1, n_bytes_b2, n_bytes_b3;
@@ -234,6 +282,7 @@ vnet_interface_output_node (vlib_main_t * vm,
u32 bi0, bi1, bi2, bi3;
vlib_buffer_t *b0, *b1, *b2, *b3;
u32 tx_swif0, tx_swif1, tx_swif2, tx_swif3;
+ u32 or_flags;
/* Prefetch next iteration. */
vlib_prefetch_buffer_with_index (vm, from[4], LOAD);
@@ -324,6 +373,22 @@ vnet_interface_output_node (vlib_main_t * vm,
thread_index, tx_swif3, 1,
n_bytes_b3);
}
+
+ or_flags = b0->flags | b1->flags | b2->flags | b3->flags;
+
+ if (do_tx_offloads)
+ {
+ if (or_flags &
+ (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
+ VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
+ VNET_BUFFER_F_OFFLOAD_IP_CKSUM))
+ {
+ calc_checksums (vm, b0);
+ calc_checksums (vm, b1);
+ calc_checksums (vm, b2);
+ calc_checksums (vm, b3);
+ }
+ }
}
while (from + 1 <= from_end && n_left_to_tx >= 1)
@@ -363,6 +428,9 @@ vnet_interface_output_node (vlib_main_t * vm,
thread_index, tx_swif0, 1,
n_bytes_b0);
}
+
+ if (do_tx_offloads)
+ calc_checksums (vm, b0);
}
vlib_put_next_frame (vm, node, next_index, n_left_to_tx);
@@ -376,6 +444,23 @@ vnet_interface_output_node (vlib_main_t * vm,
return n_buffers;
}
+static_always_inline uword
+vnet_interface_output_node (vlib_main_t * vm, vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
+{
+ vnet_main_t *vnm = vnet_get_main ();
+ vnet_hw_interface_t *hi;
+ vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
+ hi = vnet_get_sup_hw_interface (vnm, rt->sw_if_index);
+
+ if (hi->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD)
+ return vnet_interface_output_node_inline (vm, node, frame, vnm, hi,
+ /* do_tx_offloads */ 0);
+ else
+ return vnet_interface_output_node_inline (vm, node, frame, vnm, hi,
+ /* do_tx_offloads */ 1);
+}
+
VLIB_NODE_FUNCTION_MULTIARCH_CLONE (vnet_interface_output_node);
CLIB_MULTIARCH_SELECT_FN (vnet_interface_output_node);
diff --git a/src/vnet/ip/ip4.h b/src/vnet/ip/ip4.h
index 71640defb21..19198bcb66b 100644
--- a/src/vnet/ip/ip4.h
+++ b/src/vnet/ip/ip4.h
@@ -42,6 +42,7 @@
#include <vnet/ip/ip4_packet.h>
#include <vnet/ip/lookup.h>
+#include <vnet/buffer.h>
#include <vnet/feature/feature.h>
typedef struct ip4_mfib_t
@@ -346,7 +347,8 @@ vlib_buffer_push_ip4 (vlib_main_t * vm, vlib_buffer_t * b,
ih->src_address.as_u32 = src->as_u32;
ih->dst_address.as_u32 = dst->as_u32;
- ih->checksum = ip4_header_checksum (ih);
+ ih->checksum = 0;
+ b->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_IS_IP4;
return ih;
}
#endif /* included_ip_ip4_h */
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index 8dd927d4afd..4a9e7919140 100755
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -2457,7 +2457,8 @@ ip4_rewrite_inline (vlib_main_t * vm,
}
/* Verify checksum. */
- ASSERT (ip0->checksum == ip4_header_checksum (ip0));
+ ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
+ (p0->flags | VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
}
else
{
@@ -2492,7 +2493,8 @@ ip4_rewrite_inline (vlib_main_t * vm,
}
/* Verify checksum. */
- ASSERT (ip1->checksum == ip4_header_checksum (ip1));
+ ASSERT ((ip1->checksum == ip4_header_checksum (ip1)) ||
+ (p1->flags | VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
}
else
{
@@ -2630,7 +2632,8 @@ ip4_rewrite_inline (vlib_main_t * vm,
ip0->ttl = ttl0;
- ASSERT (ip0->checksum == ip4_header_checksum (ip0));
+ ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
+ (p0->flags | VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
if (PREDICT_FALSE (ttl0 <= 0))
{
diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index f34eb797c77..35f3eba15e0 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -1485,7 +1485,12 @@ tcp46_output_inline (vlib_main_t * vm,
ip4_header_t *ih0;
ih0 = vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4,
&tc0->c_rmt_ip4, IP_PROTOCOL_TCP);
- th0->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ih0);
+ b0->flags |= VNET_BUFFER_F_IS_IP4 |
+ VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
+ VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data;
+ vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
+ th0->checksum = 0;
}
else
{
@@ -1494,8 +1499,13 @@ tcp46_output_inline (vlib_main_t * vm,
ih0 = vlib_buffer_push_ip6 (vm, b0, &tc0->c_lcl_ip6,
&tc0->c_rmt_ip6, IP_PROTOCOL_TCP);
- th0->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b0, ih0,
- &bogus);
+
+ b0->flags |= VNET_BUFFER_F_IS_IP6 |
+ VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
+ VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data;
+ vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
+ th0->checksum = 0;
ASSERT (!bogus);
}