summaryrefslogtreecommitdiffstats
path: root/src/plugins/dpdk/device/node.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/plugins/dpdk/device/node.c')
-rw-r--r--src/plugins/dpdk/device/node.c113
1 files changed, 57 insertions, 56 deletions
diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c
index 1240b8d01a7..4863390ec2e 100644
--- a/src/plugins/dpdk/device/node.c
+++ b/src/plugins/dpdk/device/node.c
@@ -36,40 +36,21 @@ static char *dpdk_error_strings[] = {
};
#endif
-always_inline int
-vlib_buffer_is_ip4 (vlib_buffer_t * b)
-{
- ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
- return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4));
-}
-
-always_inline int
-vlib_buffer_is_ip6 (vlib_buffer_t * b)
-{
- ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
- return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6));
-}
-
-always_inline int
-vlib_buffer_is_mpls (vlib_buffer_t * b)
-{
- ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
- return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS));
-}
-
always_inline u32
-dpdk_rx_next_from_etype (struct rte_mbuf * mb, vlib_buffer_t * b0)
+dpdk_rx_next_from_etype (struct rte_mbuf *mb)
{
- if (PREDICT_TRUE (vlib_buffer_is_ip4 (b0)))
+ ethernet_header_t *h = rte_pktmbuf_mtod (mb, ethernet_header_t *);
+ if (PREDICT_TRUE (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4)))
{
if (PREDICT_TRUE ((mb->ol_flags & PKT_RX_IP_CKSUM_GOOD) != 0))
return VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT;
else
return VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
}
- else if (PREDICT_TRUE (vlib_buffer_is_ip6 (b0)))
+ else if (PREDICT_TRUE (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)))
return VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
- else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0)))
+ else
+ if (PREDICT_TRUE (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)))
return VNET_DEVICE_INPUT_NEXT_MPLS_INPUT;
else
return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
@@ -78,6 +59,7 @@ dpdk_rx_next_from_etype (struct rte_mbuf * mb, vlib_buffer_t * b0)
always_inline u32
dpdk_rx_next_from_packet_start (struct rte_mbuf * mb, vlib_buffer_t * b0)
{
+ ethernet_header_t *h = rte_pktmbuf_mtod (mb, ethernet_header_t *);
word start_delta;
int rv;
@@ -86,16 +68,17 @@ dpdk_rx_next_from_packet_start (struct rte_mbuf * mb, vlib_buffer_t * b0)
vlib_buffer_advance (b0, -start_delta);
- if (PREDICT_TRUE (vlib_buffer_is_ip4 (b0)))
+ if (PREDICT_TRUE (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4)))
{
if (PREDICT_TRUE ((mb->ol_flags & PKT_RX_IP_CKSUM_GOOD) != 0))
rv = VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT;
else
rv = VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
}
- else if (PREDICT_TRUE (vlib_buffer_is_ip6 (b0)))
+ else if (PREDICT_TRUE (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)))
rv = VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
- else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0)))
+ else
+ if (PREDICT_TRUE (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)))
rv = VNET_DEVICE_INPUT_NEXT_MPLS_INPUT;
else
rv = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
@@ -326,6 +309,7 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
u32 bi2, next2;
u32 bi3, next3;
u8 error0, error1, error2, error3;
+ i16 offset0, offset1, offset2, offset3;
u64 or_ol_flags;
vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
@@ -366,21 +350,10 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
b2 = vlib_buffer_from_rte_mbuf (mb2);
b3 = vlib_buffer_from_rte_mbuf (mb3);
- clib_memcpy64_x4 (b0, b1, b2, b3, bt);
-
dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 9]);
dpdk_prefetch_ethertype (xd->rx_vectors[queue_id][mb_index + 5]);
- /* current_data must be set to -RTE_PKTMBUF_HEADROOM in template */
- b0->current_data += mb0->data_off;
- b1->current_data += mb1->data_off;
- b2->current_data += mb2->data_off;
- b3->current_data += mb3->data_off;
-
- b0->current_length = mb0->data_len;
- b1->current_length = mb1->data_len;
- b2->current_length = mb2->data_len;
- b3->current_length = mb3->data_len;
+ clib_memcpy64_x4 (b0, b1, b2, b3, bt);
dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 10]);
dpdk_prefetch_ethertype (xd->rx_vectors[queue_id][mb_index + 7]);
@@ -403,10 +376,10 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
}
else
{
- next0 = dpdk_rx_next_from_etype (mb0, b0);
- next1 = dpdk_rx_next_from_etype (mb1, b1);
- next2 = dpdk_rx_next_from_etype (mb2, b2);
- next3 = dpdk_rx_next_from_etype (mb3, b3);
+ next0 = dpdk_rx_next_from_etype (mb0);
+ next1 = dpdk_rx_next_from_etype (mb1);
+ next2 = dpdk_rx_next_from_etype (mb2);
+ next3 = dpdk_rx_next_from_etype (mb3);
}
dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 11]);
@@ -426,16 +399,43 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
b3->error = node->errors[error3];
}
- vlib_buffer_advance (b0, device_input_next_node_advance[next0]);
- vlib_buffer_advance (b1, device_input_next_node_advance[next1]);
- vlib_buffer_advance (b2, device_input_next_node_advance[next2]);
- vlib_buffer_advance (b3, device_input_next_node_advance[next3]);
-
+ offset0 = device_input_next_node_advance[next0];
+ b0->current_data = mb0->data_off + offset0 - RTE_PKTMBUF_HEADROOM;
+ b0->flags |= device_input_next_node_flags[next0];
+ vnet_buffer (b0)->l3_hdr_offset = b0->current_data;
+ vnet_buffer (b0)->l2_hdr_offset =
+ mb0->data_off - RTE_PKTMBUF_HEADROOM;
+ b0->current_length = mb0->data_len - offset0;
n_rx_bytes += mb0->pkt_len;
+
+ offset1 = device_input_next_node_advance[next1];
+ b1->current_data = mb1->data_off + offset1 - RTE_PKTMBUF_HEADROOM;
+ b1->flags |= device_input_next_node_flags[next1];
+ vnet_buffer (b1)->l3_hdr_offset = b1->current_data;
+ vnet_buffer (b1)->l2_hdr_offset =
+ mb1->data_off - RTE_PKTMBUF_HEADROOM;
+ b1->current_length = mb1->data_len - offset1;
n_rx_bytes += mb1->pkt_len;
+
+ offset2 = device_input_next_node_advance[next2];
+ b2->current_data = mb2->data_off + offset2 - RTE_PKTMBUF_HEADROOM;
+ b2->flags |= device_input_next_node_flags[next2];
+ vnet_buffer (b2)->l3_hdr_offset = b2->current_data;
+ vnet_buffer (b2)->l2_hdr_offset =
+ mb2->data_off - RTE_PKTMBUF_HEADROOM;
+ b2->current_length = mb2->data_len - offset2;
n_rx_bytes += mb2->pkt_len;
+
+ offset3 = device_input_next_node_advance[next3];
+ b3->current_data = mb3->data_off + offset3 - RTE_PKTMBUF_HEADROOM;
+ b3->flags |= device_input_next_node_flags[next3];
+ vnet_buffer (b3)->l3_hdr_offset = b3->current_data;
+ vnet_buffer (b3)->l2_hdr_offset =
+ mb3->data_off - RTE_PKTMBUF_HEADROOM;
+ b3->current_length = mb3->data_len - offset3;
n_rx_bytes += mb3->pkt_len;
+
/* Process subsequent segments of multi-segment packets */
if (maybe_multiseg)
{
@@ -488,10 +488,6 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
clib_memcpy (b0, bt, CLIB_CACHE_LINE_BYTES);
- ASSERT (b0->current_data == -RTE_PKTMBUF_HEADROOM);
- b0->current_data += mb0->data_off;
- b0->current_length = mb0->data_len;
-
bi0 = vlib_get_buffer_index (vm, b0);
to_next[0] = bi0;
@@ -501,13 +497,18 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
if (PREDICT_FALSE (xd->per_interface_next_index != ~0))
next0 = xd->per_interface_next_index;
else
- next0 = dpdk_rx_next_from_etype (mb0, b0);
+ next0 = dpdk_rx_next_from_etype (mb0);
dpdk_rx_error_from_mb (mb0, &next0, &error0);
b0->error = node->errors[error0];
- vlib_buffer_advance (b0, device_input_next_node_advance[next0]);
-
+ offset0 = device_input_next_node_advance[next0];
+ b0->current_data = mb0->data_off + offset0 - RTE_PKTMBUF_HEADROOM;
+ b0->flags |= device_input_next_node_flags[next0];
+ vnet_buffer (b0)->l3_hdr_offset = b0->current_data;
+ vnet_buffer (b0)->l2_hdr_offset =
+ mb0->data_off - RTE_PKTMBUF_HEADROOM;
+ b0->current_length = mb0->data_len - offset0;
n_rx_bytes += mb0->pkt_len;
/* Process subsequent segments of multi-segment packets */