aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMichal Mazur <mkm@semihalf.com>2017-11-26 19:28:55 +0100
committerMichal Mazur <mkm@semihalf.com>2017-12-18 18:21:00 +0100
commit945f7e8ebe39ffa80dfb38b950406a7dd4935f1f (patch)
tree815c85288b34ccb3058d8b2b46a65bff846f256d
parent4c34d6c16947c5e1465319d9d44dbdaaa2f04c4e (diff)
More optimizations for Tx and Rx loops
1) Parse multiple output packets in loop. 2) Remove checking for error flags, they are never set because ODP parser is disabled. 3) Do not call vlib_buffer_advance if not necessary. 4) Remove lock in TX path. Queues are synchronized by ODP. 5) Optimize detection of packet type. 6) Create a wrapper for call to odp_packet_user_area. Change-Id: Ib50c9be9a62d67824b58ef2cd443b7fc59471655 Signed-off-by: Michal Mazur <mkm@semihalf.com>
-rw-r--r--src/plugins/odp/buffer.c5
-rwxr-xr-xsrc/plugins/odp/device.c137
-rwxr-xr-xsrc/plugins/odp/node.c144
-rwxr-xr-xsrc/plugins/odp/odp_packet.c43
-rwxr-xr-xsrc/plugins/odp/odp_packet.h33
5 files changed, 195 insertions, 167 deletions
diff --git a/src/plugins/odp/buffer.c b/src/plugins/odp/buffer.c
index 7c410c2..b3c3869 100644
--- a/src/plugins/odp/buffer.c
+++ b/src/plugins/odp/buffer.c
@@ -28,8 +28,9 @@ odp_packet_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
if (pkt == ODP_PACKET_INVALID)
break;
- buffers[total] = vlib_get_buffer_index (vm, odp_packet_user_area (pkt));
- ((vlib_buffer_t *)odp_packet_user_area (pkt))->l2_priv_data = (void *)pkt;
+ buffers[total] =
+ vlib_get_buffer_index (vm, vlib_buffer_from_odp_packet (pkt));
+ ((vlib_buffer_t *) odp_packet_user_area (pkt))->l2_priv_data = pkt;
}
while (++total < n_buffers);
diff --git a/src/plugins/odp/device.c b/src/plugins/odp/device.c
index eebc2aa..4f80983 100755
--- a/src/plugins/odp/device.c
+++ b/src/plugins/odp/device.c
@@ -67,6 +67,15 @@ format_odp_packet_tx_trace (u8 * s, va_list * args)
return s;
}
+#define NEXT_BUFFER(b0, bi, j) \
+ if (b0->flags & VLIB_BUFFER_NEXT_PRESENT) \
+ bi[j++] = b0->next_buffer; \
+ else if (n_left) \
+ { \
+ bi[j++] = *buffers++; \
+ n_left--; \
+ }
+
static uword
odp_packet_interface_tx (vlib_main_t * vm,
vlib_node_runtime_t * node, vlib_frame_t * frame)
@@ -80,53 +89,88 @@ odp_packet_interface_tx (vlib_main_t * vm,
uword queue_index = vlib_get_thread_index () % oif->m.num_tx_queues;
u32 mode = oif->m.tx_mode;
u32 burst_size = (tx_burst_size ? tx_burst_size : VLIB_FRAME_SIZE);
- odp_packet_t pkt_tbl[burst_size];
- odp_event_t evt_tbl[burst_size];
- vlib_buffer_t *b0;
- u32 bi, sent, count = 0;
-
- if (PREDICT_FALSE (oif->lockp != 0))
+ union
+ {
+ odp_packet_t pkt[burst_size];
+ odp_event_t evt[burst_size];
+ } tbl;
+ vlib_buffer_t *b0, *b1, *b2, *b3;
+ u32 bi[4];
+ u32 sent, count = 0, todo = 0;
+
+ while (n_left > 0 || todo > 0)
{
- while (__sync_lock_test_and_set (oif->lockp, 1))
- ;
- }
+ int ret;
- while (n_left > 0)
- {
- odp_packet_t pkt;
- int ret, diff;
+ for (; todo < 4 && todo < n_left; todo++, n_left--)
+ bi[todo] = *buffers++;
+
+ while ((todo == 4) && (count + 3 < burst_size))
+ {
+ odp_packet_t pkt0, pkt1, pkt2, pkt3;
+
+ b0 = vlib_get_buffer (vm, bi[0]);
+ b1 = vlib_get_buffer (vm, bi[1]);
+ b2 = vlib_get_buffer (vm, bi[2]);
+ b3 = vlib_get_buffer (vm, bi[3]);
- bi = buffers[0];
- n_left--;
- buffers++;
+ pkt0 = odp_packet_from_vlib_buffer (b0);
+ pkt1 = odp_packet_from_vlib_buffer (b1);
+ pkt2 = odp_packet_from_vlib_buffer (b2);
+ pkt3 = odp_packet_from_vlib_buffer (b3);
- next_present:
- do
+ odp_adjust_data_pointers (b0, pkt0);
+ odp_adjust_data_pointers (b1, pkt1);
+ odp_adjust_data_pointers (b2, pkt2);
+ odp_adjust_data_pointers (b3, pkt3);
+
+ if (mode == APPL_MODE_PKT_QUEUE)
+ {
+ tbl.evt[count++] = odp_packet_to_event (pkt0);
+ tbl.evt[count++] = odp_packet_to_event (pkt1);
+ tbl.evt[count++] = odp_packet_to_event (pkt2);
+ tbl.evt[count++] = odp_packet_to_event (pkt3);
+ }
+ else
+ {
+ tbl.pkt[count++] = pkt0;
+ tbl.pkt[count++] = pkt1;
+ tbl.pkt[count++] = pkt2;
+ tbl.pkt[count++] = pkt3;
+ }
+
+ todo = 0;
+ NEXT_BUFFER (b0, bi, todo);
+ NEXT_BUFFER (b1, bi, todo);
+ NEXT_BUFFER (b2, bi, todo);
+ NEXT_BUFFER (b3, bi, todo);
+ }
+
+ while (todo && (count < burst_size))
{
- b0 = vlib_get_buffer (vm, bi);
+ odp_packet_t pkt;
+
+ b0 = vlib_get_buffer (vm, bi[todo - 1]);
+
pkt = odp_packet_from_vlib_buffer (b0);
- diff = (uintptr_t) (b0->data + b0->current_data) -
- (uintptr_t) odp_packet_data (pkt);
- if (diff > 0)
- odp_packet_pull_head (pkt, diff);
- else if (diff < 0)
- odp_packet_push_head (pkt, -diff);
- diff = b0->current_length - odp_packet_len (pkt);
- if (diff > 0)
- odp_packet_push_tail (pkt, diff);
- else if (diff < 0)
- odp_packet_pull_tail (pkt, -diff);
- pkt_tbl[count] = pkt;
+ odp_adjust_data_pointers (b0, pkt);
+
if (mode == APPL_MODE_PKT_QUEUE)
- evt_tbl[count] = odp_packet_to_event (pkt_tbl[count]);
- count++;
- bi = b0->next_buffer;
- }
- while ((b0->flags & VLIB_BUFFER_NEXT_PRESENT) && (count < burst_size));
+ tbl.evt[count++] = odp_packet_to_event (pkt);
+ else
+ tbl.pkt[count++] = pkt;
- if ((n_left > 0) && (count < burst_size))
- continue;
+ if (b0->flags & VLIB_BUFFER_NEXT_PRESENT)
+ bi[todo - 1] = b0->next_buffer;
+ else if (n_left)
+ {
+ bi[todo - 1] = *buffers++;
+ n_left--;
+ }
+ else
+ todo--;
+ }
sent = 0;
while (count > 0)
@@ -134,23 +178,23 @@ odp_packet_interface_tx (vlib_main_t * vm,
switch (mode)
{
case APPL_MODE_PKT_BURST:
- ret =
- odp_pktout_send (oif->outq[queue_index], &pkt_tbl[sent],
- count);
+ ret = odp_pktout_send (oif->outq[queue_index], &tbl.pkt[sent],
+ count);
break;
case APPL_MODE_PKT_QUEUE:
ret = odp_queue_enq_multi (oif->txq[queue_index],
- &evt_tbl[sent], count);
+ &tbl.evt[sent], count);
break;
case APPL_MODE_PKT_TM:
default:
ret = 0;
clib_error ("Invalid mode\n");
}
- if (odp_unlikely (ret <= 0))
+
+ if (odp_unlikely (ret < 0))
{
/* Drop one packet and try again */
- odp_packet_free (pkt_tbl[sent]);
+ odp_packet_free (tbl.pkt[sent]);
count--;
sent++;
}
@@ -160,13 +204,8 @@ odp_packet_interface_tx (vlib_main_t * vm,
sent += ret;
}
}
- if (b0->flags & VLIB_BUFFER_NEXT_PRESENT)
- goto next_present;
}
- if (PREDICT_FALSE (oif->lockp != 0))
- *oif->lockp = 0;
-
return (frame->n_vectors - n_left);
}
diff --git a/src/plugins/odp/node.c b/src/plugins/odp/node.c
index 1a8b04c..8eb1b0c 100755
--- a/src/plugins/odp/node.c
+++ b/src/plugins/odp/node.c
@@ -51,7 +51,7 @@ format_odp_packet_input_trace (u8 * s, va_list * args)
static_always_inline void
odp_prefetch_buffer (odp_packet_t pkt)
{
- vlib_buffer_t *b = (vlib_buffer_t *) odp_packet_user_area (pkt);
+ vlib_buffer_t *b = vlib_buffer_from_odp_packet (pkt);
CLIB_PREFETCH (pkt, CLIB_CACHE_LINE_BYTES, LOAD);
CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, STORE);
}
@@ -59,7 +59,7 @@ odp_prefetch_buffer (odp_packet_t pkt)
static_always_inline void
odp_prefetch_ethertype (odp_packet_t pkt)
{
- vlib_buffer_t *b = (vlib_buffer_t *) odp_packet_user_area (pkt);
+ vlib_buffer_t *b = vlib_buffer_from_odp_packet (pkt);
CLIB_PREFETCH (vlib_buffer_get_current (b) +
STRUCT_OFFSET_OF (ethernet_header_t, type),
CLIB_CACHE_LINE_BYTES, LOAD);
@@ -74,25 +74,28 @@ odp_packet_queue_mode (odp_packet_if_t * oif, odp_packet_t pkt_tbl[],
odp_pktio_t pktio = oif->pktio;
odp_event_t evt_tbl[req_pkts];
u64 sched_wait;
+ odp_queue_t rxq = ODP_QUEUE_INVALID;
if (pktio == ODP_PKTIO_INVALID)
{
clib_warning ("invalid oif->pktio value");
return 0;
}
- if ((oif->m.rx_mode == APPL_MODE_PKT_QUEUE) &&
- (oif->rxq[queue_id] == ODP_QUEUE_INVALID))
+ if (oif->m.rx_mode == APPL_MODE_PKT_QUEUE)
{
- clib_warning ("invalid rxq[%d] queue", queue_id);
- return 0;
+ rxq = oif->rxq[queue_id];
+ if (rxq == ODP_QUEUE_INVALID)
+ {
+ clib_warning ("invalid rxq[%d] queue", queue_id);
+ return 0;
+ }
}
while (req_pkts)
{
if (oif->m.rx_mode == APPL_MODE_PKT_QUEUE)
{
- i = odp_queue_deq_multi (oif->rxq[queue_id],
- &evt_tbl[num_evts], req_pkts);
+ i = odp_queue_deq_multi (rxq, &evt_tbl[num_evts], req_pkts);
}
else
{
@@ -139,38 +142,20 @@ odp_packet_burst_mode (odp_packet_if_t * oif, odp_packet_t pkt_tbl[],
return num_pkts;
}
-always_inline int
-vlib_buffer_is_ip4 (vlib_buffer_t * b)
-{
- ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
- return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4));
-}
-
-always_inline int
-vlib_buffer_is_ip6 (vlib_buffer_t * b)
-{
- ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
- return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6));
-}
-
-always_inline int
-vlib_buffer_is_mpls (vlib_buffer_t * b)
-{
- ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
- return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS));
-}
-
always_inline u32
odp_rx_next_from_etype (void *mb, vlib_buffer_t * b0)
{
- if (PREDICT_TRUE (vlib_buffer_is_ip4 (b0)))
+ ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b0);
+ u16 type = h->type;
+
+ if (type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4))
return VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
- else if (PREDICT_TRUE (vlib_buffer_is_ip6 (b0)))
+ if (type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6))
return VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
- else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0)))
+ if (type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
return VNET_DEVICE_INPUT_NEXT_MPLS_INPUT;
- else
- return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
+
+ return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
}
static_always_inline void
@@ -197,7 +182,7 @@ odp_adjust_buffer (vlib_buffer_t * buf, odp_packet_t pkt,
tr->hw_if_index = (oif)->hw_if_index; \
}
-void
+static void
odp_trace_buffer_x4 (uword * n_trace, vlib_main_t * vm,
vlib_node_runtime_t * node, odp_packet_if_t * oif,
vlib_buffer_t * b0, vlib_buffer_t * b1,
@@ -221,54 +206,62 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
u32 n_rx_bytes = 0;
u32 *to_next = 0;
odp_packet_t pkt_tbl[VLIB_FRAME_SIZE];
- int pkts = 0, i;
+ int i;
int n_left = 0, n_left_to_next = VLIB_FRAME_SIZE;
- u32 next0 = next_index;
- u32 next1 = next_index;
- u32 next2 = next_index;
- u32 next3 = next_index;
+ u32 next0;
+ u32 next1;
+ u32 next2;
+ u32 next3;
do
{
if ((oif->m.rx_mode == (APPL_MODE_PKT_QUEUE)) ||
(oif->m.rx_mode == (APPL_MODE_PKT_SCHED)))
{
- pkts =
+ n_left =
odp_packet_queue_mode (oif, pkt_tbl, queue_id, n_left_to_next);
}
else
{
- pkts =
+ n_left =
odp_packet_burst_mode (oif, pkt_tbl, queue_id, n_left_to_next);
}
- n_left = drop_err_pkts (pkt_tbl, pkts);
if (n_left == 0)
break;
i = 0;
- if (n_rx_packets == 0)
- vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+ vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
while ((n_left >= 4) && (n_left_to_next >= 4))
{
u32 bi0 = 0, bi1 = 0, bi2 = 0, bi3 = 0;
vlib_buffer_t *b0, *b1, *b2, *b3;
+ odp_packet_t pkt0, pkt1, pkt2, pkt3;
+
+ pkt0 = pkt_tbl[i++];
+ pkt1 = pkt_tbl[i++];
+ pkt2 = pkt_tbl[i++];
+ pkt3 = pkt_tbl[i++];
- b0 = (vlib_buffer_t *) odp_packet_user_area (pkt_tbl[i]);
- b1 = (vlib_buffer_t *) odp_packet_user_area (pkt_tbl[i + 1]);
- b2 = (vlib_buffer_t *) odp_packet_user_area (pkt_tbl[i + 2]);
- b3 = (vlib_buffer_t *) odp_packet_user_area (pkt_tbl[i + 3]);
+ b0 = vlib_buffer_from_odp_packet (pkt0);
+ b1 = vlib_buffer_from_odp_packet (pkt1);
+ b2 = vlib_buffer_from_odp_packet (pkt2);
+ b3 = vlib_buffer_from_odp_packet (pkt3);
bi0 = vlib_get_buffer_index (vm, b0);
bi1 = vlib_get_buffer_index (vm, b1);
bi2 = vlib_get_buffer_index (vm, b2);
bi3 = vlib_get_buffer_index (vm, b3);
+ b0->l2_priv_data = pkt0;
+ b1->l2_priv_data = pkt1;
+ b2->l2_priv_data = pkt2;
+ b3->l2_priv_data = pkt3;
- odp_adjust_buffer (b0, pkt_tbl[i], oif);
- odp_adjust_buffer (b1, pkt_tbl[i + 1], oif);
- odp_adjust_buffer (b2, pkt_tbl[i + 2], oif);
- odp_adjust_buffer (b3, pkt_tbl[i + 3], oif);
+ odp_adjust_buffer (b0, pkt0, oif);
+ odp_adjust_buffer (b1, pkt1, oif);
+ odp_adjust_buffer (b2, pkt2, oif);
+ odp_adjust_buffer (b3, pkt3, oif);
if (PREDICT_FALSE (oif->per_interface_next_index != ~0))
{
@@ -276,20 +269,31 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
next1 = oif->per_interface_next_index;
next2 = oif->per_interface_next_index;
next3 = oif->per_interface_next_index;
+ if (oif->per_interface_next_index !=
+ VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
+ {
+ vlib_buffer_advance (b0, sizeof (ethernet_header_t));
+ vlib_buffer_advance (b1, sizeof (ethernet_header_t));
+ vlib_buffer_advance (b2, sizeof (ethernet_header_t));
+ vlib_buffer_advance (b3, sizeof (ethernet_header_t));
+ }
}
else
{
- next0 = odp_rx_next_from_etype (pkt_tbl[i], b0);
- next1 = odp_rx_next_from_etype (pkt_tbl[i + 1], b1);
- next2 = odp_rx_next_from_etype (pkt_tbl[i + 2], b2);
- next3 = odp_rx_next_from_etype (pkt_tbl[i + 3], b3);
+ next0 = odp_rx_next_from_etype (pkt0, b0);
+ next1 = odp_rx_next_from_etype (pkt1, b1);
+ next2 = odp_rx_next_from_etype (pkt2, b2);
+ next3 = odp_rx_next_from_etype (pkt3, b3);
+ if (next0 != VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
+ vlib_buffer_advance (b0, sizeof (ethernet_header_t));
+ if (next1 != VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
+ vlib_buffer_advance (b1, sizeof (ethernet_header_t));
+ if (next2 != VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
+ vlib_buffer_advance (b2, sizeof (ethernet_header_t));
+ if (next3 != VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
+ vlib_buffer_advance (b3, sizeof (ethernet_header_t));
}
- vlib_buffer_advance (b0, device_input_next_node_advance[next0]);
- vlib_buffer_advance (b1, device_input_next_node_advance[next1]);
- vlib_buffer_advance (b2, device_input_next_node_advance[next2]);
- vlib_buffer_advance (b3, device_input_next_node_advance[next3]);
-
/* trace */
if (PREDICT_FALSE ((n_trace) > 0))
odp_trace_buffer_x4 (&n_trace, vm, node, oif, b0, b1, b2, b3,
@@ -312,7 +316,6 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
n_rx_bytes += b1->current_length;
n_rx_bytes += b2->current_length;
n_rx_bytes += b3->current_length;
- i += 4;
n_left -= 4;
n_rx_packets += 4;
}
@@ -321,12 +324,13 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
{
u32 bi0 = 0;
vlib_buffer_t *b0;
+ odp_packet_t pkt = pkt_tbl[i++];
- b0 = (vlib_buffer_t *) odp_packet_user_area (pkt_tbl[i]);
+ b0 = vlib_buffer_from_odp_packet (pkt);
bi0 = vlib_get_buffer_index (vm, b0);
- b0->l2_priv_data = pkt_tbl[i];
+ b0->l2_priv_data = pkt;
- b0->current_length = odp_packet_len (pkt_tbl[i]);
+ b0->current_length = odp_packet_len (pkt);
b0->current_data = 0;
b0->total_length_not_including_first_buffer = 0;
b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
@@ -336,9 +340,10 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
if (PREDICT_FALSE (oif->per_interface_next_index != ~0))
next0 = oif->per_interface_next_index;
else
- next0 = odp_rx_next_from_etype (pkt_tbl[i], b0);
+ next0 = odp_rx_next_from_etype (pkt, b0);
- vlib_buffer_advance (b0, device_input_next_node_advance[next0]);
+ if (next0 != VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
+ vlib_buffer_advance (b0, sizeof (ethernet_header_t));
/* trace */
ODP_TRACE_BUFFER (n_trace, b0, next0, vm, node, oif);
@@ -352,8 +357,7 @@ odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
n_left_to_next, bi0, next0);
/* next packet */
- n_rx_bytes += odp_packet_len (pkt_tbl[i]);
- i++;
+ n_rx_bytes += b0->current_length;
n_left--;
n_rx_packets++;
}
diff --git a/src/plugins/odp/odp_packet.c b/src/plugins/odp/odp_packet.c
index 86992c6..271e7a3 100755
--- a/src/plugins/odp/odp_packet.c
+++ b/src/plugins/odp/odp_packet.c
@@ -30,42 +30,6 @@ odp_packet_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi,
return 0;
}
-/**
- * Drop packets which input parsing marked as containing errors.
- *
- * Frees packets with error and modifies pkt_tbl[] to only contain packets with
- * no detected errors.
- *
- * @param pkt_tbl Array of packet
- * @param len Length of pkt_tbl[]
- *
- * @return Number of packets with no detected error
- */
-u32
-drop_err_pkts (odp_packet_t pkt_tbl[], unsigned len)
-{
- odp_packet_t pkt;
- unsigned pkt_cnt = len;
- unsigned i, j;
-
- for (i = 0, j = 0; i < len; ++i)
- {
- pkt = pkt_tbl[i];
-
- if (odp_unlikely (odp_packet_has_error (pkt)))
- {
- odp_packet_free (pkt); /* Drop */
- pkt_cnt--;
- }
- else if (odp_unlikely (i != j++))
- {
- pkt_tbl[j - 1] = pkt;
- }
- }
-
- return pkt_cnt;
-}
-
static odp_pktio_t
create_pktio (const char *dev, odp_pool_t pool, odp_packet_if_t * oif)
{
@@ -227,13 +191,6 @@ odp_packet_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set,
om->if_count++;
- if (tm->n_vlib_mains > 1)
- {
- oif->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
- CLIB_CACHE_LINE_BYTES);
- memset ((void *) oif->lockp, 0, CLIB_CACHE_LINE_BYTES);
- }
-
if (mode->rx_mode == APPL_MODE_PKT_BURST)
odp_pktin_queue (oif->pktio, oif->inq, oif->m.num_rx_queues);
else if (mode->rx_mode == APPL_MODE_PKT_QUEUE)
diff --git a/src/plugins/odp/odp_packet.h b/src/plugins/odp/odp_packet.h
index a80ab32..f3b478b 100755
--- a/src/plugins/odp/odp_packet.h
+++ b/src/plugins/odp/odp_packet.h
@@ -39,7 +39,6 @@ typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
u8 *host_if_name;
- volatile u32 *lockp;
uword if_index;
odp_pktio_t pktio;
u32 hw_if_index;
@@ -89,13 +88,41 @@ always_inline odp_packet_t
odp_packet_from_vlib_buffer (vlib_buffer_t * b)
{
odp_packet_t packet;
- packet = (odp_packet_t)(b->l2_priv_data);
+ packet = (odp_packet_t) (b->l2_priv_data);
if (packet == NULL)
- clib_error("ODP packet pointer was not set properly!\n");
+ clib_error ("ODP packet pointer was not set properly!\n");
return packet;
}
+always_inline vlib_buffer_t *
+vlib_buffer_from_odp_packet (odp_packet_t p)
+{
+ return (vlib_buffer_t *) odp_packet_user_area (p);
+}
+
+always_inline void
+odp_adjust_data_pointers (vlib_buffer_t * b0, odp_packet_t pkt)
+{
+ int diff;
+
+ diff = ((uintptr_t) (b0->data + b0->current_data) -
+ (uintptr_t) odp_packet_data (pkt));
+ if (diff > 0)
+ odp_packet_pull_head (pkt, diff);
+ else if (diff < 0)
+ odp_packet_push_head (pkt, -diff);
+
+ diff = b0->current_length - odp_packet_len (pkt);
+ if (diff)
+ {
+ if (diff > 0)
+ odp_packet_push_tail (pkt, diff);
+ else
+ odp_packet_pull_tail (pkt, -diff);
+ }
+}
+
/*
* fd.io coding-style-patch-verification: ON
*