summaryrefslogtreecommitdiffstats
path: root/src/plugins/avf/output.c
blob: 90cc56fc7d8349026a5ebb30d23ee4b4e7703ef8 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

@media only all and (prefers-color-scheme: dark) {
.highlight .hll { background-color: #49483e }
.highlight .c { color: #75715e } /* Comment */
.highlight .err { color: #960050; background-color: #1e0010 } /* Error */
.highlight .k { color: #66d9ef } /* Keyword */
.highlight .l { color: #ae81ff } /* Literal */
.highlight .n { color: #f8f8f2 } /* Name */
.highlight .o { color: #f92672 } /* Operator */
.highlight .p { color: #f8f8f2 } /* Punctuation */
.highlight .ch { color: #75715e } /* Comment.Hashbang */
.highlight .cm { color: #75715e } /* Comment.Multiline */
.highlight .cp { color: #75715e } /* Comment.Preproc */
.highlight .cpf { color: #75715e } /* Comment.PreprocFile */
.highlight .c1 { color: #75715e } /* Comment.Single */
.highlight .cs { color: #75715e } /* Comment.Special */
.highlight .gd { color: #f92672 } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gi { color: #a6e22e } /* Generic.Inserted */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #75715e } /* Generic.Subheading */
.highlight .kc { color: #66d9ef } /* Keyword.Constant */
.highlight .kd { color: #66d9ef } /* Keyword.Declaration */
.highlight .kn { color: #f92672 } /* Keyword.Namespace */
.highlight .kp { color: #66d9ef } /* Keyword.Pseudo */
.highlight .kr { color: #66d9ef } /* Keyword.Reserved */
.highlight .kt { color: #66d9ef } /* Keyword.Type */
.highlight .ld { color: #e6db74 } /* Literal.Date */
.highlight .m { color: #ae81ff } /* Literal.Number */
.highlight .s { color: #e6db74 } /* Literal.String */
.highlight .na { color: #a6e22e } /* Name.Attribute */
.highlight .nb { color: #f8f8f2 } /* Name.Builtin */
.highlight .nc { color: #a6e22e } /* Name.Class */
.highlight .no { color: #66d9ef } /* Name.Constant */
.highlight .nd { color: #a6e22e } /* Name.Decorator */
.highlight .ni { color: #f8f8f2 } /* Name.Entity */
.highlight .ne { color: #a6e22e } /* Name.Exception */
.highlight .nf { color: #a6e22e } /* Name.Function */
.highlight .nl { color: #f8f8f2 } /* Name.Label */
.highlight .nn { color: #f8f8f2 } /* Name.Namespace */
.highlight .nx { color: #a6e22e } /* Name.Other */
.highlight .py { color: #f8f8f2 } /* Name.Property */
.highlight .nt { color: #f92672 } /* Name.Tag */
.highlight .nv { color: #f8f8f2 } /* Name.Variable */
.highlight .ow { color: #f92672 } /* Operator.Word */
.highlight .w { color: #f8f8f2 } /* Text.Whitespace */
.highlight .mb { color: #ae81ff } /* Literal.Number.Bin */
.highlight .mf { color: #ae81ff } /* Literal.Number.Float */
.highlight .mh { color: #ae81ff } /* Literal.Number.Hex */
.highlight .mi { color: #ae81ff } /* Literal.Number.Integer */
.highlight .mo { color: #ae81ff } /* Literal.Number.Oct */
.highlight .sa { color: #e6db74 } /* Literal.String.Affix */
.highlight .sb { color: #e6db74 } /* Literal.String.Backtick */
.highlight .sc { color: #e6db74 } /* Literal.String.Char */
.highlight .dl { color: #e6db74 } /* Literal.String.Delimiter */
.highlight .sd { color: #e6db74 } /* Literal.String.Doc */
.highlight .s2 { color: #e6db74 } /* Literal.String.Double */
.highlight .se { color: #ae81ff } /* Literal.String.Escape */
.highlight .sh { color: #e6db74 } /* Literal.String.Heredoc */
.highlight .si { color: #e6db74 } /* Literal.String.Interpol */
.highlight .sx { color: #e6db74 } /* Literal.String.Other */
.highlight .sr { color: #e6db74 } /* Literal.String.Regex */
.highlight .s1 { color: #e6db74 } /* Literal.String.Single */
.highlight .ss { color: #e6db74 } /* Literal.String.Symbol */
.highlight .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #a6e22e } /* Name.Function.Magic */
.highlight .vc { color: #f8f8f2 } /* Name.Variable.Class */
.highlight .vg { color: #f8f8f2 } /* Name.Variable.Global */
.highlight .vi { color: #f8f8f2 } /* Name.Variable.Instance */
.highlight .vm { color: #f8f8f2 } /* Name.Variable.Magic */
.highlight .il { color: #ae81ff } /* Literal.Number.Integer.Long */
}
@media (prefers-color-scheme: light) {
.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight .cpf { color: #888888 } /* Comment.PreprocFile */
.highlight .c1 { color: #888888 } /* Comment.Single */
.highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */
.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gr { color: #aa0000 } /* Generic.Error */
.highlight .gh { color: #333333 } /* Generic.Heading */
.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #555555 } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #666666 } /* Generic.Subheading */
.highlight .gt { color: #aa0000 } /* Generic.Traceback */
.highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008800 } /* Keyword.Pseudo */
.highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */
.highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */
.highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */
.highlight .na { color: #336699 } /* Name.Attribute */
.highlight .nb { color: #003388 } /* Name.Builtin */
.highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */
.highlight .no { color: #003366; font-weight: bold } /* Name.Constant */
.highlight .nd { color: #555555 } /* Name.Decorator */
.highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */
.highlight .nl { color: #336699; font-style: italic } /* Name.Label */
.highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */
.highlight .py { color: #336699; font-weight: bold } /* Name.Property */
.highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #336699 } /* Name.Variable */
.highlight .ow { color: #008800 } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */
.highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */
.highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */
.highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */
.highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */
.highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */
.highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */
.highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */
.highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */
.highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */
.highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */
.highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */
.highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */
.highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */
.highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */
.highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */
.highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */
.highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */
.highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */
.highlight .vc { color: #336699 } /* Name.Variable.Class */
.highlight .vg { color: #dd7700 } /* Name.Variable.Global */
.highlight .vi { color: #3333bb } /* Name.Variable.Instance */
.highlight .vm { color: #336699 } /* Name.Variable.Magic */
.highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */
}
/*
 * Copyright (c) 2015-2016 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

 define vxlan_add_del_tunnel
{
  u32 client_index;
  u32 context;
  u8 is_add;
  u8 is_ipv6;
  u8 src_address[16];
  u8 dst_address[16];
  u32 mcast_sw_if_index;
  u32 encap_vrf_id;
  u32 decap_next_index;
  u32 vni;
};

define vxlan_add_del_tunnel_reply
{
  u32 context;
  i32 retval;
  u32 sw_if_index;
};

define vxlan_tunnel_dump
{
  u32 client_index;
  u32 context;
  u32 sw_if_index;
};

define vxlan_tunnel_details
{
  u32 context;
  u32 sw_if_index;
  u8 src_address[16];
  u8 dst_address[16];
  u32 mcast_sw_if_index;
  u32 encap_vrf_id;
  u32 decap_next_index;
  u32 vni;
  u8 is_ipv6;
};

/** \brief Interface set vxlan-bypass request
    @param client_index - opaque cookie to identify the sender
    @param context - sender context, to match reply w/ request
    @param sw_if_index - interface used to reach neighbor
    @param is_ipv6 - if non-zero, enable ipv6-vxlan-bypass, else ipv4-vxlan-bypass
    @param enable - if non-zero enable, else disable
*/
define sw_interface_set_vxlan_bypass
{
  u32 client_index;
  u32 context;
  u32 sw_if_index;
  u8 is_ipv6;
  u8 enable;
};

/** \brief Interface set vxlan-bypass response
    @param context - sender context, to match reply w/ request
    @param retval - return code for the request
*/
define sw_interface_set_vxlan_bypass_reply
{
  u32 context;
  i32 retval;
};
' href='#n429'>429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529
/*
 *------------------------------------------------------------------
 * Copyright (c) 2018 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *------------------------------------------------------------------
 */

#include <vlib/vlib.h>
#include <vlib/unix/unix.h>
#include <vlib/pci/pci.h>
#include <vppinfra/ring.h>

#include <vnet/ethernet/ethernet.h>
#include <vnet/ip/ip4_packet.h>
#include <vnet/ip/ip6_packet.h>
#include <vnet/udp/udp_packet.h>
#include <vnet/tcp/tcp_packet.h>

#include <vnet/devices/devices.h>

#include <avf/avf.h>

static_always_inline u8
avf_tx_desc_get_dtyp (avf_tx_desc_t * d)
{
  return d->qword[1] & 0x0f;
}

struct avf_ip4_psh
{
  u32 src;
  u32 dst;
  u8 zero;
  u8 proto;
  u16 l4len;
};

struct avf_ip6_psh
{
  u32 src[4];
  u32 dst[4];
  u32 l4len;
  u32 proto;
};

static_always_inline u64
avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso)
{
  u64 flags = 0;
  if (!is_tso && !(b->flags & ((VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
				VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
				VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
    return 0;
  u32 is_tcp = is_tso || b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
  u32 is_udp = !is_tso && b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
  u32 is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
  u32 is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
  ASSERT (!is_tcp || !is_udp);
  ASSERT (is_ip4 || is_ip6);
  i16 l2_hdr_offset = vnet_buffer (b)->l2_hdr_offset;
  i16 l3_hdr_offset = vnet_buffer (b)->l3_hdr_offset;
  i16 l4_hdr_offset = vnet_buffer (b)->l4_hdr_offset;
  u16 l2_len = l3_hdr_offset - l2_hdr_offset;
  u16 l3_len = l4_hdr_offset - l3_hdr_offset;
  ip4_header_t *ip4 = (void *) (b->data + l3_hdr_offset);
  ip6_header_t *ip6 = (void *) (b->data + l3_hdr_offset);
  tcp_header_t *tcp = (void *) (b->data + l4_hdr_offset);
  udp_header_t *udp = (void *) (b->data + l4_hdr_offset);
  u16 l4_len =
    is_tcp ? tcp_header_bytes (tcp) : is_udp ? sizeof (udp_header_t) : 0;
  u16 sum = 0;

  flags |= AVF_TXD_OFFSET_MACLEN (l2_len) |
    AVF_TXD_OFFSET_IPLEN (l3_len) | AVF_TXD_OFFSET_L4LEN (l4_len);
  flags |= is_ip4 ? AVF_TXD_CMD_IIPT_IPV4 : AVF_TXD_CMD_IIPT_IPV6;
  flags |= is_tcp ? AVF_TXD_CMD_L4T_TCP : is_udp ? AVF_TXD_CMD_L4T_UDP : 0;

  if (is_ip4)
    ip4->checksum = 0;

  if (is_tso)
    {
      if (is_ip4)
	ip4->length = 0;
      else
	ip6->payload_length = 0;
    }

  if (is_tcp || is_udp)
    {
      if (is_ip4)
	{
	  struct avf_ip4_psh psh = { 0 };
	  psh.src = ip4->src_address.as_u32;
	  psh.dst = ip4->dst_address.as_u32;
	  psh.proto = ip4->protocol;
	  psh.l4len =
	    is_tso ? 0 :
	    clib_host_to_net_u16 (clib_net_to_host_u16 (ip4->length) -
				  (l4_hdr_offset - l3_hdr_offset));
	  sum = ~ip_csum (&psh, sizeof (psh));
	}
      else
	{
	  struct avf_ip6_psh psh = { 0 };
	  clib_memcpy_fast (&psh.src, &ip6->src_address, 32);
	  psh.proto = clib_host_to_net_u32 ((u32) ip6->protocol);
	  psh.l4len = is_tso ? 0 : ip6->payload_length;
	  sum = ~ip_csum (&psh, sizeof (psh));
	}
    }
  /* ip_csum does a byte swap for some reason... */
  sum = clib_net_to_host_u16 (sum);
  if (is_tcp)
    tcp->checksum = sum;
  else if (is_udp)
    udp->checksum = sum;
  return flags;
}

static_always_inline int
avf_tx_fill_ctx_desc (vlib_main_t * vm, avf_txq_t * txq, avf_tx_desc_t * d,
		      vlib_buffer_t * b)
{
  vlib_buffer_t *ctx_ph = vlib_get_buffer (vm, txq->ctx_desc_placeholder_bi);
  if (PREDICT_FALSE (ctx_ph->ref_count == 255))
    {
      /* We need a new placeholder buffer */
      u32 new_bi;
      u8 bpi = vlib_buffer_pool_get_default_for_numa (vm, vm->numa_node);
      if (PREDICT_TRUE
	  (vlib_buffer_alloc_from_pool (vm, &new_bi, 1, bpi) == 1))
	{
	  /* Remove our own reference on the current placeholder buffer */
	  ctx_ph->ref_count--;
	  /* Replace with the new placeholder buffer */
	  txq->ctx_desc_placeholder_bi = new_bi;
	  ctx_ph = vlib_get_buffer (vm, new_bi);
	}
      else
	/* Impossible to enqueue a ctx descriptor, fail */
	return 1;
    }

  /* Acquire a reference on the placeholder buffer */
  ctx_ph->ref_count++;

  u16 l234hdr_sz =
    vnet_buffer (b)->l4_hdr_offset -
    vnet_buffer (b)->l2_hdr_offset + vnet_buffer2 (b)->gso_l4_hdr_sz;
  u16 tlen = vlib_buffer_length_in_chain (vm, b) - l234hdr_sz;
  d[0].qword[0] = 0;
  d[0].qword[1] = AVF_TXD_DTYP_CTX | AVF_TXD_CTX_CMD_TSO
    | AVF_TXD_CTX_SEG_MSS (vnet_buffer2 (b)->gso_size) |
    AVF_TXD_CTX_SEG_TLEN (tlen);
  return 0;
}


static_always_inline u16
avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
		u32 * buffers, u32 n_packets, int use_va_dma)
{
  u16 next = txq->next;
  u64 bits = AVF_TXD_CMD_EOP | AVF_TXD_CMD_RSV;
  const u32 offload_mask = VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
    VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
    VNET_BUFFER_F_GSO;
  u64 one_by_one_offload_flags = 0;
  int is_tso;
  u16 n_desc = 0;
  u16 *slot, n_desc_left, n_packets_left = n_packets;
  u16 mask = txq->size - 1;
  vlib_buffer_t *b[4];
  avf_tx_desc_t *d = txq->descs + next;
  u16 n_desc_needed;
  vlib_buffer_t *b0;

  /* avoid ring wrap */
  n_desc_left = txq->size - clib_max (txq->next, txq->n_enqueued + 8);

  if (n_desc_left == 0)
    return 0;

  /* Fast path, no ring wrap */
  while (n_packets_left && n_desc_left)
    {
      u32 or_flags;
      if (n_packets_left < 8 || n_desc_left < 4)
	goto one_by_one;

      vlib_prefetch_buffer_with_index (vm, buffers[4], LOAD);
      vlib_prefetch_buffer_with_index (vm, buffers[5], LOAD);
      vlib_prefetch_buffer_with_index (vm, buffers[6], LOAD);
      vlib_prefetch_buffer_with_index (vm, buffers[7], LOAD);

      b[0] = vlib_get_buffer (vm, buffers[0]);
      b[1] = vlib_get_buffer (vm, buffers[1]);
      b[2] = vlib_get_buffer (vm, buffers[2]);
      b[3] = vlib_get_buffer (vm, buffers[3]);

      or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;

      if (or_flags & (VLIB_BUFFER_NEXT_PRESENT | offload_mask))
	goto one_by_one;

      vlib_buffer_copy_indices (txq->bufs + next, buffers, 4);

      if (use_va_dma)
	{
	  d[0].qword[0] = vlib_buffer_get_current_va (b[0]);
	  d[1].qword[0] = vlib_buffer_get_current_va (b[1]);
	  d[2].qword[0] = vlib_buffer_get_current_va (b[2]);
	  d[3].qword[0] = vlib_buffer_get_current_va (b[3]);
	}
      else
	{
	  d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);
	  d[1].qword[0] = vlib_buffer_get_current_pa (vm, b[1]);
	  d[2].qword[0] = vlib_buffer_get_current_pa (vm, b[2]);
	  d[3].qword[0] = vlib_buffer_get_current_pa (vm, b[3]);
	}

      d[0].qword[1] = ((u64) b[0]->current_length) << 34 | bits;
      d[1].qword[1] = ((u64) b[1]->current_length) << 34 | bits;
      d[2].qword[1] = ((u64) b[2]->current_length) << 34 | bits;
      d[3].qword[1] = ((u64) b[3]->current_length) << 34 | bits;

      next += 4;
      n_desc += 4;
      buffers += 4;
      n_packets_left -= 4;
      n_desc_left -= 4;
      d += 4;
      continue;

    one_by_one:
      one_by_one_offload_flags = 0;
      txq->bufs[next] = buffers[0];
      b[0] = vlib_get_buffer (vm, buffers[0]);
      is_tso = ! !(b[0]->flags & VNET_BUFFER_F_GSO);
      if (PREDICT_FALSE (is_tso || b[0]->flags & offload_mask))
	one_by_one_offload_flags |= avf_tx_prepare_cksum (b[0], is_tso);

      /* Deal with chain buffer if present */
      if (is_tso || b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
	{
	  n_desc_needed = 1 + is_tso;
	  b0 = b[0];

	  /* Wish there were a buffer count for chain buffer */
	  while (b0->flags & VLIB_BUFFER_NEXT_PRESENT)
	    {
	      b0 = vlib_get_buffer (vm, b0->next_buffer);
	      n_desc_needed++;
	    }

	  /* spec says data descriptor is limited to 8 segments */
	  if (PREDICT_FALSE (!is_tso && n_desc_needed > 8))
	    {
	      vlib_buffer_free_one (vm, buffers[0]);
	      vlib_error_count (vm, node->node_index,
				AVF_TX_ERROR_SEGMENT_SIZE_EXCEEDED, 1);
	      n_packets_left -= 1;
	      buffers += 1;
	      continue;
	    }

	  if (PREDICT_FALSE (n_desc_left < n_desc_needed))
	    /*
	     * Slow path may be able to to deal with this since it can handle
	     * ring wrap
	     */
	    break;

	  /* Enqueue a context descriptor if needed */
	  if (PREDICT_FALSE (is_tso))
	    {
	      if (avf_tx_fill_ctx_desc (vm, txq, d, b[0]))
		/* Failure to acquire ref on ctx placeholder */
		break;
	      txq->bufs[next + 1] = txq->bufs[next];
	      txq->bufs[next] = txq->ctx_desc_placeholder_bi;
	      next += 1;
	      n_desc += 1;
	      n_desc_left -= 1;
	      d += 1;
	    }
	  while (b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
	    {
	      if (use_va_dma)
		d[0].qword[0] = vlib_buffer_get_current_va (b[0]);
	      else
		d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);

	      d[0].qword[1] = (((u64) b[0]->current_length) << 34) |
		AVF_TXD_CMD_RSV | one_by_one_offload_flags;

	      next += 1;
	      n_desc += 1;
	      n_desc_left -= 1;
	      d += 1;

	      txq->bufs[next] = b[0]->next_buffer;
	      b[0] = vlib_get_buffer (vm, b[0]->next_buffer);
	    }
	}

      if (use_va_dma)
	d[0].qword[0] = vlib_buffer_get_current_va (b[0]);
      else
	d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);

      d[0].qword[1] =
	(((u64) b[0]->current_length) << 34) | bits |
	one_by_one_offload_flags;

      next += 1;
      n_desc += 1;
      buffers += 1;
      n_packets_left -= 1;
      n_desc_left -= 1;
      d += 1;
    }

  /* Slow path to support ring wrap */
  if (PREDICT_FALSE (n_packets_left))
    {
      txq->n_enqueued += n_desc;

      n_desc = 0;
      d = txq->descs + (next & mask);

      /* +8 to be consistent with fast path */
      n_desc_left = txq->size - (txq->n_enqueued + 8);

      while (n_packets_left && n_desc_left)
	{

	  txq->bufs[next & mask] = buffers[0];
	  b[0] = vlib_get_buffer (vm, buffers[0]);

	  one_by_one_offload_flags = 0;
	  is_tso = ! !(b[0]->flags & VNET_BUFFER_F_GSO);
	  if (PREDICT_FALSE (is_tso || b[0]->flags & offload_mask))
	    one_by_one_offload_flags |= avf_tx_prepare_cksum (b[0], is_tso);

	  /* Deal with chain buffer if present */
	  if (is_tso || b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
	    {
	      n_desc_needed = 1 + is_tso;
	      b0 = b[0];

	      while (b0->flags & VLIB_BUFFER_NEXT_PRESENT)
		{
		  b0 = vlib_get_buffer (vm, b0->next_buffer);
		  n_desc_needed++;
		}

	      /* Spec says data descriptor is limited to 8 segments */
	      if (PREDICT_FALSE (!is_tso && n_desc_needed > 8))
		{
		  vlib_buffer_free_one (vm, buffers[0]);
		  vlib_error_count (vm, node->node_index,
				    AVF_TX_ERROR_SEGMENT_SIZE_EXCEEDED, 1);
		  n_packets_left -= 1;
		  buffers += 1;
		  continue;
		}

	      if (PREDICT_FALSE (n_desc_left < n_desc_needed))
		break;

	      /* Enqueue a context descriptor if needed */
	      if (PREDICT_FALSE (is_tso))
		{
		  if (avf_tx_fill_ctx_desc (vm, txq, d, b[0]))
		    /* Failure to acquire ref on ctx placeholder */
		    break;

		  txq->bufs[(next + 1) & mask] = txq->bufs[next & mask];
		  txq->bufs[next & mask] = txq->ctx_desc_placeholder_bi;
		  next += 1;
		  n_desc += 1;
		  n_desc_left -= 1;
		  d = txq->descs + (next & mask);
		}
	      while (b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
		{
		  if (use_va_dma)
		    d[0].qword[0] = vlib_buffer_get_current_va (b[0]);
		  else
		    d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);

		  d[0].qword[1] = (((u64) b[0]->current_length) << 34) |
		    AVF_TXD_CMD_RSV | one_by_one_offload_flags;

		  next += 1;
		  n_desc += 1;
		  n_desc_left -= 1;
		  d = txq->descs + (next & mask);

		  txq->bufs[next & mask] = b[0]->next_buffer;
		  b[0] = vlib_get_buffer (vm, b[0]->next_buffer);
		}
	    }

	  if (use_va_dma)
	    d[0].qword[0] = vlib_buffer_get_current_va (b[0]);
	  else
	    d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);

	  d[0].qword[1] =
	    (((u64) b[0]->current_length) << 34) | bits |
	    one_by_one_offload_flags;

	  next += 1;
	  n_desc += 1;
	  buffers += 1;
	  n_packets_left -= 1;
	  n_desc_left -= 1;
	  d = txq->descs + (next & mask);
	}
    }

  if ((slot = clib_ring_enq (txq->rs_slots)))
    {
      u16 rs_slot = slot[0] = (next - 1) & mask;
      d = txq->descs + rs_slot;
      d[0].qword[1] |= AVF_TXD_CMD_RS;
    }

  txq->next = next & mask;
  clib_atomic_store_rel_n (txq->qtx_tail, txq->next);
  txq->n_enqueued += n_desc;
  return n_packets - n_packets_left;
}

VNET_DEVICE_CLASS_TX_FN (avf_device_class) (vlib_main_t * vm,
					    vlib_node_runtime_t * node,
					    vlib_frame_t * frame)
{
  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
  avf_device_t *ad = avf_get_device (rd->dev_instance);
  u32 thread_index = vm->thread_index;
  u8 qid = thread_index;
  avf_txq_t *txq = vec_elt_at_index (ad->txqs, qid % ad->num_queue_pairs);
  u32 *buffers = vlib_frame_vector_args (frame);
  u16 n_enq, n_left;
  u16 n_retry = 2;

  clib_spinlock_lock_if_init (&txq->lock);

  n_left = frame->n_vectors;

retry:
  /* release consumed bufs */
  if (txq->n_enqueued)
    {
      i32 complete_slot = -1;
      while (1)
	{
	  u16 *slot = clib_ring_get_first (txq->rs_slots);

	  if (slot == 0)
	    break;

	  if (avf_tx_desc_get_dtyp (txq->descs + slot[0]) != 0x0F)
	    break;

	  complete_slot = slot[0];

	  clib_ring_deq (txq->rs_slots);
	}

      if (complete_slot >= 0)
	{
	  u16 first, mask, n_free;
	  mask = txq->size - 1;
	  first = (txq->next - txq->n_enqueued) & mask;
	  n_free = (complete_slot + 1 - first) & mask;

	  txq->n_enqueued -= n_free;
	  vlib_buffer_free_from_ring_no_next (vm, txq->bufs, first, txq->size,
					      n_free);
	}
    }

  if (ad->flags & AVF_DEVICE_F_VA_DMA)
    n_enq = avf_tx_enqueue (vm, node, txq, buffers, n_left, 1);
  else
    n_enq = avf_tx_enqueue (vm, node, txq, buffers, n_left, 0);

  n_left -= n_enq;

  if (n_left)
    {
      buffers += n_enq;

      if (n_retry--)
	goto retry;

      vlib_buffer_free (vm, buffers, n_left);
      vlib_error_count (vm, node->node_index,
			AVF_TX_ERROR_NO_FREE_SLOTS, n_left);
    }

  clib_spinlock_unlock_if_init (&txq->lock);

  return frame->n_vectors - n_left;
}

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */