src/plugins/linux-cp/lcp_node.c


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

@media only all and (prefers-color-scheme: dark) {
.highlight .hll { background-color: #49483e }
.highlight .c { color: #75715e } /* Comment */
.highlight .err { color: #960050; background-color: #1e0010 } /* Error */
.highlight .k { color: #66d9ef } /* Keyword */
.highlight .l { color: #ae81ff } /* Literal */
.highlight .n { color: #f8f8f2 } /* Name */
.highlight .o { color: #f92672 } /* Operator */
.highlight .p { color: #f8f8f2 } /* Punctuation */
.highlight .ch { color: #75715e } /* Comment.Hashbang */
.highlight .cm { color: #75715e } /* Comment.Multiline */
.highlight .cp { color: #75715e } /* Comment.Preproc */
.highlight .cpf { color: #75715e } /* Comment.PreprocFile */
.highlight .c1 { color: #75715e } /* Comment.Single */
.highlight .cs { color: #75715e } /* Comment.Special */
.highlight .gd { color: #f92672 } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gi { color: #a6e22e } /* Generic.Inserted */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #75715e } /* Generic.Subheading */
.highlight .kc { color: #66d9ef } /* Keyword.Constant */
.highlight .kd { color: #66d9ef } /* Keyword.Declaration */
.highlight .kn { color: #f92672 } /* Keyword.Namespace */
.highlight .kp { color: #66d9ef } /* Keyword.Pseudo */
.highlight .kr { color: #66d9ef } /* Keyword.Reserved */
.highlight .kt { color: #66d9ef } /* Keyword.Type */
.highlight .ld { color: #e6db74 } /* Literal.Date */
.highlight .m { color: #ae81ff } /* Literal.Number */
.highlight .s { color: #e6db74 } /* Literal.String */
.highlight .na { color: #a6e22e } /* Name.Attribute */
.highlight .nb { color: #f8f8f2 } /* Name.Builtin */
.highlight .nc { color: #a6e22e } /* Name.Class */
.highlight .no { color: #66d9ef } /* Name.Constant */
.highlight .nd { color: #a6e22e } /* Name.Decorator */
.highlight .ni { color: #f8f8f2 } /* Name.Entity */
.highlight .ne { color: #a6e22e } /* Name.Exception */
.highlight .nf { color: #a6e22e } /* Name.Function */
.highlight .nl { color: #f8f8f2 } /* Name.Label */
.highlight .nn { color: #f8f8f2 } /* Name.Namespace */
.highlight .nx { color: #a6e22e } /* Name.Other */
.highlight .py { color: #f8f8f2 } /* Name.Property */
.highlight .nt { color: #f92672 } /* Name.Tag */
.highlight .nv { color: #f8f8f2 } /* Name.Variable */
.highlight .ow { color: #f92672 } /* Operator.Word */
.highlight .w { color: #f8f8f2 } /* Text.Whitespace */
.highlight .mb { color: #ae81ff } /* Literal.Number.Bin */
.highlight .mf { color: #ae81ff } /* Literal.Number.Float */
.highlight .mh { color: #ae81ff } /* Literal.Number.Hex */
.highlight .mi { color: #ae81ff } /* Literal.Number.Integer */
.highlight .mo { color: #ae81ff } /* Literal.Number.Oct */
.highlight .sa { color: #e6db74 } /* Literal.String.Affix */
.highlight .sb { color: #e6db74 } /* Literal.String.Backtick */
.highlight .sc { color: #e6db74 } /* Literal.String.Char */
.highlight .dl { color: #e6db74 } /* Literal.String.Delimiter */
.highlight .sd { color: #e6db74 } /* Literal.String.Doc */
.highlight .s2 { color: #e6db74 } /* Literal.String.Double */
.highlight .se { color: #ae81ff } /* Literal.String.Escape */
.highlight .sh { color: #e6db74 } /* Literal.String.Heredoc */
.highlight .si { color: #e6db74 } /* Literal.String.Interpol */
.highlight .sx { color: #e6db74 } /* Literal.String.Other */
.highlight .sr { color: #e6db74 } /* Literal.String.Regex */
.highlight .s1 { color: #e6db74 } /* Literal.String.Single */
.highlight .ss { color: #e6db74 } /* Literal.String.Symbol */
.highlight .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #a6e22e } /* Name.Function.Magic */
.highlight .vc { color: #f8f8f2 } /* Name.Variable.Class */
.highlight .vg { color: #f8f8f2 } /* Name.Variable.Global */
.highlight .vi { color: #f8f8f2 } /* Name.Variable.Instance */
.highlight .vm { color: #f8f8f2 } /* Name.Variable.Magic */
.highlight .il { color: #ae81ff } /* Literal.Number.Integer.Long */
}
@media (prefers-color-scheme: light) {
.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight .cpf { color: #888888 } /* Comment.PreprocFile */
.highlight .c1 { color: #888888 } /* Comment.Single */
.highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */
.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gr { color: #aa0000 } /* Generic.Error */
.highlight .gh { color: #333333 } /* Generic.Heading */
.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #555555 } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #666666 } /* Generic.Subheading */
.highlight .gt { color: #aa0000 } /* Generic.Traceback */
.highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008800 } /* Keyword.Pseudo */
.highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */
.highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */
.highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */
.highlight .na { color: #336699 } /* Name.Attribute */
.highlight .nb { color: #003388 } /* Name.Builtin */
.highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */
.highlight .no { color: #003366; font-weight: bold } /* Name.Constant */
.highlight .nd { color: #555555 } /* Name.Decorator */
.highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */
.highlight .nl { color: #336699; font-style: italic } /* Name.Label */
.highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */
.highlight .py { color: #336699; font-weight: bold } /* Name.Property */
.highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #336699 } /* Name.Variable */
.highlight .ow { color: #008800 } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */
.highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */
.highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */
.highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */
.highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */
.highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */
.highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */
.highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */
.highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */
.highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */
.highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */
.highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */
.highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */
.highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */
.highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */
.highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */
.highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */
.highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */
.highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */
.highlight .vc { color: #336699 } /* Name.Variable.Class */
.highlight .vg { color: #dd7700 } /* Name.Variable.Global */
.highlight .vi { color: #3333bb } /* Name.Variable.Instance */
.highlight .vm { color: #336699 } /* Name.Variable.Magic */
.highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */
}
from vpp_interface import VppInterface


class VppVhostInterface(VppInterface):
    """VPP vhost interface."""

    def __init__(self, test, sock_filename, is_server=0, renumber=0,
                 disable_mrg_rxbuf=0, disable_indirect_desc=0,
                 custom_dev_instance=0, use_custom_mac=0, mac_address='',
                 tag=''):

        """ Create VPP Vhost interface """
        super(VppVhostInterface, self).__init__(test)
        self.is_server = is_server
        self.sock_filename = sock_filename
        self.renumber = renumber
        self.disable_mrg_rxbuf = disable_mrg_rxbuf
        self.disable_indirect_desc = disable_indirect_desc
        self.custom_dev_instance = custom_dev_instance
        self.use_custom_mac = use_custom_mac
        self.mac_address = mac_address
        self.tag = tag

    def add_vpp_config(self):
        r = self.test.vapi.create_vhost_user_if(self.is_server,
                                                self.sock_filename,
                                                self.renumber,
                                                self.disable_mrg_rxbuf,
                                                self.disable_indirect_desc,
                                                self.custom_dev_instance,
                                                self.use_custom_mac,
                                                self.mac_address,
                                                self.tag)
        self.set_sw_if_index(r.sw_if_index)

    def remove_vpp_config(self):
        self.test.vapi.delete_vhost_user_if(self.sw_if_index)

    def is_interface_config_in_dump(self, dump):
        for i in dump:
            if i.sw_if_index == self.sw_if_index:
                return True
        else:
            return False
/*
 * lcp_enthernet_node.c : linux control plane ethernet node
 *
 * Copyright (c) 2021 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <sys/socket.h>
#include <linux/if.h>

#include <plugins/linux-cp/lcp_interface.h>
#include <plugins/linux-cp/lcp_adj.h>
#include <linux-cp/lcp.api_enum.h>

#include <vnet/feature/feature.h>
#include <vnet/ip/ip4_packet.h>
#include <vnet/ethernet/arp_packet.h>
#include <vnet/ethernet/ethernet.h>
#include <vnet/ip/ip_types.h>
#include <vnet/ip/lookup.h>
#include <vnet/ip/ip4.h>
#include <vnet/ip/ip6.h>
#include <vnet/l2/l2_input.h>

#define foreach_lip_punt                                                      \
  _ (IO, "punt to host")                                                      \
  _ (DROP, "unknown input interface")

typedef enum
{
#define _(sym, str) LIP_PUNT_NEXT_##sym,
  foreach_lip_punt
#undef _
    LIP_PUNT_N_NEXT,
} lip_punt_next_t;

typedef struct lip_punt_trace_t_
{
  u32 phy_sw_if_index;
  u32 host_sw_if_index;
} lip_punt_trace_t;

/* packet trace format function */
static u8 *
format_lip_punt_trace (u8 *s, va_list *args)
{
  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  lip_punt_trace_t *t = va_arg (*args, lip_punt_trace_t *);

  s =
    format (s, "lip-punt: %u -> %u", t->phy_sw_if_index, t->host_sw_if_index);

  return s;
}

/**
 * Pass punted packets from the PHY to the HOST.
 */
VLIB_NODE_FN (lip_punt_node)
(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
{
  u32 n_left_from, *from, *to_next, n_left_to_next;
  lip_punt_next_t next_index;

  next_index = node->cached_next_index;
  n_left_from = frame->n_vectors;
  from = vlib_frame_vector_args (frame);

  while (n_left_from > 0)
    {
      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);

      while (n_left_from > 0 && n_left_to_next > 0)
	{
	  vlib_buffer_t *b0;
	  const lcp_itf_pair_t *lip0 = NULL;
	  u32 next0 = ~0;
	  u32 bi0, lipi0;
	  u32 sw_if_index0;
	  u8 len0;

	  bi0 = to_next[0] = from[0];

	  from += 1;
	  to_next += 1;
	  n_left_from -= 1;
	  n_left_to_next -= 1;
	  next0 = LIP_PUNT_NEXT_DROP;

	  b0 = vlib_get_buffer (vm, bi0);

	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
	  lipi0 = lcp_itf_pair_find_by_phy (sw_if_index0);
	  if (PREDICT_FALSE (lipi0 == INDEX_INVALID))
	    goto trace0;

	  lip0 = lcp_itf_pair_get (lipi0);
	  next0 = LIP_PUNT_NEXT_IO;
	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = lip0->lip_host_sw_if_index;

	  if (PREDICT_TRUE (lip0->lip_host_type == LCP_ITF_HOST_TAP))
	    {
	      /*
	       * rewind to ethernet header
	       */
	      len0 = ((u8 *) vlib_buffer_get_current (b0) -
		      (u8 *) ethernet_buffer_get_header (b0));
	      vlib_buffer_advance (b0, -len0);
	    }
	  /* Tun packets don't need any special treatment, just need to
	   * be escorted past the TTL decrement. If we still want to use
	   * ip[46]-punt-redirect with these, we could just set the
	   * VNET_BUFFER_F_LOCALLY_ORIGINATED in an 'else {}' here and
	   * then pass to the next node on the ip[46]-punt feature arc
	   */

	trace0:
	  if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_IS_TRACED)))
	    {
	      lip_punt_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
	      t->phy_sw_if_index = sw_if_index0;
	      t->host_sw_if_index =
		(lipi0 == INDEX_INVALID) ? ~0 : lip0->lip_host_sw_if_index;
	    }

	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
					   n_left_to_next, bi0, next0);
	}

      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
    }

  return frame->n_vectors;
}

VLIB_REGISTER_NODE (lip_punt_node) = {
  .name = "linux-cp-punt",
  .vector_size = sizeof (u32),
  .format_trace = format_lip_punt_trace,
  .type = VLIB_NODE_TYPE_INTERNAL,

  .n_next_nodes = LIP_PUNT_N_NEXT,
  .next_nodes = {
    [LIP_PUNT_NEXT_DROP] = "error-drop",
    [LIP_PUNT_NEXT_IO] = "interface-output",
  },
};

#define foreach_lcp_punt_l3 _ (DROP, "unknown error")

typedef enum
{
#define _(sym, str) LCP_LOCAL_NEXT_##sym,
  foreach_lcp_punt_l3
#undef _
    LCP_LOCAL_N_NEXT,
} lcp_punt_l3_next_t;

typedef struct lcp_punt_l3_trace_t_
{
  u32 phy_sw_if_index;
} lcp_punt_l3_trace_t;

/* packet trace format function */
static u8 *
format_lcp_punt_l3_trace (u8 *s, va_list *args)
{
  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  lcp_punt_l3_trace_t *t = va_arg (*args, lcp_punt_l3_trace_t *);

  s = format (s, "linux-cp-punt-l3: %u", t->phy_sw_if_index);

  return s;
}

VLIB_NODE_FN (lcp_punt_l3_node)
(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
{
  u32 n_left_from, *from, *to_next, n_left_to_next;
  lip_punt_next_t next_index;

  next_index = node->cached_next_index;
  n_left_from = frame->n_vectors;
  from = vlib_frame_vector_args (frame);

  while (n_left_from > 0)
    {
      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);

      while (n_left_from > 0 && n_left_to_next > 0)
	{
	  vlib_buffer_t *b0;
	  u32 next0 = LCP_LOCAL_NEXT_DROP;
	  u32 bi0;
	  index_t lipi0;
	  lcp_itf_pair_t *lip0;

	  bi0 = to_next[0] = from[0];

	  from += 1;
	  to_next += 1;
	  n_left_from -= 1;
	  n_left_to_next -= 1;

	  b0 = vlib_get_buffer (vm, bi0);
	  vnet_feature_next (&next0, b0);

	  lipi0 =
	    lcp_itf_pair_find_by_phy (vnet_buffer (b0)->sw_if_index[VLIB_RX]);
	  if (lipi0 != INDEX_INVALID)
	    {
	      /*
	       * Avoid TTL check for packets which arrived on a tunnel and
	       * are being punted to the local host.
	       */
	      lip0 = lcp_itf_pair_get (lipi0);
	      if (lip0->lip_host_type == LCP_ITF_HOST_TUN)
		b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
	    }

	  if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_IS_TRACED)))
	    {
	      lcp_punt_l3_trace_t *t =
		vlib_add_trace (vm, node, b0, sizeof (*t));
	      t->phy_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
	    }

	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
					   n_left_to_next, bi0, next0);
	}

      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
    }

  return frame->n_vectors;
}

VLIB_REGISTER_NODE (lcp_punt_l3_node) = {
  .name = "linux-cp-punt-l3",
  .vector_size = sizeof (u32),
  .format_trace = format_lcp_punt_l3_trace,
  .type = VLIB_NODE_TYPE_INTERNAL,

  .n_next_nodes = 1,
  .next_nodes = {
    [LCP_LOCAL_NEXT_DROP] = "error-drop",
  },
};

VNET_FEATURE_INIT (lcp_punt_l3_ip4, static) = {
  .arc_name = "ip4-punt",
  .node_name = "linux-cp-punt-l3",
  .runs_before = VNET_FEATURES ("ip4-punt-redirect"),
};

VNET_FEATURE_INIT (lip_punt_l3_ip6, static) = {
  .arc_name = "ip6-punt",
  .node_name = "linux-cp-punt-l3",
  .runs_before = VNET_FEATURES ("ip6-punt-redirect"),
};

#define foreach_lcp_xc                                                        \
  _ (DROP, "drop")                                                            \
  _ (XC_IP4, "x-connnect-ip4")                                                \
  _ (XC_IP6, "x-connnect-ip6")

typedef enum
{
#define _(sym, str) LCP_XC_NEXT_##sym,
  foreach_lcp_xc
#undef _
    LCP_XC_N_NEXT,
} lcp_xc_next_t;

typedef struct lcp_xc_trace_t_
{
  u32 phy_sw_if_index;
  adj_index_t adj_index;
} lcp_xc_trace_t;

/* packet trace format function */
static u8 *
format_lcp_xc_trace (u8 *s, va_list *args)
{
  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  lcp_xc_trace_t *t = va_arg (*args, lcp_xc_trace_t *);

  s = format (s, "lcp-xc: itf:%d adj:%d", t->phy_sw_if_index, t->adj_index);

  return s;
}

/**
 * X-connect all packets from the HOST to the PHY.
 *
 * This runs in either the IP4 or IP6 path. The MAC rewrite on the received
 * packet from the host is used as a key to find the adjacency used on the phy.
 * This allows this code to start the feature arc on that adjacency.
 * Consequently, all packet sent from the host are also subject to output
 * features, which is symmetric w.r.t. to input features.
 */
static_always_inline u32
lcp_xc_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame,
	       ip_address_family_t af)
{
  u32 n_left_from, *from, *to_next, n_left_to_next;
  lcp_xc_next_t next_index;
  ip_lookup_main_t *lm;

  next_index = 0;
  n_left_from = frame->n_vectors;
  from = vlib_frame_vector_args (frame);

  if (AF_IP4 == af)
    lm = &ip4_main.lookup_main;
  else
    lm = &ip6_main.lookup_main;

  while (n_left_from > 0)
    {
      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);

      while (n_left_from > 0 && n_left_to_next > 0)
	{
	  const ethernet_header_t *eth;
	  const lcp_itf_pair_t *lip;
	  u32 next0, bi0, lipi, ai;
	  vlib_buffer_t *b0;

	  bi0 = to_next[0] = from[0];

	  from += 1;
	  to_next += 1;
	  n_left_from -= 1;
	  n_left_to_next -= 1;

	  b0 = vlib_get_buffer (vm, bi0);

	  lipi =
	    lcp_itf_pair_find_by_host (vnet_buffer (b0)->sw_if_index[VLIB_RX]);
	  lip = lcp_itf_pair_get (lipi);

	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = lip->lip_phy_sw_if_index;
	  vlib_buffer_advance (b0, -lip->lip_rewrite_len);
	  eth = vlib_buffer_get_current (b0);

	  if (ethernet_address_cast (eth->dst_address))
	    ai = lip->lip_phy_adjs.adj_index[af];
	  else
	    ai = lcp_adj_lkup ((u8 *) eth, lip->lip_rewrite_len,
			       vnet_buffer (b0)->sw_if_index[VLIB_TX]);

	  if (ADJ_INDEX_INVALID != ai)
	    {
	      const ip_adjacency_t *adj;

	      adj = adj_get (ai);
	      vnet_buffer (b0)->ip.adj_index[VLIB_TX] = ai;
	      next0 = adj->rewrite_header.next_index;
	      vnet_buffer (b0)->ip.save_rewrite_length = lip->lip_rewrite_len;

	      if (PREDICT_FALSE (adj->rewrite_header.flags &
				 VNET_REWRITE_HAS_FEATURES))
		vnet_feature_arc_start_w_cfg_index (
		  lm->output_feature_arc_index,
		  vnet_buffer (b0)->sw_if_index[VLIB_TX], &next0, b0,
		  adj->ia_cfg_index);
	    }
	  else
	    next0 = LCP_XC_NEXT_DROP;

	  if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_IS_TRACED)))
	    {
	      lcp_xc_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
	      t->phy_sw_if_index = lip->lip_phy_sw_if_index;
	      t->adj_index = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
	    }

	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
					   n_left_to_next, bi0, next0);
	}

      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
    }

  return frame->n_vectors;
}

VLIB_NODE_FN (lcp_xc_ip4)
(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
{
  return (lcp_xc_inline (vm, node, frame, AF_IP4));
}

VLIB_NODE_FN (lcp_xc_ip6)
(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
{
  return (lcp_xc_inline (vm, node, frame, AF_IP6));
}

VLIB_REGISTER_NODE (lcp_xc_ip4) = { .name = "linux-cp-xc-ip4",
				    .vector_size = sizeof (u32),
				    .format_trace = format_lcp_xc_trace,
				    .type = VLIB_NODE_TYPE_INTERNAL,
				    .sibling_of = "ip4-rewrite" };

VNET_FEATURE_INIT (lcp_xc_ip4_ucast_node, static) = {
  .arc_name = "ip4-unicast",
  .node_name = "linux-cp-xc-ip4",
};
VNET_FEATURE_INIT (lcp_xc_ip4_mcast_node, static) = {
  .arc_name = "ip4-multicast",
  .node_name = "linux-cp-xc-ip4",
};

VLIB_REGISTER_NODE (lcp_xc_ip6) = { .name = "linux-cp-xc-ip6",
				    .vector_size = sizeof (u32),
				    .format_trace = format_lcp_xc_trace,
				    .type = VLIB_NODE_TYPE_INTERNAL,
				    .sibling_of = "ip6-rewrite" };

VNET_FEATURE_INIT (lcp_xc_ip6_ucast_node, static) = {
  .arc_name = "ip6-unicast",
  .node_name = "linux-cp-xc-ip6",
};
VNET_FEATURE_INIT (lcp_xc_ip6_mcast_node, static) = {
  .arc_name = "ip6-multicast",
  .node_name = "linux-cp-xc-ip6",
};

typedef enum
{
  LCP_XC_L3_NEXT_XC,
  LCP_XC_L3_N_NEXT,
} lcp_xc_l3_next_t;

/**
 * X-connect all packets from the HOST to the PHY on L3 interfaces
 *
 * There's only one adjacency that can be used on thises links.
 */
static_always_inline u32
lcp_xc_l3_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
		  vlib_frame_t *frame, ip_address_family_t af)
{
  u32 n_left_from, *from, *to_next, n_left_to_next;
  lcp_xc_next_t next_index;

  next_index = 0;
  n_left_from = frame->n_vectors;
  from = vlib_frame_vector_args (frame);

  while (n_left_from > 0)
    {
      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);

      while (n_left_from > 0 && n_left_to_next > 0)
	{
	  vlib_buffer_t *b0;
	  const lcp_itf_pair_t *lip;
	  u32 next0 = ~0;
	  u32 bi0, lipi;

	  bi0 = to_next[0] = from[0];

	  from += 1;
	  to_next += 1;
	  n_left_from -= 1;
	  n_left_to_next -= 1;

	  b0 = vlib_get_buffer (vm, bi0);

	  /* Flag buffers as locally originated. Otherwise their TTL will
	   * be checked & decremented. That would break services like BGP
	   * which set a TTL of 1 by default.
	   */
	  b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;

	  lipi =
	    lcp_itf_pair_find_by_host (vnet_buffer (b0)->sw_if_index[VLIB_RX]);
	  lip = lcp_itf_pair_get (lipi);

	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = lip->lip_phy_sw_if_index;
	  next0 = LCP_XC_L3_NEXT_XC;
	  vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
	    lip->lip_phy_adjs.adj_index[af];

	  if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_IS_TRACED)))
	    {
	      lcp_xc_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
	      t->phy_sw_if_index = lip->lip_phy_sw_if_index;
	      t->adj_index = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
	    }

	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
					   n_left_to_next, bi0, next0);
	}

      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
    }

  return frame->n_vectors;
}

/**
 * X-connect all packets from the HOST to the PHY.
 */
VLIB_NODE_FN (lcp_xc_l3_ip4_node)
(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
{
  return (lcp_xc_l3_inline (vm, node, frame, AF_IP4));
}

VLIB_NODE_FN (lcp_xc_l3_ip6_node)
(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
{
  return (lcp_xc_l3_inline (vm, node, frame, AF_IP6));
}

VLIB_REGISTER_NODE (lcp_xc_l3_ip4_node) = {
  .name = "linux-cp-xc-l3-ip4",
  .vector_size = sizeof (u32),
  .format_trace = format_lcp_xc_trace,
  .type = VLIB_NODE_TYPE_INTERNAL,

  .n_next_nodes = LCP_XC_L3_N_NEXT,
  .next_nodes = {
    [LCP_XC_L3_NEXT_XC] = "ip4-midchain",
  },
};

VNET_FEATURE_INIT (lcp_xc_node_l3_ip4_unicast, static) = {
  .arc_name = "ip4-unicast",
  .node_name = "linux-cp-xc-l3-ip4",
};

VNET_FEATURE_INIT (lcp_xc_node_l3_ip4_multicaast, static) = {
  .arc_name = "ip4-multicast",
  .node_name = "linux-cp-xc-l3-ip4",
};

VLIB_REGISTER_NODE (lcp_xc_l3_ip6_node) = {
  .name = "linux-cp-xc-l3-ip6",
  .vector_size = sizeof (u32),
  .format_trace = format_lcp_xc_trace,
  .type = VLIB_NODE_TYPE_INTERNAL,

  .n_next_nodes = LCP_XC_L3_N_NEXT,
  .next_nodes = {
    [LCP_XC_L3_NEXT_XC] = "ip6-midchain",
  },
};

VNET_FEATURE_INIT (lcp_xc_node_l3_ip6_unicast, static) = {
  .arc_name = "ip6-unicast",
  .node_name = "linux-cp-xc-l3-ip6",
};

VNET_FEATURE_INIT (lcp_xc_node_l3_ip6_multicast, static) = {
  .arc_name = "ip6-multicast",
  .node_name = "linux-cp-xc-l3-ip6",
};

#define foreach_lcp_arp                                                       \
  _ (DROP, "error-drop")                                                      \
  _ (IO, "interface-output")

typedef enum
{
#define _(sym, str) LCP_ARP_NEXT_##sym,
  foreach_lcp_arp
#undef _
    LCP_ARP_N_NEXT,
} lcp_arp_next_t;

typedef struct lcp_arp_trace_t_
{
  u32 rx_sw_if_index;
  u16 arp_opcode;
} lcp_arp_trace_t;

/* packet trace format function */
static u8 *
format_lcp_arp_trace (u8 *s, va_list *args)
{
  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  lcp_arp_trace_t *t = va_arg (*args, lcp_arp_trace_t *);

  s = format (s, "rx-sw-if-index: %u opcode: %u", t->rx_sw_if_index,
	      t->arp_opcode);

  return s;
}

/**
 * punt ARP replies to the host
 */
VLIB_NODE_FN (lcp_arp_phy_node)
(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
{
  u32 n_left_from, *from, *to_next, n_left_to_next;
  lcp_arp_next_t next_index;
  u32 reply_copies[VLIB_FRAME_SIZE];
  u32 n_copies = 0;

  next_index = node->cached_next_index;
  n_left_from = frame->n_vectors;
  from = vlib_frame_vector_args (frame);

  while (n_left_from > 0)
    {
      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);

      while (n_left_from >= 2 && n_left_to_next >= 2)
	{
	  u32 next0, next1, bi0, bi1;
	  vlib_buffer_t *b0, *b1;
	  ethernet_arp_header_t *arp0, *arp1;

	  bi0 = to_next[0] = from[0];
	  bi1 = to_next[1] = from[1];

	  from += 2;
	  n_left_from -= 2;
	  to_next += 2;
	  n_left_to_next -= 2;

	  next0 = next1 = LCP_ARP_NEXT_DROP;

	  b0 = vlib_get_buffer (vm, bi0);
	  b1 = vlib_get_buffer (vm, bi1);

	  arp0 = vlib_buffer_get_current (b0);
	  arp1 = vlib_buffer_get_current (b1);

	  vnet_feature_next (&next0, b0);
	  vnet_feature_next (&next1, b1);

	  /*
	   * Replies might need to be received by the host, so we
	   * make a copy of them.
	   */
	  if (arp0->opcode == clib_host_to_net_u16 (ETHERNET_ARP_OPCODE_reply))
	    {
	      lcp_itf_pair_t *lip0 = 0;
	      u32 lipi0;
	      vlib_buffer_t *c0;
	      u8 len0;

	      lipi0 = lcp_itf_pair_find_by_phy (
		vnet_buffer (b0)->sw_if_index[VLIB_RX]);
	      lip0 = lcp_itf_pair_get (lipi0);

	      if (lip0)
		{
		  /*
		   * rewind to eth header, copy, advance back to current
		   */
		  len0 = ((u8 *) vlib_buffer_get_current (b0) -
			  (u8 *) ethernet_buffer_get_header (b0));
		  vlib_buffer_advance (b0, -len0);
		  c0 = vlib_buffer_copy (vm, b0);
		  vlib_buffer_advance (b0, len0);

		  /* Send to the host */
		  vnet_buffer (c0)->sw_if_index[VLIB_TX] =
		    lip0->lip_host_sw_if_index;
		  reply_copies[n_copies++] = vlib_get_buffer_index (vm, c0);
		}
	    }
	  if (arp1->opcode == clib_host_to_net_u16 (ETHERNET_ARP_OPCODE_reply))
	    {
	      lcp_itf_pair_t *lip1 = 0;
	      u32 lipi1;
	      vlib_buffer_t *c1;
	      u8 len1;

	      lipi1 = lcp_itf_pair_find_by_phy (
		vnet_buffer (b1)->sw_if_index[VLIB_RX]);
	      lip1 = lcp_itf_pair_get (lipi1);

	      if (lip1)
		{
		  /*
		   * rewind to reveal the ethernet header
		   */
		  len1 = ((u8 *) vlib_buffer_get_current (b1) -
			  (u8 *) ethernet_buffer_get_header (b1));
		  vlib_buffer_advance (b1, -len1);
		  c1 = vlib_buffer_copy (vm, b1);
		  vlib_buffer_advance (b1, len1);

		  /* Send to the host */
		  vnet_buffer (c1)->sw_if_index[VLIB_TX] =
		    lip1->lip_host_sw_if_index;
		  reply_copies[n_copies++] = vlib_get_buffer_index (vm, c1);
		}
	    }

	  if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_IS_TRACED)))
	    {
	      lcp_arp_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
	      t->rx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
	    }
	  if (PREDICT_FALSE ((b1->flags & VLIB_BUFFER_IS_TRACED)))
	    {
	      lcp_arp_trace_t *t = vlib_add_trace (vm, node, b1, sizeof (*t));
	      t->rx_sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX];
	    }

	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
					   n_left_to_next, bi0, bi1, next0,
					   next1);
	}

      while (n_left_from > 0 && n_left_to_next > 0)
	{
	  u32 next0, bi0;
	  vlib_buffer_t *b0;
	  ethernet_arp_header_t *arp0;
	  u16 arp_opcode;

	  bi0 = to_next[0] = from[0];

	  from += 1;
	  n_left_from -= 1;
	  to_next += 1;
	  n_left_to_next -= 1;
	  next0 = LCP_ARP_NEXT_DROP;

	  b0 = vlib_get_buffer (vm, bi0);
	  arp0 = vlib_buffer_get_current (b0);

	  vnet_feature_next (&next0, b0);

	  /*
	   * Replies might need to be received by the host, so we
	   * make a copy of them.
	   */
	  arp_opcode = clib_host_to_net_u16 (arp0->opcode);

	  if (arp_opcode == ETHERNET_ARP_OPCODE_reply)
	    {
	      lcp_itf_pair_t *lip0 = 0;
	      vlib_buffer_t *c0;
	      u32 lipi0;
	      u8 len0;

	      lipi0 = lcp_itf_pair_find_by_phy (
		vnet_buffer (b0)->sw_if_index[VLIB_RX]);
	      lip0 = lcp_itf_pair_get (lipi0);

	      if (lip0)
		{

		  /*
		   * rewind to reveal the ethernet header
		   */
		  len0 = ((u8 *) vlib_buffer_get_current (b0) -
			  (u8 *) ethernet_buffer_get_header (b0));
		  vlib_buffer_advance (b0, -len0);
		  c0 = vlib_buffer_copy (vm, b0);
		  vlib_buffer_advance (b0, len0);

		  /* Send to the host */
		  vnet_buffer (c0)->sw_if_index[VLIB_TX] =
		    lip0->lip_host_sw_if_index;
		  reply_copies[n_copies++] = vlib_get_buffer_index (vm, c0);
		}
	    }

	  if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_IS_TRACED)))
	    {
	      lcp_arp_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
	      t->rx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
	      t->arp_opcode = arp_opcode;
	    }

	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
					   n_left_to_next, bi0, next0);
	}

      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
    }

  if (n_copies)
    vlib_buffer_enqueue_to_single_next (vm, node, reply_copies,
					LCP_ARP_NEXT_IO, n_copies);

  return frame->n_vectors;
}

VLIB_REGISTER_NODE (lcp_arp_phy_node) = {
  .name = "linux-cp-arp-phy",
  .vector_size = sizeof (u32),
  .format_trace = format_lcp_arp_trace,
  .type = VLIB_NODE_TYPE_INTERNAL,

  .n_errors = LINUXCP_N_ERROR,
  .error_counters = linuxcp_error_counters,

  .n_next_nodes = LCP_ARP_N_NEXT,
  .next_nodes = {
    [LCP_ARP_NEXT_DROP] = "error-drop",
    [LCP_ARP_NEXT_IO] = "interface-output",
  },
};

VNET_FEATURE_INIT (lcp_arp_phy_arp_feat, static) = {
  .arc_name = "arp",
  .node_name = "linux-cp-arp-phy",
  .runs_before = VNET_FEATURES ("arp-reply"),
};

/**
 * x-connect ARP packets from the host to the phy
 */
VLIB_NODE_FN (lcp_arp_host_node)
(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
{
  u32 n_left_from, *from, *to_next, n_left_to_next;
  lcp_arp_next_t next_index;

  next_index = node->cached_next_index;
  n_left_from = frame->n_vectors;
  from = vlib_frame_vector_args (frame);

  while (n_left_from > 0)
    {
      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);

      while (n_left_from > 0 && n_left_to_next > 0)
	{
	  const lcp_itf_pair_t *lip0;
	  lcp_arp_next_t next0;
	  vlib_buffer_t *b0;
	  u32 bi0, lipi0;
	  u8 len0;

	  bi0 = to_next[0] = from[0];

	  from += 1;
	  n_left_from -= 1;
	  to_next += 1;
	  n_left_to_next -= 1;
	  next0 = LCP_ARP_NEXT_IO;

	  b0 = vlib_get_buffer (vm, bi0);

	  lipi0 =
	    lcp_itf_pair_find_by_host (vnet_buffer (b0)->sw_if_index[VLIB_RX]);
	  lip0 = lcp_itf_pair_get (lipi0);

	  /* Send to the phy */
	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = lip0->lip_phy_sw_if_index;

	  len0 = ((u8 *) vlib_buffer_get_current (b0) -
		  (u8 *) ethernet_buffer_get_header (b0));
	  vlib_buffer_advance (b0, -len0);

	  if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_IS_TRACED)))
	    {
	      lcp_arp_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
	      t->rx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
	    }

	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
					   n_left_to_next, bi0, next0);
	}

      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
    }

  return frame->n_vectors;
}

VLIB_REGISTER_NODE (lcp_arp_host_node) = {
  .name = "linux-cp-arp-host",
  .vector_size = sizeof (u32),
  .format_trace = format_lcp_arp_trace,
  .type = VLIB_NODE_TYPE_INTERNAL,

  .n_errors = LINUXCP_N_ERROR,
  .error_counters = linuxcp_error_counters,

  .n_next_nodes = LCP_ARP_N_NEXT,
  .next_nodes = {
    [LCP_ARP_NEXT_DROP] = "error-drop",
    [LCP_ARP_NEXT_IO] = "interface-output",
  },
};

VNET_FEATURE_INIT (lcp_arp_host_arp_feat, static) = {
  .arc_name = "arp",
  .node_name = "linux-cp-arp-host",
  .runs_before = VNET_FEATURES ("arp-reply"),
};

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */