summaryrefslogtreecommitdiffstats
path: root/src/vnet/mpls/mpls_lookup.c
blob: 2d34cbde341f381ba8e6990e3bb8a6ec0bed24be (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

@media only all and (prefers-color-scheme: dark) {
.highlight .hll { background-color: #49483e }
.highlight .c { color: #75715e } /* Comment */
.highlight .err { color: #960050; background-color: #1e0010 } /* Error */
.highlight .k { color: #66d9ef } /* Keyword */
.highlight .l { color: #ae81ff } /* Literal */
.highlight .n { color: #f8f8f2 } /* Name */
.highlight .o { color: #f92672 } /* Operator */
.highlight .p { color: #f8f8f2 } /* Punctuation */
.highlight .ch { color: #75715e } /* Comment.Hashbang */
.highlight .cm { color: #75715e } /* Comment.Multiline */
.highlight .cp { color: #75715e } /* Comment.Preproc */
.highlight .cpf { color: #75715e } /* Comment.PreprocFile */
.highlight .c1 { color: #75715e } /* Comment.Single */
.highlight .cs { color: #75715e } /* Comment.Special */
.highlight .gd { color: #f92672 } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gi { color: #a6e22e } /* Generic.Inserted */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #75715e } /* Generic.Subheading */
.highlight .kc { color: #66d9ef } /* Keyword.Constant */
.highlight .kd { color: #66d9ef } /* Keyword.Declaration */
.highlight .kn { color: #f92672 } /* Keyword.Namespace */
.highlight .kp { color: #66d9ef } /* Keyword.Pseudo */
.highlight .kr { color: #66d9ef } /* Keyword.Reserved */
.highlight .kt { color: #66d9ef } /* Keyword.Type */
.highlight .ld { color: #e6db74 } /* Literal.Date */
.highlight .m { color: #ae81ff } /* Literal.Number */
.highlight .s { color: #e6db74 } /* Literal.String */
.highlight .na { color: #a6e22e } /* Name.Attribute */
.highlight .nb { color: #f8f8f2 } /* Name.Builtin */
.highlight .nc { color: #a6e22e } /* Name.Class */
.highlight .no { color: #66d9ef } /* Name.Constant */
.highlight .nd { color: #a6e22e } /* Name.Decorator */
.highlight .ni { color: #f8f8f2 } /* Name.Entity */
.highlight .ne { color: #a6e22e } /* Name.Exception */
.highlight .nf { color: #a6e22e } /* Name.Function */
.highlight .nl { color: #f8f8f2 } /* Name.Label */
.highlight .nn { color: #f8f8f2 } /* Name.Namespace */
.highlight .nx { color: #a6e22e } /* Name.Other */
.highlight .py { color: #f8f8f2 } /* Name.Property */
.highlight .nt { color: #f92672 } /* Name.Tag */
.highlight .nv { color: #f8f8f2 } /* Name.Variable */
.highlight .ow { color: #f92672 } /* Operator.Word */
.highlight .w { color: #f8f8f2 } /* Text.Whitespace */
.highlight .mb { color: #ae81ff } /* Literal.Number.Bin */
.highlight .mf { color: #ae81ff } /* Literal.Number.Float */
.highlight .mh { color: #ae81ff } /* Literal.Number.Hex */
.highlight .mi { color: #ae81ff } /* Literal.Number.Integer */
.highlight .mo { color: #ae81ff } /* Literal.Number.Oct */
.highlight .sa { color: #e6db74 } /* Literal.String.Affix */
.highlight .sb { color: #e6db74 } /* Literal.String.Backtick */
.highlight .sc { color: #e6db74 } /* Literal.String.Char */
.highlight .dl { color: #e6db74 } /* Literal.String.Delimiter */
.highlight .sd { color: #e6db74 } /* Literal.String.Doc */
.highlight .s2 { color: #e6db74 } /* Literal.String.Double */
.highlight .se { color: #ae81ff } /* Literal.String.Escape */
.highlight .sh { color: #e6db74 } /* Literal.String.Heredoc */
.highlight .si { color: #e6db74 } /* Literal.String.Interpol */
.highlight .sx { color: #e6db74 } /* Literal.String.Other */
.highlight .sr { color: #e6db74 } /* Literal.String.Regex */
.highlight .s1 { color: #e6db74 } /* Literal.String.Single */
.highlight .ss { color: #e6db74 } /* Literal.String.Symbol */
.highlight .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #a6e22e } /* Name.Function.Magic */
.highlight .vc { color: #f8f8f2 } /* Name.Variable.Class */
.highlight .vg { color: #f8f8f2 } /* Name.Variable.Global */
.highlight .vi { color: #f8f8f2 } /* Name.Variable.Instance */
.highlight .vm { color: #f8f8f2 } /* Name.Variable.Magic */
.highlight .il { color: #ae81ff } /* Literal.Number.Integer.Long */
}
@media (prefers-color-scheme: light) {
.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight .cpf { color: #888888 } /* Comment.PreprocFile */
.highlight .c1 { color: #888888 } /* Comment.Single */
.highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */
.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gr { color: #aa0000 } /* Generic.Error */
.highlight .gh { color: #333333 } /* Generic.Heading */
.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #555555 } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #666666 } /* Generic.Subheading */
.highlight .gt { color: #aa0000 } /* Generic.Traceback */
.highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008800 } /* Keyword.Pseudo */
.highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */
.highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */
.highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */
.highlight .na { color: #336699 } /* Name.Attribute */
.highlight .nb { color: #003388 } /* Name.Builtin */
.highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */
.highlight .no { color: #003366; font-weight: bold } /* Name.Constant */
.highlight .nd { color: #555555 } /* Name.Decorator */
.highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */
.highlight .nl { color: #336699; font-style: italic } /* Name.Label */
.highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */
.highlight .py { color: #336699; font-weight: bold } /* Name.Property */
.highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #336699 } /* Name.Variable */
.highlight .ow { color: #008800 } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */
.highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */
.highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */
.highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */
.highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */
.highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */
.highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */
.highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */
.highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */
.highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */
.highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */
.highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */
.highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */
.highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */
.highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */
.highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */
.highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */
.highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */
.highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */
.highlight .vc { color: #336699 } /* Name.Variable.Class */
.highlight .vg { color: #dd7700 } /* Name.Variable.Global */
.highlight .vi { color: #3333bb } /* Name.Variable.Instance */
.highlight .vm { color: #336699 } /* Name.Variable.Magic */
.highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */
}
/*
 * dhcp_proxy_error.def: dhcp proxy errors
 *
 * Copyright (c) 2013 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

dhcp_proxy_error (NONE, "no error")
dhcp_proxy_error (NO_SERVER, "no dhcp server configured")
dhcp_proxy_error (RELAY_TO_SERVER, "DHCP packets relayed to the server")
dhcp_proxy_error (RELAY_TO_CLIENT, "DHCP packets relayed to clients")
dhcp_proxy_error (OPTION_82_ERROR, "DHCP failed to insert option 82")
dhcp_proxy_error (NO_OPTION_82, "DHCP option 82 missing")
dhcp_proxy_error (BAD_OPTION_82_ITF, "Bad DHCP option 82 interface value")
dhcp_proxy_error (BAD_OPTION_82_ADDR, "Bad DHCP option 82 address value")
dhcp_proxy_error (BAD_FIB_ID, "DHCP option 82 fib-id to fib-index map failure")
dhcp_proxy_error (NO_INTERFACE_ADDRESS, "DHCP no interface address")
dhcp_proxy_error (OPTION_82_VSS_NOT_PROCESSED, "DHCP VSS not processed by DHCP server")
dhcp_proxy_error (BAD_YIADDR, "DHCP packets with bad your_ip_address fields")
dhcp_proxy_error (BAD_SVR_FIB_OR_ADDRESS, "DHCP packets not from DHCP server or server FIB.")
dhcp_proxy_error (PKT_TOO_BIG, "DHCP packets which are too big.")
id='n450' href='#n450'>450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531
/*
 * mpls_lookup.c: MPLS lookup
 *
 * Copyright (c) 2012-2014 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vlib/vlib.h>
#include <vnet/pg/pg.h>
#include <vnet/mpls/mpls.h>
#include <vnet/fib/mpls_fib.h>
#include <vnet/dpo/load_balance.h>

vlib_node_registration_t mpls_lookup_node;

typedef struct {
  u32 next_index;
  u32 lb_index;
  u32 lfib_index;
  u32 label_net_byte_order;
  u32 hash;
} mpls_lookup_trace_t;

static u8 *
format_mpls_lookup_trace (u8 * s, va_list * args)
{
  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  mpls_lookup_trace_t * t = va_arg (*args, mpls_lookup_trace_t *);

  s = format (s, "MPLS: next [%d], lookup fib index %d, LB index %d hash %d"
              "label %d eos %d", 
              t->next_index, t->lfib_index, t->lb_index, t->hash,
              vnet_mpls_uc_get_label(
                  clib_net_to_host_u32(t->label_net_byte_order)),
              vnet_mpls_uc_get_s(t->label_net_byte_order));
  return s;
}

/*
 * Compute flow hash. 
 * We'll use it to select which adjacency to use for this flow.  And other things.
 */
always_inline u32
mpls_compute_flow_hash (const mpls_unicast_header_t * hdr,
                        flow_hash_config_t flow_hash_config)
{
    // FIXME
    return (vnet_mpls_uc_get_label(hdr->label_exp_s_ttl));
}

static inline uword
mpls_lookup (vlib_main_t * vm,
             vlib_node_runtime_t * node,
             vlib_frame_t * from_frame)
{
  vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
  u32 n_left_from, next_index, * from, * to_next;
  mpls_main_t * mm = &mpls_main;
  u32 cpu_index = os_get_cpu_number();

  from = vlib_frame_vector_args (from_frame);
  n_left_from = from_frame->n_vectors;
  next_index = node->cached_next_index;

  while (n_left_from > 0)
    {
      u32 n_left_to_next;

      vlib_get_next_frame (vm, node, next_index,
                           to_next, n_left_to_next);

      while (n_left_from >= 4 && n_left_to_next >= 2)
        {
          u32 lbi0, next0, lfib_index0, bi0, hash_c0;
          const mpls_unicast_header_t * h0;
          const load_balance_t *lb0;
          const dpo_id_t *dpo0;
          vlib_buffer_t * b0;
          u32 lbi1, next1, lfib_index1, bi1, hash_c1;
          const mpls_unicast_header_t * h1;
          const load_balance_t *lb1;
          const dpo_id_t *dpo1;
          vlib_buffer_t * b1;

           /* Prefetch next iteration. */
          {
            vlib_buffer_t * p2, * p3;

            p2 = vlib_get_buffer (vm, from[2]);
            p3 = vlib_get_buffer (vm, from[3]);

            vlib_prefetch_buffer_header (p2, STORE);
            vlib_prefetch_buffer_header (p3, STORE);

            CLIB_PREFETCH (p2->data, sizeof (h0[0]), STORE);
            CLIB_PREFETCH (p3->data, sizeof (h0[0]), STORE);
          }

          bi0 = to_next[0] = from[0];
          bi1 = to_next[1] = from[1];

          from += 2;
          n_left_from -= 2;
          to_next += 2;
          n_left_to_next -= 2;

          b0 = vlib_get_buffer (vm, bi0);
          b1 = vlib_get_buffer (vm, bi1);
          h0 = vlib_buffer_get_current (b0);
          h1 = vlib_buffer_get_current (b1);

          lfib_index0 = vec_elt(mm->fib_index_by_sw_if_index,
                                vnet_buffer(b0)->sw_if_index[VLIB_RX]);
          lfib_index1 = vec_elt(mm->fib_index_by_sw_if_index,
                                vnet_buffer(b1)->sw_if_index[VLIB_RX]);

          lbi0 = mpls_fib_table_forwarding_lookup (lfib_index0, h0);
          lbi1 = mpls_fib_table_forwarding_lookup (lfib_index1, h1);
          lb0 = load_balance_get(lbi0);
          lb1 = load_balance_get(lbi1);

          hash_c0 = vnet_buffer(b0)->ip.flow_hash = 0;
          hash_c1 = vnet_buffer(b1)->ip.flow_hash = 0;

          if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
          {
              hash_c0 = vnet_buffer (b0)->ip.flow_hash =
                  mpls_compute_flow_hash(h0, lb0->lb_hash_config);
          }
          if (PREDICT_FALSE(lb1->lb_n_buckets > 1))
          {
              hash_c1 = vnet_buffer (b1)->ip.flow_hash =
                  mpls_compute_flow_hash(h1, lb1->lb_hash_config);
          }

          ASSERT (lb0->lb_n_buckets > 0);
          ASSERT (is_pow2 (lb0->lb_n_buckets));
          ASSERT (lb1->lb_n_buckets > 0);
          ASSERT (is_pow2 (lb1->lb_n_buckets));

          dpo0 = load_balance_get_bucket_i(lb0,
                                           (hash_c0 &
                                            (lb0->lb_n_buckets_minus_1)));
          dpo1 = load_balance_get_bucket_i(lb1,
                                           (hash_c1 &
                                            (lb1->lb_n_buckets_minus_1)));

          next0 = dpo0->dpoi_next_node;
          next1 = dpo1->dpoi_next_node;

          vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
          vnet_buffer (b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;

          vlib_increment_combined_counter
              (cm, cpu_index, lbi0, 1,
               vlib_buffer_length_in_chain (vm, b0));
          vlib_increment_combined_counter
              (cm, cpu_index, lbi1, 1,
               vlib_buffer_length_in_chain (vm, b1));

          /*
           * before we pop the label copy th values we need to maintain.
           * The label header is in network byte order.
           *  last byte is the TTL.
           *  bits 2 to 4 inclusive are the EXP bits
           */
          vnet_buffer (b0)->mpls.ttl = ((char*)h0)[3];
          vnet_buffer (b0)->mpls.exp = (((char*)h0)[2] & 0xe) >> 1;
          vnet_buffer (b0)->mpls.first = 1;
          vnet_buffer (b1)->mpls.ttl = ((char*)h1)[3];
          vnet_buffer (b1)->mpls.exp = (((char*)h1)[2] & 0xe) >> 1;
          vnet_buffer (b1)->mpls.first = 1;

          /*
           * pop the label that was just used in the lookup
           */
          vlib_buffer_advance(b0, sizeof(*h0));
          vlib_buffer_advance(b1, sizeof(*h1));

          if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
          {
              mpls_lookup_trace_t *tr = vlib_add_trace (vm, node,
                                                        b0, sizeof (*tr));
              tr->next_index = next0;
              tr->lb_index = lbi0;
              tr->lfib_index = lfib_index0;
              tr->hash = hash_c0;
              tr->label_net_byte_order = h0->label_exp_s_ttl;
          }

          if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
          {
              mpls_lookup_trace_t *tr = vlib_add_trace (vm, node,
                                                        b1, sizeof (*tr));
              tr->next_index = next1;
              tr->lb_index = lbi1;
              tr->lfib_index = lfib_index1;
              tr->hash = hash_c1;
              tr->label_net_byte_order = h1->label_exp_s_ttl;
          }

          vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
                                           to_next, n_left_to_next,
                                           bi0, bi1, next0, next1);
        }

      while (n_left_from > 0 && n_left_to_next > 0)
      {
          u32 lbi0, next0, lfib_index0, bi0, hash_c0;
          const mpls_unicast_header_t * h0;
          const load_balance_t *lb0;
          const dpo_id_t *dpo0;
          vlib_buffer_t * b0;

          bi0 = from[0];
          to_next[0] = bi0;
          from += 1;
          to_next += 1;
          n_left_from -= 1;
          n_left_to_next -= 1;

          b0 = vlib_get_buffer (vm, bi0);
          h0 = vlib_buffer_get_current (b0);

          lfib_index0 = vec_elt(mm->fib_index_by_sw_if_index,
                                vnet_buffer(b0)->sw_if_index[VLIB_RX]);

          lbi0 = mpls_fib_table_forwarding_lookup(lfib_index0, h0);
	  lb0 = load_balance_get(lbi0);

          hash_c0 = vnet_buffer(b0)->ip.flow_hash = 0;
          if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
          {
              hash_c0 = vnet_buffer (b0)->ip.flow_hash =
                  mpls_compute_flow_hash(h0, lb0->lb_hash_config);
          }

          ASSERT (lb0->lb_n_buckets > 0);
          ASSERT (is_pow2 (lb0->lb_n_buckets));

          dpo0 = load_balance_get_bucket_i(lb0,
                                           (hash_c0 &
                                            (lb0->lb_n_buckets_minus_1)));

          next0 = dpo0->dpoi_next_node;
          vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;

          vlib_increment_combined_counter
              (cm, cpu_index, lbi0, 1,
               vlib_buffer_length_in_chain (vm, b0));

          /*
           * before we pop the label copy th values we need to maintain.
           * The label header is in network byte order.
           *  last byte is the TTL.
           *  bits 2 to 4 inclusive are the EXP bits
           */
          vnet_buffer (b0)->mpls.ttl = ((char*)h0)[3];
          vnet_buffer (b0)->mpls.exp = (((char*)h0)[2] & 0xe) >> 1;
          vnet_buffer (b0)->mpls.first = 1;

          /*
           * pop the label that was just used in the lookup
           */
          vlib_buffer_advance(b0, sizeof(*h0));

          if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
          {
              mpls_lookup_trace_t *tr = vlib_add_trace (vm, node,
                                                        b0, sizeof (*tr));
              tr->next_index = next0;
              tr->lb_index = lbi0;
              tr->lfib_index = lfib_index0;
              tr->hash = hash_c0;
              tr->label_net_byte_order = h0->label_exp_s_ttl;
          }

          vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
                                           to_next, n_left_to_next,
                                           bi0, next0);
        }

      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
    }
  vlib_node_increment_counter (vm, mpls_lookup_node.index,
                               MPLS_ERROR_PKTS_DECAP, from_frame->n_vectors);
  return from_frame->n_vectors;
}

static char * mpls_error_strings[] = {
#define mpls_error(n,s) s,
#include "error.def"
#undef mpls_error
};

VLIB_REGISTER_NODE (mpls_lookup_node) = {
  .function = mpls_lookup,
  .name = "mpls-lookup",
  /* Takes a vector of packets. */
  .vector_size = sizeof (u32),
  .n_errors = MPLS_N_ERROR,
  .error_strings = mpls_error_strings,

  .sibling_of = "ip4-lookup",

  .format_buffer = format_mpls_header,
  .format_trace = format_mpls_lookup_trace,
  .unformat_buffer = unformat_mpls_header,
};

VLIB_NODE_FUNCTION_MULTIARCH (mpls_lookup_node, mpls_lookup)

typedef struct {
  u32 next_index;
  u32 lb_index;
  u32 hash;
} mpls_load_balance_trace_t;

static u8 *
format_mpls_load_balance_trace (u8 * s, va_list * args)
{
  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  mpls_load_balance_trace_t * t = va_arg (*args, mpls_load_balance_trace_t *);

  s = format (s, "MPLS: next [%d], LB index %d hash %d",
              t->next_index, t->lb_index, t->hash);
  return s;
}

always_inline uword
mpls_load_balance (vlib_main_t * vm,
                  vlib_node_runtime_t * node,
                  vlib_frame_t * frame)
{
  vlib_combined_counter_main_t * cm = &load_balance_main.lbm_via_counters;
  u32 n_left_from, n_left_to_next, * from, * to_next;
  u32 cpu_index = os_get_cpu_number();
  u32 next;

  from = vlib_frame_vector_args (frame);
  n_left_from = frame->n_vectors;
  next = node->cached_next_index;

  while (n_left_from > 0)
    {
      vlib_get_next_frame (vm, node, next,
                           to_next, n_left_to_next);


      while (n_left_from >= 4 && n_left_to_next >= 2)
        {
          mpls_lookup_next_t next0, next1;
          const load_balance_t *lb0, *lb1;
          vlib_buffer_t * p0, *p1;
          u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
          const mpls_unicast_header_t *mpls0, *mpls1;
          const dpo_id_t *dpo0, *dpo1;

          /* Prefetch next iteration. */
          {
            vlib_buffer_t * p2, * p3;

            p2 = vlib_get_buffer (vm, from[2]);
            p3 = vlib_get_buffer (vm, from[3]);

            vlib_prefetch_buffer_header (p2, STORE);
            vlib_prefetch_buffer_header (p3, STORE);

            CLIB_PREFETCH (p2->data, sizeof (mpls0[0]), STORE);
            CLIB_PREFETCH (p3->data, sizeof (mpls0[0]), STORE);
          }

          pi0 = to_next[0] = from[0];
          pi1 = to_next[1] = from[1];

          from += 2;
          n_left_from -= 2;
          to_next += 2;
          n_left_to_next -= 2;

          p0 = vlib_get_buffer (vm, pi0);
          p1 = vlib_get_buffer (vm, pi1);

          mpls0 = vlib_buffer_get_current (p0);
          mpls1 = vlib_buffer_get_current (p1);
          lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
          lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];

          lb0 = load_balance_get(lbi0);
          lb1 = load_balance_get(lbi1);

          /*
           * this node is for via FIBs we can re-use the hash value from the
           * to node if present.
           * We don't want to use the same hash value at each level in the recursion
           * graph as that would lead to polarisation
           */
          hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
          hc1 = vnet_buffer (p1)->ip.flow_hash = 0;

          if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
          {
              if (PREDICT_TRUE (vnet_buffer(p0)->ip.flow_hash))
              {
                  hc0 = vnet_buffer(p0)->ip.flow_hash = vnet_buffer(p0)->ip.flow_hash >> 1;
              }
              else
              {
                  hc0 = vnet_buffer(p0)->ip.flow_hash = mpls_compute_flow_hash(mpls0, hc0);
              }
          }
          if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
          {
              if (PREDICT_TRUE (vnet_buffer(p1)->ip.flow_hash))
              {
                  hc1 = vnet_buffer(p1)->ip.flow_hash = vnet_buffer(p1)->ip.flow_hash >> 1;
              }
              else
              {
                  hc1 = vnet_buffer(p1)->ip.flow_hash = mpls_compute_flow_hash(mpls1, hc1);
              }
          }

          dpo0 = load_balance_get_bucket_i(lb0, hc0 & (lb0->lb_n_buckets_minus_1));
          dpo1 = load_balance_get_bucket_i(lb1, hc1 & (lb1->lb_n_buckets_minus_1));

          next0 = dpo0->dpoi_next_node;
          next1 = dpo1->dpoi_next_node;

          vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
          vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;

          vlib_increment_combined_counter
              (cm, cpu_index, lbi0, 1,
               vlib_buffer_length_in_chain (vm, p0));
          vlib_increment_combined_counter
              (cm, cpu_index, lbi1, 1,
               vlib_buffer_length_in_chain (vm, p1));

          if (PREDICT_FALSE(p0->flags & VLIB_BUFFER_IS_TRACED))
          {
              mpls_load_balance_trace_t *tr = vlib_add_trace (vm, node,
                                                              p0, sizeof (*tr));
              tr->next_index = next0;
              tr->lb_index = lbi0;
              tr->hash = hc0;
          }

          vlib_validate_buffer_enqueue_x2 (vm, node, next,
                                           to_next, n_left_to_next,
                                           pi0, pi1, next0, next1);
       }

      while (n_left_from > 0 && n_left_to_next > 0)
        {
          mpls_lookup_next_t next0;
          const load_balance_t *lb0;
          vlib_buffer_t * p0;
          u32 pi0, lbi0, hc0;
          const mpls_unicast_header_t *mpls0;
          const dpo_id_t *dpo0;

          pi0 = from[0];
          to_next[0] = pi0;
          from += 1;
          to_next += 1;
          n_left_to_next -= 1;
          n_left_from -= 1;

          p0 = vlib_get_buffer (vm, pi0);

          mpls0 = vlib_buffer_get_current (p0);
          lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];

          lb0 = load_balance_get(lbi0);

          hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
          if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
          {
              if (PREDICT_TRUE (vnet_buffer(p0)->ip.flow_hash))
              {
                  hc0 = vnet_buffer(p0)->ip.flow_hash = vnet_buffer(p0)->ip.flow_hash >> 1;
              }
              else
              {
                  hc0 = vnet_buffer(p0)->ip.flow_hash = mpls_compute_flow_hash(mpls0, hc0);
              }
          }

          dpo0 = load_balance_get_bucket_i(lb0, hc0 & (lb0->lb_n_buckets_minus_1));

          next0 = dpo0->dpoi_next_node;
          vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;

          vlib_increment_combined_counter
              (cm, cpu_index, lbi0, 1,
               vlib_buffer_length_in_chain (vm, p0));

          vlib_validate_buffer_enqueue_x1 (vm, node, next,
                                           to_next, n_left_to_next,
                                           pi0, next0);
        }

      vlib_put_next_frame (vm, node, next, n_left_to_next);
    }

  return frame->n_vectors;
}

VLIB_REGISTER_NODE (mpls_load_balance_node) = {
  .function = mpls_load_balance,
  .name = "mpls-load-balance",
  .vector_size = sizeof (u32),
  .sibling_of = "mpls-lookup",

  .format_trace = format_mpls_load_balance_trace,
};

VLIB_NODE_FUNCTION_MULTIARCH (mpls_load_balance_node, mpls_load_balance)