aboutsummaryrefslogtreecommitdiffstats
path: root/src/plugins/lacp
diff options
context:
space:
mode:
authorSteven <sluong@cisco.com>2018-03-29 10:35:41 -0700
committerDamjan Marion <dmarion.lists@gmail.com>2018-03-30 20:44:56 +0000
commitdd4889e21cb15df541cf3b2c2d8bb971ce460359 (patch)
tree8725f6c47f27e754e9712816c7b15e4db596c421 /src/plugins/lacp
parent70488ab7f87096534aaadd7eab30e8515ac3d60d (diff)
lacp: faster convergence for slow-rate config option
Do fast-rate if we are not yet synchronized with the partner. Stop sending LACP updates as a flash in the worker thread. Just expire the timer and let the lacp_process handle sending LACP PDU. Change-Id: I8b36fe74e752e7f45bd4a8d70512c0341cc197a1 Signed-off-by: Steven <sluong@cisco.com>
Diffstat (limited to 'src/plugins/lacp')
-rw-r--r--src/plugins/lacp/lacp.c8
-rw-r--r--src/plugins/lacp/mux_machine.c12
-rw-r--r--src/plugins/lacp/ptx_machine.c2
-rw-r--r--src/plugins/lacp/ptx_machine.h15
-rw-r--r--src/plugins/lacp/rx_machine.c6
-rw-r--r--src/plugins/lacp/tx_machine.c4
6 files changed, 29 insertions, 18 deletions
diff --git a/src/plugins/lacp/lacp.c b/src/plugins/lacp/lacp.c
index 5fe505a4f0f..378d22be931 100644
--- a/src/plugins/lacp/lacp.c
+++ b/src/plugins/lacp/lacp.c
@@ -89,7 +89,7 @@ lacp_send_ethernet_lacp_pdu (slave_if_t * sif)
vlib_put_frame_to_node (vm, hw->output_node_index, f);
- sif->last_lacpdu_time = vlib_time_now (vm);
+ sif->last_lacpdu_time = vlib_time_now (lm->vlib_main);
}
/*
@@ -106,8 +106,6 @@ lacp_pick_packet_template (slave_if_t * sif)
void
lacp_send_lacp_pdu (vlib_main_t * vm, slave_if_t * sif)
{
- lacp_main_t *lm = &lacp_main;
-
if (sif->mode != BOND_MODE_LACP)
{
lacp_stop_timer (&sif->periodic_timer);
@@ -133,10 +131,6 @@ lacp_send_lacp_pdu (vlib_main_t * vm, slave_if_t * sif)
default:
ASSERT (0);
}
-
- lacp_start_periodic_timer (lm->vlib_main, sif, sif->is_long_timeout ?
- LACP_SLOW_PERIODIC_TIMER :
- LACP_FAST_PERIODIC_TIMER);
}
void
diff --git a/src/plugins/lacp/mux_machine.c b/src/plugins/lacp/mux_machine.c
index f33c2642f20..9c7cf746660 100644
--- a/src/plugins/lacp/mux_machine.c
+++ b/src/plugins/lacp/mux_machine.c
@@ -98,14 +98,14 @@ lacp_mux_action_detached (void *p1, void *p2)
{
vlib_main_t *vm = (vlib_main_t *) p1;
slave_if_t *sif = (slave_if_t *) p2;
+ lacp_main_t *lm = &lacp_main;
lacp_detach_mux_from_aggregator (vm, sif);
sif->actor.state &= ~LACP_STATE_COLLECTING;
bond_disable_collecting_distributing (vm, sif);
sif->actor.state &= ~LACP_STATE_DISTRIBUTING;
sif->ntt = 1;
- lacp_machine_dispatch (&lacp_tx_machine, vm, sif, LACP_TX_EVENT_NTT,
- &sif->tx_state);
+ lacp_start_periodic_timer (lm->vlib_main, sif, 0);
if (sif->selected == LACP_PORT_SELECTED)
lacp_machine_dispatch (&lacp_mux_machine, vm, sif,
@@ -123,14 +123,14 @@ lacp_mux_action_attached (void *p1, void *p2)
{
vlib_main_t *vm = (vlib_main_t *) p1;
slave_if_t *sif = (slave_if_t *) p2;
+ lacp_main_t *lm = &lacp_main;
lacp_attach_mux_to_aggregator (vm, sif);
sif->actor.state &= ~LACP_STATE_COLLECTING;
bond_disable_collecting_distributing (vm, sif);
sif->actor.state &= ~LACP_STATE_DISTRIBUTING;
sif->ntt = 1;
- lacp_machine_dispatch (&lacp_tx_machine, vm, sif, LACP_TX_EVENT_NTT,
- &sif->tx_state);
+ lacp_start_periodic_timer (lm->vlib_main, sif, 0);
if ((sif->selected == LACP_PORT_UNSELECTED) ||
(sif->selected == LACP_PORT_STANDBY))
@@ -171,13 +171,13 @@ lacp_mux_action_collecting_distributing (void *p1, void *p2)
{
vlib_main_t *vm = (vlib_main_t *) p1;
slave_if_t *sif = (slave_if_t *) p2;
+ lacp_main_t *lm = &lacp_main;
sif->actor.state |= LACP_STATE_SYNCHRONIZATION | LACP_STATE_COLLECTING |
LACP_STATE_DISTRIBUTING;
bond_enable_collecting_distributing (vm, sif);
sif->ntt = 1;
- lacp_machine_dispatch (&lacp_tx_machine, vm, sif, LACP_TX_EVENT_NTT,
- &sif->tx_state);
+ lacp_start_periodic_timer (lm->vlib_main, sif, 0);
if ((sif->selected == LACP_PORT_UNSELECTED) ||
(sif->selected == LACP_PORT_STANDBY) ||
!(sif->partner.state & LACP_STATE_SYNCHRONIZATION))
diff --git a/src/plugins/lacp/ptx_machine.c b/src/plugins/lacp/ptx_machine.c
index ac83444b08a..5a1c6d88621 100644
--- a/src/plugins/lacp/ptx_machine.c
+++ b/src/plugins/lacp/ptx_machine.c
@@ -99,7 +99,7 @@ lacp_ptx_action_slow_periodic (void *p1, void *p2)
else
timer_expired = 0;
- lacp_start_periodic_timer (lm->vlib_main, sif, LACP_SLOW_PERIODIC_TIMER);
+ lacp_schedule_periodic_timer (lm->vlib_main, sif);
if (timer_expired || (sif->partner.state & LACP_STATE_LACP_TIMEOUT))
lacp_machine_dispatch (&lacp_ptx_machine, vm, sif,
diff --git a/src/plugins/lacp/ptx_machine.h b/src/plugins/lacp/ptx_machine.h
index a9af4bb89d3..6183b633ae1 100644
--- a/src/plugins/lacp/ptx_machine.h
+++ b/src/plugins/lacp/ptx_machine.h
@@ -69,6 +69,21 @@ lacp_start_periodic_timer (vlib_main_t * vm, slave_if_t * sif, u8 expiration)
sif->periodic_timer = vlib_time_now (vm) + expiration;
}
+static inline void
+lacp_schedule_periodic_timer (vlib_main_t * vm, slave_if_t * sif)
+{
+ // do fast rate if we are not yet synchronized
+ if (((sif->actor.state & (LACP_STATE_SYNCHRONIZATION |
+ LACP_STATE_COLLECTING |
+ LACP_STATE_DISTRIBUTING)) !=
+ (LACP_STATE_SYNCHRONIZATION | LACP_STATE_COLLECTING |
+ LACP_STATE_DISTRIBUTING))
+ && (sif->partner.state & LACP_STATE_AGGREGATION))
+ lacp_start_periodic_timer (vm, sif, LACP_FAST_PERIODIC_TIMER);
+ else
+ lacp_start_periodic_timer (vm, sif, LACP_SLOW_PERIODIC_TIMER);
+}
+
#endif /* __LACP_PTX_MACHINE_H__ */
/*
diff --git a/src/plugins/lacp/rx_machine.c b/src/plugins/lacp/rx_machine.c
index 374e3f84939..fd65aa3ae8e 100644
--- a/src/plugins/lacp/rx_machine.c
+++ b/src/plugins/lacp/rx_machine.c
@@ -174,6 +174,7 @@ lacp_update_ntt (vlib_main_t * vm, slave_if_t * sif)
lacp_pdu_t *lacpdu = (lacp_pdu_t *) sif->last_rx_pkt;
u8 states = LACP_STATE_LACP_ACTIVITY | LACP_STATE_LACP_TIMEOUT |
LACP_STATE_SYNCHRONIZATION | LACP_STATE_AGGREGATION;
+ lacp_main_t *lm = &lacp_main;
if ((states & lacpdu->partner.port_info.state) !=
(states & sif->actor.state)
@@ -181,8 +182,7 @@ lacp_update_ntt (vlib_main_t * vm, slave_if_t * sif)
sizeof (sif->actor) - sizeof (sif->actor.state)))
{
sif->ntt = 1;
- lacp_machine_dispatch (&lacp_tx_machine, vm, sif, LACP_TX_EVENT_NTT,
- &sif->tx_state);
+ lacp_start_periodic_timer (lm->vlib_main, sif, 0);
}
}
@@ -292,7 +292,7 @@ lacp_rx_action_expired (void *p1, void *p2)
timer_expired = 1;
else
timer_expired = 0;
- lacp_start_current_while_timer (lm->vlib_main, sif, LACP_SHORT_TIMOUT_TIME);
+ lacp_start_current_while_timer (lm->vlib_main, sif, sif->ttl_in_seconds);
sif->actor.state |= LACP_STATE_EXPIRED;
if (timer_expired)
lacp_machine_dispatch (&lacp_rx_machine, vm, sif,
diff --git a/src/plugins/lacp/tx_machine.c b/src/plugins/lacp/tx_machine.c
index 794b4f10d95..21b767c39d2 100644
--- a/src/plugins/lacp/tx_machine.c
+++ b/src/plugins/lacp/tx_machine.c
@@ -40,7 +40,8 @@ lacp_tx_action_transmit (void *p1, void *p2)
{
vlib_main_t *vm = (vlib_main_t *) p1;
slave_if_t *sif = (slave_if_t *) p2;
- f64 now = vlib_time_now (vm);
+ lacp_main_t *lm = &lacp_main;
+ f64 now = vlib_time_now (lm->vlib_main);
if (!lacp_timer_is_running (sif->periodic_timer))
return 0;
@@ -52,6 +53,7 @@ lacp_tx_action_transmit (void *p1, void *p2)
if (sif->ntt)
{
lacp_send_lacp_pdu (vm, sif);
+ lacp_schedule_periodic_timer (lm->vlib_main, sif);
}
sif->ntt = 0;
69 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
/*
 * Copyright (c) 2016 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vnet/adj/adj_nbr.h>
#include <vnet/adj/adj_internal.h>
#include <vnet/ethernet/arp_packet.h>
#include <vnet/fib/fib_walk.h>

/*
 * Vector Hash tables of neighbour (traditional) adjacencies
 *  Key: interface(for the vector index), address (and its proto),
 *       link-type/ether-type.
 */
static BVT(clib_bihash) **adj_nbr_tables[FIB_PROTOCOL_MAX];

// FIXME SIZE APPROPRIATELY. ASK DAVEB.
#define ADJ_NBR_DEFAULT_HASH_NUM_BUCKETS (64 * 64)
#define ADJ_NBR_DEFAULT_HASH_MEMORY_SIZE (32<<20)


#define ADJ_NBR_SET_KEY(_key, _lt, _nh)         \
{						\
    _key.key[0] = (_nh)->as_u64[0];		\
    _key.key[1] = (_nh)->as_u64[1];		\
    _key.key[2] = (_lt);			\
}

#define ADJ_NBR_ITF_OK(_proto, _itf)			\
    (((_itf) < vec_len(adj_nbr_tables[_proto])) &&	\
     (NULL != adj_nbr_tables[_proto][sw_if_index]))

static void
adj_nbr_insert (fib_protocol_t nh_proto,
		vnet_link_t link_type,
		const ip46_address_t *nh_addr,
		u32 sw_if_index,
		adj_index_t adj_index)
{
    BVT(clib_bihash_kv) kv;

    if (sw_if_index >= vec_len(adj_nbr_tables[nh_proto]))
    {
	vec_validate(adj_nbr_tables[nh_proto], sw_if_index);
    }
    if (NULL == adj_nbr_tables[nh_proto][sw_if_index])
    {
	adj_nbr_tables[nh_proto][sw_if_index] =
	    clib_mem_alloc_aligned(sizeof(BVT(clib_bihash)),
				   CLIB_CACHE_LINE_BYTES);
	memset(adj_nbr_tables[nh_proto][sw_if_index],
	       0,
	       sizeof(BVT(clib_bihash)));

	BV(clib_bihash_init) (adj_nbr_tables[nh_proto][sw_if_index],
			      "Adjacency Neighbour table",
			      ADJ_NBR_DEFAULT_HASH_NUM_BUCKETS,
			      ADJ_NBR_DEFAULT_HASH_MEMORY_SIZE);
    }

    ADJ_NBR_SET_KEY(kv, link_type, nh_addr);
    kv.value = adj_index;

    BV(clib_bihash_add_del) (adj_nbr_tables[nh_proto][sw_if_index], &kv, 1);
}

void
adj_nbr_remove (adj_index_t ai,
                fib_protocol_t nh_proto,
		vnet_link_t link_type,
		const ip46_address_t *nh_addr,
		u32 sw_if_index)
{
    BVT(clib_bihash_kv) kv;

    if (!ADJ_NBR_ITF_OK(nh_proto, sw_if_index))
	return;

    ADJ_NBR_SET_KEY(kv, link_type, nh_addr);
    kv.value = ai;

    BV(clib_bihash_add_del) (adj_nbr_tables[nh_proto][sw_if_index], &kv, 0);
}

static adj_index_t
adj_nbr_find (fib_protocol_t nh_proto,
	      vnet_link_t link_type,
	      const ip46_address_t *nh_addr,
	      u32 sw_if_index)
{
    BVT(clib_bihash_kv) kv;

    ADJ_NBR_SET_KEY(kv, link_type, nh_addr);

    if (!ADJ_NBR_ITF_OK(nh_proto, sw_if_index))
	return (ADJ_INDEX_INVALID);

    if (BV(clib_bihash_search)(adj_nbr_tables[nh_proto][sw_if_index],
			       &kv, &kv) < 0)
    {
	return (ADJ_INDEX_INVALID);
    }
    else
    {
	return (kv.value);
    }
}

static inline u32
adj_get_nd_node (fib_protocol_t proto)
{
    switch (proto) {
    case FIB_PROTOCOL_IP4:
	return (ip4_arp_node.index);
    case FIB_PROTOCOL_IP6:
	return (ip6_discover_neighbor_node.index);
    case FIB_PROTOCOL_MPLS:
	break;
    }
    ASSERT(0);
    return (ip4_arp_node.index);
}

/**
 * @brief Check and set feature flags if o/p interface has any o/p features.
 */
static void
adj_nbr_evaluate_feature (adj_index_t ai)
{
    ip_adjacency_t *adj;
    vnet_feature_main_t *fm = &feature_main;
    i16 feature_count;
    u8 arc_index;
    u32 sw_if_index;

    adj = adj_get(ai);

    switch (adj->ia_link)
    {
    case VNET_LINK_IP4:
        arc_index = ip4_main.lookup_main.output_feature_arc_index;
        break;
    case VNET_LINK_IP6:
        arc_index = ip6_main.lookup_main.output_feature_arc_index;
        break;
    case VNET_LINK_MPLS:
        arc_index = mpls_main.output_feature_arc_index;
        break;
    default:
        return;
    }

    sw_if_index = adj->rewrite_header.sw_if_index;
    if (vec_len(fm->feature_count_by_sw_if_index[arc_index]) > sw_if_index)
    {
        feature_count = fm->feature_count_by_sw_if_index[arc_index][sw_if_index];
        if (feature_count > 0)
            adj->rewrite_header.flags |= VNET_REWRITE_HAS_FEATURES;
    }

    return;
}

static ip_adjacency_t*
adj_nbr_alloc (fib_protocol_t nh_proto,
	       vnet_link_t link_type,
	       const ip46_address_t *nh_addr,
	       u32 sw_if_index)
{
    ip_adjacency_t *adj;

    adj = adj_alloc(nh_proto);

    adj_nbr_insert(nh_proto, link_type, nh_addr,
		   sw_if_index,
		   adj_get_index(adj));

    /*
     * since we just added the ADJ we have no rewrite string for it,
     * so its for ARP
     */
    adj->lookup_next_index = IP_LOOKUP_NEXT_ARP;
    adj->sub_type.nbr.next_hop = *nh_addr;
    adj->ia_link = link_type;
    adj->ia_nh_proto = nh_proto;
    adj->rewrite_header.sw_if_index = sw_if_index;
    memset(&adj->sub_type.midchain.next_dpo, 0,
           sizeof(adj->sub_type.midchain.next_dpo));

    adj_nbr_evaluate_feature (adj_get_index(adj));
    return (adj);
}

/*
 * adj_nbr_add_or_lock
 *
 * Add an adjacency for the neighbour requested.
 *
 * The key for an adj is:
 *   - the Next-hops protocol (i.e. v4 or v6)
 *   - the address of the next-hop
 *   - the interface the next-hop is reachable through
 */
adj_index_t
adj_nbr_add_or_lock (fib_protocol_t nh_proto,
		     vnet_link_t link_type,
		     const ip46_address_t *nh_addr,
		     u32 sw_if_index)
{
    adj_index_t adj_index;
    ip_adjacency_t *adj;

    adj_index = adj_nbr_find(nh_proto, link_type, nh_addr, sw_if_index);

    if (ADJ_INDEX_INVALID == adj_index)
    {
	vnet_main_t *vnm;

	vnm = vnet_get_main();
	adj = adj_nbr_alloc(nh_proto, link_type, nh_addr, sw_if_index);
	adj_index = adj_get_index(adj);
	adj_lock(adj_index);

	vnet_rewrite_init(vnm, sw_if_index,
			  adj_get_nd_node(nh_proto),
			  vnet_tx_node_index_for_sw_interface(vnm, sw_if_index),
			  &adj->rewrite_header);

	/*
	 * we need a rewrite where the destination IP address is converted
	 * to the appropriate link-layer address. This is interface specific.
	 * So ask the interface to do it.
	 */
	vnet_update_adjacency_for_sw_interface(vnm, sw_if_index, adj_index);
    }
    else
    {
	adj_lock(adj_index);
    }

    return (adj_index);
}

adj_index_t
adj_nbr_add_or_lock_w_rewrite (fib_protocol_t nh_proto,
			       vnet_link_t link_type,
			       const ip46_address_t *nh_addr,
			       u32 sw_if_index,
			       u8 *rewrite)
{
    adj_index_t adj_index;
    ip_adjacency_t *adj;

    adj_index = adj_nbr_find(nh_proto, link_type, nh_addr, sw_if_index);

    if (ADJ_INDEX_INVALID == adj_index)
    {
	adj = adj_nbr_alloc(nh_proto, link_type, nh_addr, sw_if_index);
	adj->rewrite_header.sw_if_index = sw_if_index;
    }
    else
    {
        adj = adj_get(adj_index);
    }

    adj_lock(adj_get_index(adj));
    adj_nbr_update_rewrite(adj_get_index(adj),
			   ADJ_NBR_REWRITE_FLAG_COMPLETE,
			   rewrite);

    return (adj_get_index(adj));
}

/**
 * adj_nbr_update_rewrite
 *
 * Update the adjacency's rewrite string. A NULL string implies the
 * rewirte is reset (i.e. when ARP/ND etnry is gone).
 * NB: the adj being updated may be handling traffic in the DP.
 */
void
adj_nbr_update_rewrite (adj_index_t adj_index,
			adj_nbr_rewrite_flag_t flags,
			u8 *rewrite)
{
    ip_adjacency_t *adj;

    ASSERT(ADJ_INDEX_INVALID != adj_index);

    adj = adj_get(adj_index);

    if (flags & ADJ_NBR_REWRITE_FLAG_COMPLETE)
    {
	/*
	 * update the adj's rewrite string and build the arc
	 * from the rewrite node to the interface's TX node
	 */
	adj_nbr_update_rewrite_internal(adj, IP_LOOKUP_NEXT_REWRITE,
					adj_get_rewrite_node(adj->ia_link),
					vnet_tx_node_index_for_sw_interface(
					    vnet_get_main(),
					    adj->rewrite_header.sw_if_index),
					rewrite);
    }
    else
    {
	adj_nbr_update_rewrite_internal(adj, IP_LOOKUP_NEXT_ARP,
					adj_get_nd_node(adj->ia_nh_proto),
					vnet_tx_node_index_for_sw_interface(
					    vnet_get_main(),
					    adj->rewrite_header.sw_if_index),
					rewrite);
    }
}

/**
 * adj_nbr_update_rewrite_internal
 *
 * Update the adjacency's rewrite string. A NULL string implies the
 * rewirte is reset (i.e. when ARP/ND etnry is gone).
 * NB: the adj being updated may be handling traffic in the DP.
 */
void
adj_nbr_update_rewrite_internal (ip_adjacency_t *adj,
				 ip_lookup_next_t adj_next_index,
				 u32 this_node,
				 u32 next_node,
				 u8 *rewrite)
{
    ip_adjacency_t *walk_adj;
    adj_index_t walk_ai;
    vlib_main_t * vm;
    u32 old_next;
    int do_walk;

    vm = vlib_get_main();
    old_next = adj->lookup_next_index;

    walk_ai = adj_get_index(adj);
    if (VNET_LINK_MPLS == adj->ia_link)
    {
        /*
         * The link type MPLS has no children in the control plane graph, it only
         * has children in the data-palne graph. The backwalk is up the former.
         * So we need to walk from its IP cousin.
         */
        walk_ai = adj_nbr_find(adj->ia_nh_proto,
                               fib_proto_to_link(adj->ia_nh_proto),
                               &adj->sub_type.nbr.next_hop,
                               adj->rewrite_header.sw_if_index);
    }

    /*
     * Don't call the walk re-entrantly
     */
    if (ADJ_INDEX_INVALID != walk_ai)
    {
        walk_adj = adj_get(walk_ai);
        if (ADJ_FLAG_SYNC_WALK_ACTIVE & walk_adj->ia_flags)
        {
            do_walk = 0;
        }
        else
        {
            /*
             * Prevent re-entrant walk of the same adj
             */
            walk_adj->ia_flags |= ADJ_FLAG_SYNC_WALK_ACTIVE;
            do_walk = 1;
        }
    }
    else
    {
        do_walk = 0;
    }

    /*
     * lock the adjacencies that are affected by updates this walk will provoke.
     * Since the aim of the walk is to update children to link to a different
     * DPO, this adj will no longer be in use and its lock count will drop to 0.
     * We don't want it to be deleted as part of this endevour.
     */
    adj_lock(adj_get_index(adj));
    adj_lock(walk_ai);

    /*
     * Updating a rewrite string is not atomic;
     *  - the rewrite string is too long to write in one instruction
     *  - when swapping from incomplete to complete, we also need to update
     *    the VLIB graph next-index of the adj.
     * ideally we would only want to suspend forwarding via this adj whilst we
     * do this, but we do not have that level of granularity - it's suspend all
     * worker threads or nothing.
     * The other chioces are:
     *  - to mark the adj down and back walk so child load-balances drop this adj
     *    from the set.
     *  - update the next_node index of this adj to point to error-drop
     * both of which will mean for MAC change we will drop for this adj
     * which is not acceptable. However, when the adj changes type (from
     * complete to incomplete and vice-versa) the child DPOs, which have the
     * VLIB graph next node index, will be sending packets to the wrong graph
     * node. So from the options above, updating the next_node of the adj to
     * be drop will work, but it relies on each graph node v4/v6/mpls, rewrite/
     * arp/midchain always be valid w.r.t. a mis-match of adj type and node type
     * (i.e. a rewrite adj in the arp node). This is not enforcable. Getting it
     * wrong will lead to hard to find bugs since its a race condition. So we
     * choose the more reliable method of updating the children to use the drop,
     * then switching adj's type, then updating the children again. Did I mention
     * that this doesn't happen often...
     * So we need to distinguish between the two cases:
     *  1 - mac change
     *  2 - adj type change
     */
    if (do_walk &&
        old_next != adj_next_index &&
        ADJ_INDEX_INVALID != walk_ai)
    {
        /*
         * the adj is changing type. we need to fix all children so that they
         * stack momentarily on a drop, while the adj changes. If we don't do
         * this  the children will send packets to a VLIB graph node that does
         * not correspond to the adj's type - and it goes downhill from there.
         */
	fib_node_back_walk_ctx_t bw_ctx = {
	    .fnbw_reason = FIB_NODE_BW_REASON_FLAG_ADJ_DOWN,
            /*
             * force this walk to be synchrous. if we don't and a node in the graph
             * (a heavily shared path-list) chooses to back-ground the walk (make it
             * async) then it will pause and we will do the adj update below, before
             * all the children are updated. not good.
             */
            .fnbw_flags = FIB_NODE_BW_FLAG_FORCE_SYNC,
	};

	fib_walk_sync(FIB_NODE_TYPE_ADJ, walk_ai, &bw_ctx);
    }

    /*
     * If we are just updating the MAC string of the adj (which we also can't
     * do atomically), then we need to stop packets switching through the adj.
     * We can't do that on a per-adj basis, so it's all the packets.
     * If we are updating the type, and we walked back to the children above,
     * then this barrier serves to flush the queues/frames.
     */
    vlib_worker_thread_barrier_sync(vm);

    adj->lookup_next_index = adj_next_index;

    if (NULL != rewrite)
    {
	/*
	 * new rewrite provided.
	 * fill in the adj's rewrite string, and build the VLIB graph arc.
	 */
	vnet_rewrite_set_data_internal(&adj->rewrite_header,
				       sizeof(adj->rewrite_data),
				       rewrite,
				       vec_len(rewrite));
	vec_free(rewrite);
    }
    else
    {
	vnet_rewrite_clear_data_internal(&adj->rewrite_header,
					 sizeof(adj->rewrite_data));
    }
    adj->rewrite_header.next_index = vlib_node_add_next(vlib_get_main(),
                                                        this_node,
                                                        next_node);

    /*
     * done with the rewirte update - let the workers loose.
     */
    vlib_worker_thread_barrier_release(vm);

    if (do_walk &&
        (old_next != adj->lookup_next_index) &&
        (ADJ_INDEX_INVALID != walk_ai))
    {
        /*
         * backwalk to the children so they can stack on the now updated
         * adjacency
         */
        fib_node_back_walk_ctx_t bw_ctx = {
	    .fnbw_reason = FIB_NODE_BW_REASON_FLAG_ADJ_UPDATE,
	};

	fib_walk_sync(FIB_NODE_TYPE_ADJ, walk_ai, &bw_ctx);
    }
    /*
     * Prevent re-entrant walk of the same adj
     */
    if (do_walk)
    {
        walk_adj->ia_flags &= ~ADJ_FLAG_SYNC_WALK_ACTIVE;
    }

    adj_unlock(adj_get_index(adj));
    adj_unlock(walk_ai);
}

typedef struct adj_db_count_ctx_t_ {
    u64 count;
} adj_db_count_ctx_t;

static void
adj_db_count (BVT(clib_bihash_kv) * kvp,
	      void *arg)
{
    adj_db_count_ctx_t * ctx = arg;
    ctx->count++;
}

u32
adj_nbr_db_size (void)
{
    adj_db_count_ctx_t ctx = {
	.count = 0,
    };
    fib_protocol_t proto;
    u32 sw_if_index = 0;

    for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
    {
	vec_foreach_index(sw_if_index, adj_nbr_tables[proto])
	{
	    if (NULL != adj_nbr_tables[proto][sw_if_index])
	    {
		BV(clib_bihash_foreach_key_value_pair) (
		    adj_nbr_tables[proto][sw_if_index],
		    adj_db_count,
		    &ctx);
	    }
	}
    }
    return (ctx.count);
}

/**
 * @brief Context for a walk of the adjacency neighbour DB
 */
typedef struct adj_walk_ctx_t_
{
    adj_walk_cb_t awc_cb;
    void *awc_ctx;
} adj_walk_ctx_t;

static void
adj_nbr_walk_cb (BVT(clib_bihash_kv) * kvp,
		 void *arg)
{
    adj_walk_ctx_t *ctx = arg;

    // FIXME: can't stop early...
    ctx->awc_cb(kvp->value, ctx->awc_ctx);
}

void
adj_nbr_walk (u32 sw_if_index,
	      fib_protocol_t adj_nh_proto,
	      adj_walk_cb_t cb,
	      void *ctx)
{
    if (!ADJ_NBR_ITF_OK(adj_nh_proto, sw_if_index))
	return;

    adj_walk_ctx_t awc = {
	.awc_ctx = ctx,
	.awc_cb = cb,
    };

    BV(clib_bihash_foreach_key_value_pair) (
	adj_nbr_tables[adj_nh_proto][sw_if_index],
	adj_nbr_walk_cb,
	&awc);
}

/**
 * @brief Context for a walk of the adjacency neighbour DB
 */
typedef struct adj_walk_nh_ctx_t_
{
    adj_walk_cb_t awc_cb;
    void *awc_ctx;
    const ip46_address_t *awc_nh;
} adj_walk_nh_ctx_t;

static void
adj_nbr_walk_nh_cb (BVT(clib_bihash_kv) * kvp,
		    void *arg)
{
    ip_adjacency_t *adj;
    adj_walk_nh_ctx_t *ctx = arg;

    adj = adj_get(kvp->value);

    if (!ip46_address_cmp(&adj->sub_type.nbr.next_hop, ctx->awc_nh)) 
	ctx->awc_cb(kvp->value, ctx->awc_ctx);
}

/**
 * @brief Walk adjacencies on a link with a given v4 next-hop.
 * that is visit the adjacencies with different link types.
 */
void
adj_nbr_walk_nh4 (u32 sw_if_index,
		 const ip4_address_t *addr,
		 adj_walk_cb_t cb,
		 void *ctx)
{
    if (!ADJ_NBR_ITF_OK(FIB_PROTOCOL_IP4, sw_if_index))
	return;

    ip46_address_t nh = {
	.ip4 = *addr,
    };

    adj_walk_nh_ctx_t awc = {
	.awc_ctx = ctx,
	.awc_cb = cb,
	.awc_nh = &nh,
    };

    BV(clib_bihash_foreach_key_value_pair) (
	adj_nbr_tables[FIB_PROTOCOL_IP4][sw_if_index],
	adj_nbr_walk_nh_cb,
	&awc);
}

/**
 * @brief Walk adjacencies on a link with a given v6 next-hop.
 * that is visit the adjacencies with different link types.
 */
void
adj_nbr_walk_nh6 (u32 sw_if_index,
		 const ip6_address_t *addr,
		 adj_walk_cb_t cb,
		 void *ctx)
{
    if (!ADJ_NBR_ITF_OK(FIB_PROTOCOL_IP6, sw_if_index))
	return;

    ip46_address_t nh = {
	.ip6 = *addr,
    };

    adj_walk_nh_ctx_t awc = {
	.awc_ctx = ctx,
	.awc_cb = cb,
	.awc_nh = &nh,
    };

    BV(clib_bihash_foreach_key_value_pair) (
	adj_nbr_tables[FIB_PROTOCOL_IP6][sw_if_index],
	adj_nbr_walk_nh_cb,
	&awc);
}

/**
 * @brief Walk adjacencies on a link with a given next-hop.
 * that is visit the adjacencies with different link types.
 */
void
adj_nbr_walk_nh (u32 sw_if_index,
		 fib_protocol_t adj_nh_proto,
		 const ip46_address_t *nh,
		 adj_walk_cb_t cb,
		 void *ctx)
{
    if (!ADJ_NBR_ITF_OK(adj_nh_proto, sw_if_index))
	return;

    adj_walk_nh_ctx_t awc = {
	.awc_ctx = ctx,
	.awc_cb = cb,
	.awc_nh = nh,
    };

    BV(clib_bihash_foreach_key_value_pair) (
	adj_nbr_tables[adj_nh_proto][sw_if_index],
	adj_nbr_walk_nh_cb,
	&awc);
}

/**
 * Flags associated with the interface state walks
 */
typedef enum adj_nbr_interface_flags_t_
{
    ADJ_NBR_INTERFACE_UP = (1 << 0),
} adj_nbr_interface_flags_t;

/**
 * Context for the state change walk of the DB
 */
typedef struct adj_nbr_interface_state_change_ctx_t_
{
    /**
     * Flags on the interface
     */
    adj_nbr_interface_flags_t flags;
} adj_nbr_interface_state_change_ctx_t;

static adj_walk_rc_t
adj_nbr_interface_state_change_one (adj_index_t ai,
                                    void *arg)
{
    /*
     * Back walk the graph to inform the forwarding entries
     * that this interface state has changed. Do this synchronously
     * since this is the walk that provides convergence
     */
    adj_nbr_interface_state_change_ctx_t *ctx = arg;

    fib_node_back_walk_ctx_t bw_ctx = {
	.fnbw_reason = ((ctx->flags & ADJ_NBR_INTERFACE_UP) ?
                        FIB_NODE_BW_REASON_FLAG_INTERFACE_UP :
                        FIB_NODE_BW_REASON_FLAG_INTERFACE_DOWN),
        /*
         * the force sync applies only as far as the first fib_entry.
         * And it's the fib_entry's we need to converge away from
         * the adjacencies on the now down link
         */
        .fnbw_flags = (!(ctx->flags & ADJ_NBR_INTERFACE_UP) ?
                       FIB_NODE_BW_FLAG_FORCE_SYNC :
                       0),
    };

    fib_walk_sync(FIB_NODE_TYPE_ADJ, ai, &bw_ctx);

    return (ADJ_WALK_RC_CONTINUE);
}

/**
 * @brief Registered function for SW interface state changes
 */
static clib_error_t *
adj_nbr_sw_interface_state_change (vnet_main_t * vnm,
                                   u32 sw_if_index,
                                   u32 flags)
{
    fib_protocol_t proto;

    /*
     * walk each adj on the interface and trigger a walk from that adj
     */
    for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
    {
	adj_nbr_interface_state_change_ctx_t ctx = {
	    .flags = ((flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ?
                      ADJ_NBR_INTERFACE_UP :
                      0),
	};

	adj_nbr_walk(sw_if_index, proto,
		     adj_nbr_interface_state_change_one,
		     &ctx);
    }

    return (NULL);
}

VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION_PRIO(
    adj_nbr_sw_interface_state_change,
    VNET_ITF_FUNC_PRIORITY_HIGH);

/**
 * @brief Invoked on each SW interface of a HW interface when the
 * HW interface state changes
 */
static void
adj_nbr_hw_sw_interface_state_change (vnet_main_t * vnm,
                                      u32 sw_if_index,
                                      void *arg)
{
    adj_nbr_interface_state_change_ctx_t *ctx = arg;
    fib_protocol_t proto;

    /*
     * walk each adj on the interface and trigger a walk from that adj
     */
    for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
    {
	adj_nbr_walk(sw_if_index, proto,
		     adj_nbr_interface_state_change_one,
		     ctx);
    }
}

/**
 * @brief Registered callback for HW interface state changes
 */
static clib_error_t *
adj_nbr_hw_interface_state_change (vnet_main_t * vnm,
                                   u32 hw_if_index,
                                   u32 flags)
{
    /*
     * walk SW interface on the HW
     */
    adj_nbr_interface_state_change_ctx_t ctx = {
        .flags = ((flags & VNET_HW_INTERFACE_FLAG_LINK_UP) ?
                  ADJ_NBR_INTERFACE_UP :
                  0),
    };

    vnet_hw_interface_walk_sw(vnm, hw_if_index,
                              adj_nbr_hw_sw_interface_state_change,
                              &ctx);

    return (NULL);
}

VNET_HW_INTERFACE_LINK_UP_DOWN_FUNCTION_PRIO(
    adj_nbr_hw_interface_state_change,
    VNET_ITF_FUNC_PRIORITY_HIGH);

static adj_walk_rc_t
adj_nbr_interface_delete_one (adj_index_t ai,
			      void *arg)
{
    /*
     * Back walk the graph to inform the forwarding entries
     * that this interface has been deleted.
     */
    fib_node_back_walk_ctx_t bw_ctx = {
	.fnbw_reason = FIB_NODE_BW_REASON_FLAG_INTERFACE_DELETE,
    };

    fib_walk_sync(FIB_NODE_TYPE_ADJ, ai, &bw_ctx);

    return (ADJ_WALK_RC_CONTINUE);
}

/**
 * adj_nbr_interface_add_del
 *
 * Registered to receive interface Add and delete notifications
 */
static clib_error_t *
adj_nbr_interface_add_del (vnet_main_t * vnm,
			   u32 sw_if_index,
			   u32 is_add)
{
    fib_protocol_t proto;

    if (is_add)
    {
	/*
	 * not interested in interface additions. we will not back walk
	 * to resolve paths through newly added interfaces. Why? The control
	 * plane should have the brains to add interfaces first, then routes.
	 * So the case where there are paths with a interface that matches
	 * one just created is the case where the path resolved through an
	 * interface that was deleted, and still has not been removed. The
	 * new interface added, is NO GUARANTEE that the interface being
	 * added now, even though it may have the same sw_if_index, is the
	 * same interface that the path needs. So tough!
	 * If the control plane wants these routes to resolve it needs to
	 * remove and add them again.
	 */
	return (NULL);
    }

    for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
    {
	adj_nbr_walk(sw_if_index, proto,
		     adj_nbr_interface_delete_one,
		     NULL);
    }

    return (NULL);
   
}

VNET_SW_INTERFACE_ADD_DEL_FUNCTION(adj_nbr_interface_add_del);


static adj_walk_rc_t
adj_nbr_show_one (adj_index_t ai,
		  void *arg)
{
    vlib_cli_output (arg, "[@%d]  %U",
                     ai,
                     format_ip_adjacency, ai,
		     FORMAT_IP_ADJACENCY_NONE);

    return (ADJ_WALK_RC_CONTINUE);
}

static clib_error_t *
adj_nbr_show (vlib_main_t * vm,
	      unformat_input_t * input,
	      vlib_cli_command_t * cmd)
{
    adj_index_t ai = ADJ_INDEX_INVALID;
    u32 sw_if_index = ~0;

    while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
    {
	if (unformat (input, "%d", &ai))
	    ;
	else if (unformat (input, "%U",
			   unformat_vnet_sw_interface, vnet_get_main(),
			   &sw_if_index))
	    ;
	else
	    break;
    }

    if (ADJ_INDEX_INVALID != ai)
    {
	vlib_cli_output (vm, "[@%d] %U",
                         ai,
                         format_ip_adjacency, ai,
			 FORMAT_IP_ADJACENCY_DETAIL);
    }
    else if (~0 != sw_if_index)
    {
	fib_protocol_t proto;

	for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
	{
	    adj_nbr_walk(sw_if_index, proto,
			 adj_nbr_show_one,
			 vm);
	}
    }
    else
    {
	fib_protocol_t proto;

	for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
	{
	    vec_foreach_index(sw_if_index, adj_nbr_tables[proto])
	    {
		adj_nbr_walk(sw_if_index, proto,
			     adj_nbr_show_one,
			     vm);
	    }
	}
    }

    return 0;
}

/*?
 * Show all neighbour adjacencies.
 * @cliexpar
 * @cliexstart{sh adj nbr}
 * [@2] ipv4 via 1.0.0.2 loop0: IP4: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
 * [@3] mpls via 1.0.0.2 loop0: MPLS_UNICAST: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
 * [@4] ipv4 via 1.0.0.3 loop0: IP4: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
 * [@5] mpls via 1.0.0.3 loop0: MPLS_UNICAST: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
 * @cliexend
 ?*/
VLIB_CLI_COMMAND (ip4_show_fib_command, static) = {
    .path = "show adj nbr",
    .short_help = "show adj nbr [<adj_index>] [interface]",
    .function = adj_nbr_show,
};

static ip46_type_t
adj_proto_to_46 (fib_protocol_t proto)
{
    switch (proto)
    {
    case FIB_PROTOCOL_IP4:
	return (IP46_TYPE_IP4);
    case FIB_PROTOCOL_IP6:
	return (IP46_TYPE_IP6);
    default:
	return (IP46_TYPE_IP4);
    }
    return (IP46_TYPE_IP4);
}

u8*
format_adj_nbr_incomplete (u8* s, va_list *ap)
{
    index_t index = va_arg(*ap, index_t);
    CLIB_UNUSED(u32 indent) = va_arg(*ap, u32);
    vnet_main_t * vnm = vnet_get_main();
    ip_adjacency_t * adj = adj_get(index);

    s = format (s, "arp-%U", format_vnet_link, adj->ia_link);
    s = format (s, ": via %U",
                format_ip46_address, &adj->sub_type.nbr.next_hop,
		adj_proto_to_46(adj->ia_nh_proto));
    s = format (s, " %U",
                format_vnet_sw_interface_name,
                vnm,
                vnet_get_sw_interface(vnm,
                                      adj->rewrite_header.sw_if_index));

    return (s);
}

u8*
format_adj_nbr (u8* s, va_list *ap)
{
    index_t index = va_arg(*ap, index_t);
    CLIB_UNUSED(u32 indent) = va_arg(*ap, u32);
    ip_adjacency_t * adj = adj_get(index);

    s = format (s, "%U", format_vnet_link, adj->ia_link);
    s = format (s, " via %U ",
		format_ip46_address, &adj->sub_type.nbr.next_hop,
		adj_proto_to_46(adj->ia_nh_proto));
    s = format (s, "%U",
		format_vnet_rewrite,
		&adj->rewrite_header, sizeof (adj->rewrite_data), 0);

    return (s);
}

static void
adj_dpo_lock (dpo_id_t *dpo)
{
    adj_lock(dpo->dpoi_index);
}
static void
adj_dpo_unlock (dpo_id_t *dpo)
{
    adj_unlock(dpo->dpoi_index);
}

static void
adj_mem_show (void)
{
    fib_show_memory_usage("Adjacency",
			  pool_elts(adj_pool),
			  pool_len(adj_pool),
			  sizeof(ip_adjacency_t));
}

const static dpo_vft_t adj_nbr_dpo_vft = {
    .dv_lock = adj_dpo_lock,
    .dv_unlock = adj_dpo_unlock,
    .dv_format = format_adj_nbr,
    .dv_mem_show = adj_mem_show,
};
const static dpo_vft_t adj_nbr_incompl_dpo_vft = {
    .dv_lock = adj_dpo_lock,
    .dv_unlock = adj_dpo_unlock,
    .dv_format = format_adj_nbr_incomplete,
};

/**
 * @brief The per-protocol VLIB graph nodes that are assigned to an adjacency
 *        object.
 *
 * this means that these graph nodes are ones from which a nbr is the
 * parent object in the DPO-graph.
 */
const static char* const nbr_ip4_nodes[] =
{
    "ip4-rewrite",
    NULL,
};
const static char* const nbr_ip6_nodes[] =
{
    "ip6-rewrite",
    NULL,
};
const static char* const nbr_mpls_nodes[] =
{
    "mpls-output",
    NULL,
};
const static char* const nbr_ethernet_nodes[] =
{
    "adj-l2-rewrite",
    NULL,
};
const static char* const * const nbr_nodes[DPO_PROTO_NUM] =
{
    [DPO_PROTO_IP4]  = nbr_ip4_nodes,
    [DPO_PROTO_IP6]  = nbr_ip6_nodes,
    [DPO_PROTO_MPLS] = nbr_mpls_nodes,
    [DPO_PROTO_ETHERNET] = nbr_ethernet_nodes,
};

const static char* const nbr_incomplete_ip4_nodes[] =
{
    "ip4-arp",
    NULL,
};
const static char* const nbr_incomplete_ip6_nodes[] =
{
    "ip6-discover-neighbor",
    NULL,
};
const static char* const nbr_incomplete_mpls_nodes[] =
{
    "mpls-adj-incomplete",
    NULL,
};

const static char* const * const nbr_incomplete_nodes[DPO_PROTO_NUM] =
{
    [DPO_PROTO_IP4]  = nbr_incomplete_ip4_nodes,
    [DPO_PROTO_IP6]  = nbr_incomplete_ip6_nodes,
    [DPO_PROTO_MPLS] = nbr_incomplete_mpls_nodes,
};

void
adj_nbr_module_init (void)
{
    dpo_register(DPO_ADJACENCY,
                 &adj_nbr_dpo_vft,
                 nbr_nodes);
    dpo_register(DPO_ADJACENCY_INCOMPLETE,
                 &adj_nbr_incompl_dpo_vft,
                 nbr_incomplete_nodes);
}