summaryrefslogtreecommitdiffstats
path: root/src/plugins/nat/nat44-ed/nat44_ed.c
diff options
context:
space:
mode:
authorKlement Sekera <ksekera@cisco.com>2021-03-15 19:52:57 +0100
committerKlement Sekera <ksekera@cisco.com>2021-03-15 22:07:43 +0100
commit3a5bd85476311d4ad2b0cbbca3f98d53f13a3afb (patch)
tree2a17c25f10fe3227b12460f3bf82743856d18d47 /src/plugins/nat/nat44-ed/nat44_ed.c
parent7db75dd0d32d92674bbb7aa1b23341e8a349e05a (diff)
nat: get rid of worker selection callbacks
Make code easier to read and debug. Type: improvement Signed-off-by: Klement Sekera <ksekera@cisco.com> Change-Id: Ib52a4cdd3bcdcc475053aa32af3964c00859e1cd
Diffstat (limited to 'src/plugins/nat/nat44-ed/nat44_ed.c')
-rw-r--r--src/plugins/nat/nat44-ed/nat44_ed.c49
1 files changed, 21 insertions, 28 deletions
diff --git a/src/plugins/nat/nat44-ed/nat44_ed.c b/src/plugins/nat/nat44-ed/nat44_ed.c
index 007296e065f..c01a2454e0f 100644
--- a/src/plugins/nat/nat44-ed/nat44_ed.c
+++ b/src/plugins/nat/nat44-ed/nat44_ed.c
@@ -176,13 +176,6 @@ static void nat44_ed_db_init (u32 translations, u32 translation_buckets);
static void nat44_ed_db_free ();
-static u32
-nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
- u32 rx_fib_index, u8 is_output);
-
-static u32 nat44_ed_get_worker_in2out_cb (vlib_buffer_t *b, ip4_header_t *ip,
- u32 rx_fib_index, u8 is_output);
-
u32 nat_calc_bihash_buckets (u32 n_elts);
u8 *
@@ -723,8 +716,8 @@ snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
ip4_header_t ip = {
.src_address = m->local_addr,
};
- vec_add1 (m->workers,
- sm->worker_in2out_cb (0, &ip, m->fib_index, 0));
+ vec_add1 (m->workers, nat44_ed_get_in2out_worker_index (
+ 0, &ip, m->fib_index, 0));
tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
}
else
@@ -971,7 +964,8 @@ nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
.src_address = locals[i].addr,
};
bitmap = clib_bitmap_set (
- bitmap, sm->worker_in2out_cb (0, &ip, m->fib_index, 0), 1);
+ bitmap,
+ nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0), 1);
}
}
@@ -1051,7 +1045,7 @@ nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
};
tsm = vec_elt_at_index (
sm->per_thread_data,
- sm->worker_in2out_cb (0, &ip, m->fib_index, 0));
+ nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
}
else
tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
@@ -1164,9 +1158,9 @@ nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
ip4_header_t ip = {
.src_address = local->addr,
};
- tsm =
- vec_elt_at_index (sm->per_thread_data,
- sm->worker_in2out_cb (0, &ip, m->fib_index, 0));
+ tsm = vec_elt_at_index (
+ sm->per_thread_data,
+ nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
}
else
tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
@@ -1197,7 +1191,8 @@ nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
ip4_header_t ip;
ip.src_address.as_u32 = local->addr.as_u32,
bitmap = clib_bitmap_set (
- bitmap, sm->worker_in2out_cb (0, &ip, local->fib_index, 0), 1);
+ bitmap,
+ nat44_ed_get_in2out_worker_index (0, &ip, local->fib_index, 0), 1);
}
}
@@ -2140,9 +2135,6 @@ nat44_plugin_enable (nat44_config_t c)
sm->outside_fib_index = fib_table_find_or_create_and_lock (
FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
- sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
- sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
-
nat44_ed_db_init (sm->max_translations_per_thread, sm->translation_buckets);
nat_affinity_enable ();
@@ -2423,8 +2415,8 @@ snat_static_mapping_match (vlib_main_t *vm, snat_main_t *sm,
.src_address = local->addr,
};
- if (sm->worker_in2out_cb (0, &ip, m->fib_index, 0) ==
- thread_index)
+ if (nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index,
+ 0) == thread_index)
{
vec_add1 (tmp, i);
}
@@ -2499,9 +2491,9 @@ end:
return 0;
}
-static u32
-nat44_ed_get_worker_in2out_cb (vlib_buffer_t *b, ip4_header_t *ip,
- u32 rx_fib_index, u8 is_output)
+u32
+nat44_ed_get_in2out_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
+ u32 rx_fib_index, u8 is_output)
{
snat_main_t *sm = &snat_main;
u32 next_worker_index = sm->first_worker_index;
@@ -2602,9 +2594,9 @@ out:
return next_worker_index;
}
-static u32
-nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
- u32 rx_fib_index, u8 is_output)
+u32
+nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
+ u32 rx_fib_index, u8 is_output)
{
snat_main_t *sm = &snat_main;
clib_bihash_kv_8_8_t kv, value;
@@ -3162,8 +3154,9 @@ nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
if (sm->num_workers > 1)
- tsm = vec_elt_at_index (sm->per_thread_data,
- sm->worker_in2out_cb (0, &ip, fib_index, 0));
+ tsm = vec_elt_at_index (
+ sm->per_thread_data,
+ nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
else
tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
ef='#n397'>397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
#!/usr/bin/env python3
"""IP6 VRF Multi-instance Test Case HLD:

**NOTES:**
    - higher number of pg-ip6 interfaces causes problems => only 15 pg-ip6 \
    interfaces in 5 VRFs are tested
    - jumbo packets in configuration with 15 pg-ip6 interfaces leads to \
    problems too

**config 1**
    - add 15 pg-ip6 interfaces
    - configure 5 hosts per pg-ip6 interface
    - configure 4 VRFs
    - add 3 pg-ip6 interfaces per VRF

**test 1**
    - send IP6 packets between all pg-ip6 interfaces in all VRF groups

**verify 1**
    - check VRF data by parsing output of ip_route_dump API command
    - all packets received correctly in case of pg-ip6 interfaces in the same
    VRF
    - no packet received in case of pg-ip6 interfaces not in VRF
    - no packet received in case of pg-ip6 interfaces in different VRFs

**config 2**
    - reset 2 VRFs

**test 2**
    - send IP6 packets between all pg-ip6 interfaces in all VRF groups

**verify 2**
    - check VRF data by parsing output of ip_route_dump API command
    - all packets received correctly in case of pg-ip6 interfaces in the same
    VRF
    - no packet received in case of pg-ip6 interfaces not in VRF
    - no packet received in case of pg-ip6 interfaces in different VRFs

**config 3**
    - add 1 of reset VRFs and 1 new VRF

**test 3**
    - send IP6 packets between all pg-ip6 interfaces in all VRF groups

**verify 3**
    - check VRF data by parsing output of ip_route_dump API command
    - all packets received correctly in case of pg-ip6 interfaces in the same
    VRF
    - no packet received in case of pg-ip6 interfaces not in VRF
    - no packet received in case of pg-ip6 interfaces in different VRFs

**config 4**
    - reset all VRFs (i.e. no VRF except VRF=0 created)

**test 4**
    - send IP6 packets between all pg-ip6 interfaces in all VRF groups

**verify 4**
    - check VRF data by parsing output of ip_route_dump API command
    - all packets received correctly in case of pg-ip6 interfaces in the same
    VRF
    - no packet received in case of pg-ip6 interfaces not in VRF
    - no packet received in case of pg-ip6 interfaces in different VRFs
"""

import unittest
import random
import socket

from scapy.packet import Raw
from scapy.layers.l2 import Ether
from scapy.layers.inet6 import UDP, IPv6, ICMPv6ND_NS, ICMPv6ND_RA, \
    RouterAlert, IPv6ExtHdrHopByHop
from scapy.utils6 import in6_ismaddr, in6_isllsnmaddr, in6_getAddrType
from scapy.pton_ntop import inet_ntop

from framework import VppTestCase, VppTestRunner
from util import ppp
from vrf import VRFState


def is_ipv6_misc_ext(p):
    """ Is packet one of uninteresting IPv6 broadcasts (extended to filter out
    ICMPv6 Neighbor Discovery - Neighbor Advertisement packets too)? """
    if p.haslayer(ICMPv6ND_RA):
        if in6_ismaddr(p[IPv6].dst):
            return True
    if p.haslayer(ICMPv6ND_NS):
        if in6_isllsnmaddr(p[IPv6].dst):
            return True
    if p.haslayer(IPv6ExtHdrHopByHop):
        for o in p[IPv6ExtHdrHopByHop].options:
            if isinstance(o, RouterAlert):
                return True
    return False


class TestIP6VrfMultiInst(VppTestCase):
    """ IP6 VRF  Multi-instance Test Case """

    @classmethod
    def setUpClass(cls):
        """
        Perform standard class setup (defined by class method setUpClass in
        class VppTestCase) before running the test case, set test case related
        variables and configure VPP.
        """
        super(TestIP6VrfMultiInst, cls).setUpClass()

        # Test variables
        cls.hosts_per_pg = 5
        cls.nr_of_vrfs = 5
        cls.pg_ifs_per_vrf = 3

        try:
            # Create pg interfaces
            cls.create_pg_interfaces(
                range(cls.nr_of_vrfs * cls.pg_ifs_per_vrf))

            # Packet flows mapping pg0 -> pg1, pg2 etc.
            cls.flows = dict()
            for i in range(len(cls.pg_interfaces)):
                multiplicand = i // cls.pg_ifs_per_vrf
                pg_list = [
                    cls.pg_interfaces[multiplicand * cls.pg_ifs_per_vrf + j]
                    for j in range(cls.pg_ifs_per_vrf)
                    if (multiplicand * cls.pg_ifs_per_vrf + j) != i]
                cls.flows[cls.pg_interfaces[i]] = pg_list

            # Packet sizes - jumbo packet (9018 bytes) skipped
            cls.pg_if_packet_sizes = [64, 512, 1518]

            # Set up all interfaces
            for pg_if in cls.pg_interfaces:
                pg_if.admin_up()
                pg_if.generate_remote_hosts(cls.hosts_per_pg)

            # Create list of VRFs
            cls.vrf_list = list()

            # Create list of reset VRFs
            cls.vrf_reset_list = list()

            # Create list of pg_interfaces in VRFs
            cls.pg_in_vrf = list()

            # Create list of pg_interfaces not in VRFs
            cls.pg_not_in_vrf = [pg_if for pg_if in cls.pg_interfaces]

            # Create mapping of pg_interfaces to VRF IDs
            cls.pg_if_by_vrf_id = dict()
            for i in range(cls.nr_of_vrfs):
                vrf_id = i + 1
                pg_list = [
                    cls.pg_interfaces[i * cls.pg_ifs_per_vrf + j]
                    for j in range(cls.pg_ifs_per_vrf)]
                cls.pg_if_by_vrf_id[vrf_id] = pg_list

        except Exception:
            super(TestIP6VrfMultiInst, cls).tearDownClass()
            raise

    @classmethod
    def tearDownClass(cls):
        super(TestIP6VrfMultiInst, cls).tearDownClass()

    def setUp(self):
        """
        Clear trace and packet infos before running each test.
        """
        super(TestIP6VrfMultiInst, self).setUp()
        self.reset_packet_infos()

    def tearDown(self):
        """
        Show various debug prints after each test.
        """
        super(TestIP6VrfMultiInst, self).tearDown()

    def show_commands_at_teardown(self):
        self.logger.info(self.vapi.ppcli("show ip6 fib"))
        self.logger.info(self.vapi.ppcli("show ip6 neighbors"))

    def create_vrf_and_assign_interfaces(self, count, start=1):
        """
        Create required number of FIB tables / VRFs, put 3 pg-ip6 interfaces
        to every FIB table / VRF.

        :param int count: Number of FIB tables / VRFs to be created.
        :param int start: Starting number of the FIB table / VRF ID. \
        (Default value = 1)
        """
        for i in range(count):
            vrf_id = i + start
            pg_if = self.pg_if_by_vrf_id[vrf_id][0]
            self.vapi.ip_table_add_del(is_add=1,
                                       table={'table_id': vrf_id, 'is_ip6': 1})
            self.logger.info("IPv6 VRF ID %d created" % vrf_id)
            if vrf_id not in self.vrf_list:
                self.vrf_list.append(vrf_id)
            if vrf_id in self.vrf_reset_list:
                self.vrf_reset_list.remove(vrf_id)
            for j in range(self.pg_ifs_per_vrf):
                pg_if = self.pg_if_by_vrf_id[vrf_id][j]
                pg_if.set_table_ip6(vrf_id)
                self.logger.info("pg-interface %s added to IPv6 VRF ID %d"
                                 % (pg_if.name, vrf_id))
                if pg_if not in self.pg_in_vrf:
                    self.pg_in_vrf.append(pg_if)
                if pg_if in self.pg_not_in_vrf:
                    self.pg_not_in_vrf.remove(pg_if)
                pg_if.config_ip6()
                pg_if.disable_ipv6_ra()
                pg_if.configure_ipv6_neighbors()
        self.logger.debug(self.vapi.ppcli("show ip6 fib"))
        self.logger.debug(self.vapi.ppcli("show ip6 neighbors"))

    def reset_vrf_and_remove_from_vrf_list(self, vrf_id):
        """
        Reset required FIB table / VRF and remove it from VRF list.

        :param int vrf_id: The FIB table / VRF ID to be reset.
        """
        self.vapi.ip_table_flush(table={'table_id': vrf_id, 'is_ip6': 1})
        if vrf_id in self.vrf_list:
            self.vrf_list.remove(vrf_id)
        if vrf_id not in self.vrf_reset_list:
            self.vrf_reset_list.append(vrf_id)
        for j in range(self.pg_ifs_per_vrf):
            pg_if = self.pg_if_by_vrf_id[vrf_id][j]
            pg_if.unconfig_ip6()
            if pg_if in self.pg_in_vrf:
                self.pg_in_vrf.remove(pg_if)
            if pg_if not in self.pg_not_in_vrf:
                self.pg_not_in_vrf.append(pg_if)
        self.logger.info("IPv6 VRF ID %d reset finished" % vrf_id)
        self.logger.debug(self.vapi.ppcli("show ip6 fib"))
        self.logger.debug(self.vapi.ppcli("show ip6 neighbors"))
        self.vapi.ip_table_add_del(is_add=0,
                                   table={'table_id': vrf_id, 'is_ip6': 1})

    def create_stream(self, src_if, packet_sizes):
        """
        Create input packet stream for defined interface using hosts list.

        :param object src_if: Interface to create packet stream for.
        :param list packet_sizes: List of required packet sizes.
        :return: Stream of packets.
        """
        pkts = []
        src_hosts = src_if.remote_hosts
        for dst_if in self.flows[src_if]:
            for dst_host in dst_if.remote_hosts:
                src_host = random.choice(src_hosts)
                pkt_info = self.create_packet_info(src_if, dst_if)
                payload = self.info_to_payload(pkt_info)
                p = (Ether(dst=src_if.local_mac, src=src_host.mac) /
                     IPv6(src=src_host.ip6, dst=dst_host.ip6) /
                     UDP(sport=1234, dport=1234) /
                     Raw(payload))
                pkt_info.data = p.copy()
                size = random.choice(packet_sizes)
                self.extend_packet(p, size)
                pkts.append(p)
        self.logger.debug("Input stream created for port %s. Length: %u pkt(s)"
                          % (src_if.name, len(pkts)))
        return pkts

    def create_stream_crosswise_vrf(self, src_if, vrf_id, packet_sizes):
        """
        Create input packet stream for negative test for leaking across
        different VRFs for defined interface using hosts list.

        :param object src_if: Interface to create packet stream for.
        :param int vrf_id: The FIB table / VRF ID where src_if is assigned.
        :param list packet_sizes: List of required packet sizes.
        :return: Stream of packets.
        """
        pkts = []
        src_hosts = src_if.remote_hosts
        vrf_lst = list(self.vrf_list)
        vrf_lst.remove(vrf_id)
        for vrf in vrf_lst:
            for dst_if in self.pg_if_by_vrf_id[vrf]:
                for dst_host in dst_if.remote_hosts:
                    src_host = random.choice(src_hosts)
                    pkt_info = self.create_packet_info(src_if, dst_if)
                    payload = self.info_to_payload(pkt_info)
                    p = (Ether(dst=src_if.local_mac, src=src_host.mac) /
                         IPv6(src=src_host.ip6, dst=dst_host.ip6) /
                         UDP(sport=1234, dport=1234) /
                         Raw(payload))
                    pkt_info.data = p.copy()
                    size = random.choice(packet_sizes)
                    self.extend_packet(p, size)
                    pkts.append(p)
        self.logger.debug("Input stream created for port %s. Length: %u pkt(s)"
                          % (src_if.name, len(pkts)))
        return pkts

    def verify_capture(self, pg_if, capture):
        """
        Verify captured input packet stream for defined interface.

        :param object pg_if: Interface to verify captured packet stream for.
        :param list capture: Captured packet stream.
        """
        last_info = dict()
        for i in self.pg_interfaces:
            last_info[i.sw_if_index] = None
        dst_sw_if_index = pg_if.sw_if_index
        for packet in capture:
            try:
                ip = packet[IPv6]
                udp = packet[UDP]
                payload_info = self.payload_to_info(packet[Raw])
                packet_index = payload_info.index
                self.assertEqual(payload_info.dst, dst_sw_if_index)
                self.logger.debug("Got packet on port %s: src=%u (id=%u)" %
                                  (pg_if.name, payload_info.src, packet_index))
                next_info = self.get_next_packet_info_for_interface2(
                    payload_info.src, dst_sw_if_index,
                    last_info[payload_info.src])
                last_info[payload_info.src] = next_info
                self.assertIsNotNone(next_info)
                self.assertEqual(packet_index, next_info.index)
                saved_packet = next_info.data
                # Check standard fields
                self.assertEqual(ip.src, saved_packet[IPv6].src)
                self.assertEqual(ip.dst, saved_packet[IPv6].dst)
                self.assertEqual(udp.sport, saved_packet[UDP].sport)
                self.assertEqual(udp.dport, saved_packet[UDP].dport)
            except:
                self.logger.error(ppp("Unexpected or invalid packet:", packet))
                raise
        for i in self.pg_interfaces:
            remaining_packet = self.get_next_packet_info_for_interface2(
                i, dst_sw_if_index, last_info[i.sw_if_index])
            self.assertIsNone(
                remaining_packet,
                "Port %u: Packet expected from source %u didn't arrive" %
                (dst_sw_if_index, i.sw_if_index))

    def verify_vrf(self, vrf_id):
        """
        Check if the FIB table / VRF ID is configured.

        :param int vrf_id: The FIB table / VRF ID to be verified.
        :return: 1 if the FIB table / VRF ID is configured, otherwise return 0.
        """
        ip6_fib_dump = self.vapi.ip_route_dump(vrf_id, True)
        vrf_exist = len(ip6_fib_dump)
        vrf_count = 0
        for ip6_fib_details in ip6_fib_dump:
            addr = ip6_fib_details.route.prefix.network_address
            found = False
            for pg_if in self.pg_if_by_vrf_id[vrf_id]:
                if found:
                    break
                for host in pg_if.remote_hosts:
                    if str(addr) == host.ip6:
                        vrf_count += 1
                        found = True
                        break
        if not vrf_exist and vrf_count == 0:
            self.logger.info("IPv6 VRF ID %d is not configured" % vrf_id)
            return VRFState.not_configured
        elif vrf_exist and vrf_count == 0:
            self.logger.info("IPv6 VRF ID %d has been reset" % vrf_id)
            return VRFState.reset
        else:
            self.logger.info("IPv6 VRF ID %d is configured" % vrf_id)
            return VRFState.configured

    def run_verify_test(self):
        """
        Create packet streams for all configured pg interfaces, send all \
        prepared packet streams and verify that:
            - all packets received correctly on all pg-ip6 interfaces assigned
              to VRFs
            - no packet received on all pg-ip6 interfaces not assigned to VRFs

        :raise RuntimeError: If no packet captured on pg-ip6 interface assigned
            to VRF or if any packet is captured on pg-ip6 interface not
            assigned to VRF.
        """
        # Test
        # Create incoming packet streams for packet-generator interfaces
        for pg_if in self.pg_interfaces:
            pkts = self.create_stream(pg_if, self.pg_if_packet_sizes)
            pg_if.add_stream(pkts)

        # Enable packet capture and start packet sending
        self.pg_enable_capture(self.pg_interfaces)
        self.pg_start()

        # Verify
        # Verify outgoing packet streams per packet-generator interface
        for pg_if in self.pg_interfaces:
            if pg_if in self.pg_in_vrf:
                capture = pg_if.get_capture(remark="interface is in VRF")
                self.verify_capture(pg_if, capture)
            elif pg_if in self.pg_not_in_vrf:
                pg_if.assert_nothing_captured(remark="interface is not in VRF",
                                              filter_out_fn=is_ipv6_misc_ext)
                self.logger.debug("No capture for interface %s" % pg_if.name)
            else:
                raise Exception("Unknown interface: %s" % pg_if.name)

    def run_crosswise_vrf_test(self):
        """
        Create packet streams for every pg-ip6 interface in VRF towards all
        pg-ip6 interfaces in other VRFs, send all prepared packet streams and \
        verify that:
             - no packet received on all configured pg-ip6 interfaces

        :raise RuntimeError: If any packet is captured on any pg-ip6 interface.
        """
        # Test
        # Create incoming packet streams for packet-generator interfaces
        for vrf_id in self.vrf_list:
            for pg_if in self.pg_if_by_vrf_id[vrf_id]:
                pkts = self.create_stream_crosswise_vrf(
                    pg_if, vrf_id, self.pg_if_packet_sizes)
                pg_if.add_stream(pkts)

        # Enable packet capture and start packet sending
        self.pg_enable_capture(self.pg_interfaces)
        self.pg_start()

        # Verify
        # Verify outgoing packet streams per packet-generator interface
        for pg_if in self.pg_interfaces:
            pg_if.assert_nothing_captured(remark="interface is in other VRF",
                                          filter_out_fn=is_ipv6_misc_ext)
            self.logger.debug("No capture for interface %s" % pg_if.name)

    def test_ip6_vrf_01(self):
        """ IP6 VRF  Multi-instance test 1 - create 4 VRFs
        """
        # Config 1
        # Create 4 VRFs
        self.create_vrf_and_assign_interfaces(4)

        # Verify 1
        for vrf_id in self.vrf_list:
            self.assert_equal(self.verify_vrf(vrf_id),
                              VRFState.configured, VRFState)

        # Test 1
        self.run_verify_test()
        self.run_crosswise_vrf_test()

    def test_ip6_vrf_02(self):
        """ IP6 VRF  Multi-instance test 2 - reset 2 VRFs
        """
        # Config 2
        # Delete 2 VRFs
        self.reset_vrf_and_remove_from_vrf_list(1)
        self.reset_vrf_and_remove_from_vrf_list(2)

        # Verify 2
        for vrf_id in self.vrf_reset_list:
            self.assert_equal(self.verify_vrf(vrf_id),
                              VRFState.reset, VRFState)
        for vrf_id in self.vrf_list:
            self.assert_equal(self.verify_vrf(vrf_id),
                              VRFState.configured, VRFState)

        # Test 2
        self.run_verify_test()
        self.run_crosswise_vrf_test()

        # Reset routes learned from ICMPv6 Neighbor Discovery
        for vrf_id in self.vrf_reset_list:
            self.reset_vrf_and_remove_from_vrf_list(vrf_id)

    def test_ip6_vrf_03(self):
        """ IP6 VRF  Multi-instance 3 - add 2 VRFs
        """
        # Config 3
        # Add 1 of reset VRFs and 1 new VRF
        self.create_vrf_and_assign_interfaces(1)
        self.create_vrf_and_assign_interfaces(1, start=5)

        # Verify 3
        for vrf_id in self.vrf_reset_list:
            self.assert_equal(self.verify_vrf(vrf_id),
                              VRFState.reset, VRFState)
        for vrf_id in self.vrf_list:
            self.assert_equal(self.verify_vrf(vrf_id),
                              VRFState.configured, VRFState)

        # Test 3
        self.run_verify_test()
        self.run_crosswise_vrf_test()

        # Reset routes learned from ICMPv6 Neighbor Discovery
        for vrf_id in self.vrf_reset_list:
            self.reset_vrf_and_remove_from_vrf_list(vrf_id)

    def test_ip6_vrf_04(self):
        """ IP6 VRF  Multi-instance test 4 - reset 4 VRFs
        """
        # Config 4
        # Reset all VRFs (i.e. no VRF except VRF=0 configured)
        for i in range(len(self.vrf_list)):
            self.reset_vrf_and_remove_from_vrf_list(self.vrf_list[0])

        # Verify 4
        for vrf_id in self.vrf_reset_list:
            self.assert_equal(self.verify_vrf(vrf_id),
                              VRFState.reset, VRFState)
        vrf_list_length = len(self.vrf_list)
        self.assertEqual(
            vrf_list_length, 0,
            "List of configured VRFs is not empty: %s != 0" % vrf_list_length)

        # Test 4
        self.run_verify_test()
        self.run_crosswise_vrf_test()


if __name__ == '__main__':
    unittest.main(testRunner=VppTestRunner)