summaryrefslogtreecommitdiffstats
path: root/src/plugins/quic/quic.c
diff options
context:
space:
mode:
authorAloys Augustin <aloaugus@cisco.com>2019-07-30 15:46:56 +0200
committerFlorin Coras <florin.coras@gmail.com>2019-08-02 06:31:15 +0000
commit2253abad90f9350fa7fff7e1e7a31963f16c30ca (patch)
tree3a3a481bef6c8f55a47095ec2111f39e2f652c03 /src/plugins/quic/quic.c
parent778ce573180618d4f694f91bd662d92b3eada725 (diff)
quic: fix connection move with multiple workers
Change-Id: Ibd1012c2c1382de442ed294dd9e8a0ec89349fa1 Type: fix Signed-off-by: Aloys Augustin <aloaugus@cisco.com>
Diffstat (limited to 'src/plugins/quic/quic.c')
-rw-r--r--src/plugins/quic/quic.c48
1 files changed, 35 insertions, 13 deletions
diff --git a/src/plugins/quic/quic.c b/src/plugins/quic/quic.c
index 77e9c33dec9..af482d9dd36 100644
--- a/src/plugins/quic/quic.c
+++ b/src/plugins/quic/quic.c
@@ -1412,7 +1412,7 @@ quic_receive_connection (void *arg)
memcpy (new_ctx, temp_ctx, sizeof (quic_ctx_t));
- free (temp_ctx);
+ clib_mem_free (temp_ctx);
new_ctx->c_thread_index = thread_index;
new_ctx->c_c_index = new_ctx_id;
@@ -1434,22 +1434,17 @@ quic_transfer_connection (u32 ctx_index, u32 dest_thread)
{
tw_timer_wheel_1t_3w_1024sl_ov_t *tw;
quic_ctx_t *ctx, *temp_ctx;
- clib_bihash_kv_16_8_t kv;
- quicly_conn_t *conn;
u32 thread_index = vlib_get_thread_index ();
QUIC_DBG (2, "Transferring conn %u to thread %u", ctx_index, dest_thread);
- temp_ctx = malloc (sizeof (quic_ctx_t));
+ temp_ctx = clib_mem_alloc (sizeof (quic_ctx_t));
ASSERT (temp_ctx);
ctx = quic_ctx_get (ctx_index, thread_index);
memcpy (temp_ctx, ctx, sizeof (quic_ctx_t));
- /* Remove from lookup hash, timer wheel and thread-local pool */
- conn = ctx->conn;
- quic_make_connection_key (&kv, quicly_get_master_id (conn));
- clib_bihash_add_del_16_8 (&quic_main.connection_hash, &kv, 0 /* is_add */ );
+ /* Remove from timer wheel and thread-local pool */
if (ctx->timer_handle != QUIC_TIMER_HANDLE_INVALID)
{
tw = &quic_main.wrk_ctx[thread_index].timer_wheel;
@@ -1480,8 +1475,35 @@ quic_transfer_connection_rpc (void *arg)
*/
static void
quic_move_connection_to_thread (u32 ctx_index, u32 owner_thread,
- u32 to_thread)
+ u32 to_thread,
+ quicly_decoded_packet_t * packet)
{
+ clib_bihash_kv_16_8_t kv;
+ clib_bihash_16_8_t *h;
+
+ if (owner_thread == UINT32_MAX)
+ {
+ QUIC_DBG (3, "Connection already moving to right thread");
+ return;
+ }
+
+ /* Mark connection as moving in the conn map */
+ h = &quic_main.connection_hash;
+ quic_make_connection_key (&kv, &packet->cid.dest.plaintext);
+ if (clib_bihash_search_16_8 (h, &kv, &kv) != 0)
+ {
+ QUIC_DBG (0, "Bug: conn to move not found");
+ return;
+ }
+ kv.value |= (u64) UINT32_MAX << 32;
+ if (clib_bihash_add_del_16_8
+ (&quic_main.connection_hash, &kv, /* is_add */ 1))
+ {
+ QUIC_DBG (0, "Bug: cannot update conn in lookup hash");
+ return;
+ }
+
+ /* Send rpc to owner thread to move conn */
QUIC_DBG (2, "Requesting transfer of conn %u from thread %u", ctx_index,
owner_thread);
u64 arg = ((u64) ctx_index) << 32 | to_thread;
@@ -1531,7 +1553,6 @@ quic_session_connected_callback (u32 quic_app_index, u32 ctx_index,
ctx->udp_session_handle = session_handle (udp_session);
udp_session->opaque = ctx->parent_app_id;
- udp_session->session_state = SESSION_STATE_READY;
/* Init QUIC lib connection
* Generate required sockaddr & salen */
@@ -1693,6 +1714,7 @@ tx_end:
/*
* Returns 0 if a matching connection is found and is on the right thread.
+ * Otherwise returns -1.
* If a connection is found, even on the wrong thread, ctx_thread and ctx_index
* will be set.
*/
@@ -1714,7 +1736,7 @@ quic_find_packet_ctx (u32 * ctx_thread, u32 * ctx_index,
if (clib_bihash_search_16_8 (h, &kv, &kv) == 0)
{
u32 index = kv.value & UINT32_MAX;
- u8 thread_id = kv.value >> 32;
+ u32 thread_id = kv.value >> 32;
/* Check if this connection belongs to this thread, otherwise
* ask for it to be moved */
if (thread_id != caller_thread_index)
@@ -1969,11 +1991,11 @@ quic_app_rx_callback (session_t * udp_session)
ctx = quic_ctx_get (ctx_index, thread_index);
quic_receive (ctx, ctx->conn, packet);
}
- else if (ctx_thread != UINT32_MAX)
+ else if (ctx_index != UINT32_MAX)
{
/* Connection found but on wrong thread, ask move */
quic_move_connection_to_thread (ctx_index, ctx_thread,
- thread_index);
+ thread_index, &packet);
}
else if ((packet.octets.base[0] & QUICLY_PACKET_TYPE_BITMASK) ==
QUICLY_PACKET_TYPE_INITIAL)
n359'>359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478
#!/usr/bin/env python
"""L2BD Multi-instance Test Case HLD:

**NOTES:**
    - higher number of pg-l2 interfaces causes problems => only 15 pg-l2 \
    interfaces in 5 bridge domains are tested
    - jumbo packets in configuration with 14 l2-pg interfaces leads to \
    problems too

**config 1**
    - add 15 pg-l2 interfaces
    - configure one host per pg-l2 interface
    - configure 5 bridge domains (BD)
    - add 3 pg-l2 interfaces per BD

**test 1**
    - send L2 MAC frames between all pg-l2 interfaces of all BDs

**verify 1**
    - check BD data by parsing output of bridge_domain_dump API command
    - all packets received correctly

**config 2**
    - update data of 5 BD
        - disable learning, forwarding, flooding and uu_flooding for BD1
        - disable forwarding for BD2
        - disable flooding for BD3
        - disable uu_flooding for BD4
        - disable learning for BD5

**verify 2**
    - check BD data by parsing output of bridge_domain_dump API command

**config 3**
    - delete 2 BDs

**test 3**
    - send L2 MAC frames between all pg-l2 interfaces of all BDs
    - send L2 MAC frames between all pg-l2 interfaces formerly assigned to \
    deleted BDs

**verify 3**
    - check BD data by parsing output of bridge_domain_dump API command
    - all packets received correctly on all 3 pg-l2 interfaces assigned to BDs
    - no packet received on all 3 pg-l2 interfaces of all deleted BDs

**config 4**
    - add 2 BDs
    - add 3 pg-l2 interfaces per BD

**test 4**
    - send L2 MAC frames between all pg-l2 interfaces of all BDs

**verify 4**
    - check BD data by parsing output of bridge_domain_dump API command
    - all packets received correctly

**config 5**
    - delete 5 BDs

**verify 5**
    - check BD data by parsing output of bridge_domain_dump API command
"""

import unittest
import random

from scapy.packet import Raw
from scapy.layers.l2 import Ether
from scapy.layers.inet import IP, UDP

from framework import VppTestCase, VppTestRunner
from util import Host, ppp


class TestL2bdMultiInst(VppTestCase):
    """ L2BD Multi-instance Test Case """

    @classmethod
    def setUpClass(cls):
        """
        Perform standard class setup (defined by class method setUpClass in
        class VppTestCase) before running the test case, set test case related
        variables and configure VPP.
        """
        super(TestL2bdMultiInst, cls).setUpClass()

        try:
            # Create pg interfaces
            n_bd = 5
            cls.ifs_per_bd = ifs_per_bd = 3
            n_ifs = n_bd * ifs_per_bd
            cls.create_pg_interfaces(range(n_ifs))

            # Packet flows mapping pg0 -> pg1, pg2 etc.
            cls.flows = dict()
            for b in range(n_bd):
                bd_ifs = cls.bd_if_range(b + 1)
                for j in bd_ifs:
                    cls.flows[cls.pg_interfaces[j]] = [
                        cls.pg_interfaces[x] for x in bd_ifs if x != j]
                    assert(
                        len(cls.flows[cls.pg_interfaces[j]]) == ifs_per_bd - 1)

            # Mapping between packet-generator index and lists of test hosts
            cls.hosts_by_pg_idx = dict()

            # Create test host entries
            cls.create_hosts(5)

            # Packet sizes - jumbo packet (9018 bytes) skipped
            cls.pg_if_packet_sizes = [64, 512, 1518]

            # Set up all interfaces
            for i in cls.pg_interfaces:
                i.admin_up()

            # Create list of BDs
            cls.bd_list = list()

            # Create list of deleted BDs
            cls.bd_deleted_list = list()

            # Create list of pg_interfaces in BDs
            cls.pg_in_bd = list()

        except Exception:
            super(TestL2bdMultiInst, cls).tearDownClass()
            raise

    def setUp(self):
        """
        Clear trace and packet infos before running each test.
        """
        self.reset_packet_infos()
        super(TestL2bdMultiInst, self).setUp()

    def tearDown(self):
        """
        Show various debug prints after each test.
        """
        super(TestL2bdMultiInst, self).tearDown()
        if not self.vpp_dead:
            self.logger.info(self.vapi.ppcli("show l2fib verbose"))
            self.logger.info(self.vapi.ppcli("show bridge-domain"))

    @classmethod
    def create_hosts(cls, hosts_per_if):
        """
        Create required number of host MAC addresses and distribute them
        among interfaces. Create host IPv4 address for every host MAC
        address.

        :param int hosts_per_if: Number of hosts per if to create MAC/IPv4
                                 addresses for.
        """
        c = hosts_per_if
        assert(not cls.hosts_by_pg_idx)
        for i in range(len(cls.pg_interfaces)):
            pg_idx = cls.pg_interfaces[i].sw_if_index
            cls.hosts_by_pg_idx[pg_idx] = [Host(
                "00:00:00:ff:%02x:%02x" % (pg_idx, j + 1),
                "172.17.1%02u.%u" % (pg_idx, j + 1)) for j in range(c)]

    @classmethod
    def bd_if_range(cls, b):
        n = cls.ifs_per_bd
        start = (b - 1) * n
        return range(start, start + n)

    def create_bd_and_mac_learn(self, count, start=1):
        """
        Create required number of bridge domains with MAC learning enabled,
        put 3 l2-pg interfaces to every bridge domain and send MAC learning
        packets.

        :param int count: Number of bridge domains to be created.
        :param int start: Starting number of the bridge domain ID.
            (Default value = 1)
        """
        for b in range(start, start + count):
            self.vapi.bridge_domain_add_del(bd_id=b)
            self.logger.info("Bridge domain ID %d created" % b)
            if self.bd_list.count(b) == 0:
                self.bd_list.append(b)
            if self.bd_deleted_list.count(b) == 1:
                self.bd_deleted_list.remove(b)
            for j in self.bd_if_range(b):
                pg_if = self.pg_interfaces[j]
                self.vapi.sw_interface_set_l2_bridge(pg_if.sw_if_index,
                                                     bd_id=b)
                self.logger.info("pg-interface %s added to bridge domain ID %d"
                                 % (pg_if.name, b))
                self.pg_in_bd.append(pg_if)
                hosts = self.hosts_by_pg_idx[pg_if.sw_if_index]
                packets = [Ether(dst="ff:ff:ff:ff:ff:ff", src=host.mac)
                           for host in hosts]
                pg_if.add_stream(packets)
        self.logger.info("Sending broadcast eth frames for MAC learning")
        self.pg_start()
        self.logger.info(self.vapi.ppcli("show bridge-domain"))
        self.logger.info(self.vapi.ppcli("show l2fib"))

    def delete_bd(self, count, start=1):
        """
        Delete required number of bridge domains.

        :param int count: Number of bridge domains to be created.
        :param int start: Starting number of the bridge domain ID.
            (Default value = 1)
        """
        for b in range(start, start + count):
            for j in self.bd_if_range(b):
                pg_if = self.pg_interfaces[j]
                self.vapi.sw_interface_set_l2_bridge(pg_if.sw_if_index,
                                                     bd_id=b, enable=0)
                self.pg_in_bd.remove(pg_if)
            self.vapi.bridge_domain_add_del(bd_id=b, is_add=0)
            self.bd_list.remove(b)
            self.bd_deleted_list.append(b)
            self.logger.info("Bridge domain ID %d deleted" % b)

    def create_stream(self, src_if):
        """
        Create input packet stream for defined interface using hosts list.

        :param object src_if: Interface to create packet stream for.
        :param list packet_sizes: List of required packet sizes.
        :return: Stream of packets.
        """
        packet_sizes = self.pg_if_packet_sizes
        pkts = []
        src_hosts = self.hosts_by_pg_idx[src_if.sw_if_index]
        for dst_if in self.flows[src_if]:
            dst_hosts = self.hosts_by_pg_idx[dst_if.sw_if_index]
            for dst_host in dst_hosts:
                pkt_info = self.create_packet_info(src_if, dst_if)
                payload = self.info_to_payload(pkt_info)
                src_host = random.choice(src_hosts)
                p = (Ether(dst=dst_host.mac, src=src_host.mac) /
                     IP(src=src_host.ip4, dst=dst_host.ip4) /
                     UDP(sport=1234, dport=1234) /
                     Raw(payload))
                pkt_info.data = p.copy()
                size = random.choice(packet_sizes)
                self.extend_packet(p, size)
                pkts.append(p)
        self.logger.debug("Input stream created for port %s. Length: %u pkt(s)"
                          % (src_if.name, len(pkts)))
        return pkts

    def verify_capture(self, dst_if):
        """
        Verify captured input packet stream for defined interface.

        :param object dst_if: Interface to verify captured packet stream for.
        """
        last_info = dict()
        for i in self.flows[dst_if]:
            last_info[i.sw_if_index] = None
        dst = dst_if.sw_if_index
        for packet in dst_if.get_capture():
            try:
                ip = packet[IP]
                udp = packet[UDP]
                info = self.payload_to_info(str(packet[Raw]))
                self.assertEqual(info.dst, dst)
                self.logger.debug("Got packet on port %s: src=%u (id=%u)" %
                                  (dst_if.name, info.src, info.index))
                last_info[info.src] = self.get_next_packet_info_for_interface2(
                    info.src, dst, last_info[info.src])
                pkt_info = last_info[info.src]
                self.assertTrue(pkt_info is not None)
                self.assertEqual(info.index, pkt_info.index)
                # Check standard fields against saved data in pkt
                saved = pkt_info.data
                self.assertEqual(ip.src, saved[IP].src)
                self.assertEqual(ip.dst, saved[IP].dst)
                self.assertEqual(udp.sport, saved[UDP].sport)
                self.assertEqual(udp.dport, saved[UDP].dport)
            except:
                self.logger.error(ppp("Unexpected or invalid packet:", packet))
                raise
        s = ""
        remaining = 0
        for src in self.flows[dst_if]:
            remaining_packet = self.get_next_packet_info_for_interface2(
                src.sw_if_index, dst, last_info[src.sw_if_index])
            if remaining_packet is None:
                s += "Port %u: Packet expected from source %u didn't arrive\n"\
                     % (dst, src.sw_if_index)
                remaining += 1
            self.assertNotEqual(0, remaining, s)

    def set_bd_flags(self, bd_id, **args):
        """
        Enable/disable defined feature(s) of the bridge domain.

        :param int bd_id: Bridge domain ID.
        :param list args: List of feature/status pairs. Allowed features: \
        learn, forward, flood, uu_flood and arp_term. Status False means \
        disable, status True means enable the feature.
        :raise: ValueError in case of unknown feature in the input.
        """
        for flag in args:
            if flag == "learn":
                feature_bitmap = 1 << 0
            elif flag == "forward":
                feature_bitmap = 1 << 1
            elif flag == "flood":
                feature_bitmap = 1 << 2
            elif flag == "uu_flood":
                feature_bitmap = 1 << 3
            elif flag == "arp_term":
                feature_bitmap = 1 << 4
            else:
                raise ValueError("Unknown feature used: %s" % flag)
            is_set = 1 if args[flag] else 0
            self.vapi.bridge_flags(bd_id, is_set, feature_bitmap)
        self.logger.info("Bridge domain ID %d updated" % bd_id)

    def verify_bd(self, bd_id, **args):
        """
        Check if the bridge domain is configured and verify expected status
        of listed features.

        :param int bd_id: Bridge domain ID.
        :param list args: List of feature/status pairs. Allowed features: \
        learn, forward, flood, uu_flood and arp_term. Status False means \
        disable, status True means enable the feature.
        :return: 1 if bridge domain is configured, otherwise return 0.
        :raise: ValueError in case of unknown feature in the input.
        """
        bd_dump = self.vapi.bridge_domain_dump(bd_id)
        if len(bd_dump) == 0:
            self.logger.info("Bridge domain ID %d is not configured" % bd_id)
            return 0
        else:
            bd_dump = bd_dump[0]
            if len(args) > 0:
                for flag in args:
                    expected_status = 1 if args[flag] else 0
                    if flag == "learn":
                        flag_status = bd_dump[6]
                    elif flag == "forward":
                        flag_status = bd_dump[5]
                    elif flag == "flood":
                        flag_status = bd_dump[3]
                    elif flag == "uu_flood":
                        flag_status = bd_dump[4]
                    elif flag == "arp_term":
                        flag_status = bd_dump[7]
                    else:
                        raise ValueError("Unknown feature used: %s" % flag)
                    self.assertEqual(expected_status, flag_status)
            return 1

    def run_verify_test(self):
        """
        Create packet streams for all configured l2-pg interfaces, send all \
        prepared packet streams and verify that:
            - all packets received correctly on all pg-l2 interfaces assigned
              to bridge domains
            - no packet received on all pg-l2 interfaces not assigned to
              bridge domains

        :raise RuntimeError: if no packet captured on l2-pg interface assigned
                             to the bridge domain or if any packet is captured
                             on l2-pg interface not assigned to the bridge
                             domain.
        """
        # Test
        # Create incoming packet streams for packet-generator interfaces
        # for pg_if in self.pg_interfaces:
        assert(len(self._packet_count_for_dst_if_idx) == 0)
        for pg_if in self.pg_in_bd:
            pkts = self.create_stream(pg_if)
            pg_if.add_stream(pkts)

        # Enable packet capture and start packet sending
        self.pg_enable_capture(self.pg_in_bd)
        self.pg_start()

        # Verify
        # Verify outgoing packet streams per packet-generator interface
        for pg_if in self.pg_in_bd:
            self.verify_capture(pg_if)

    def test_l2bd_inst_01(self):
        """ L2BD Multi-instance test 1 - create 5 BDs
        """
        # Config 1
        # Create 5 BDs, put interfaces to these BDs and send MAC learning
        # packets
        self.create_bd_and_mac_learn(5)

        # Verify 1
        for bd_id in self.bd_list:
            self.assertEqual(self.verify_bd(bd_id), 1)

        # Test 1
        # self.vapi.cli("clear trace")
        self.run_verify_test()

    def test_l2bd_inst_02(self):
        """ L2BD Multi-instance test 2 - update data of 5 BDs
        """
        # Config 2
        # Update data of 5 BDs (disable learn, forward, flood, uu-flood)
        self.set_bd_flags(self.bd_list[0], learn=False, forward=False,
                          flood=False, uu_flood=False)
        self.set_bd_flags(self.bd_list[1], forward=False)
        self.set_bd_flags(self.bd_list[2], flood=False)
        self.set_bd_flags(self.bd_list[3], uu_flood=False)
        self.set_bd_flags(self.bd_list[4], learn=False)

        # Verify 2
        # Skipping check of uu_flood as it is not returned by
        # bridge_domain_dump api command
        self.verify_bd(self.bd_list[0], learn=False, forward=False,
                       flood=False, uu_flood=False)
        self.verify_bd(self.bd_list[1], learn=True, forward=False,
                       flood=True, uu_flood=True)
        self.verify_bd(self.bd_list[2], learn=True, forward=True,
                       flood=False, uu_flood=True)
        self.verify_bd(self.bd_list[3], learn=True, forward=True,
                       flood=True, uu_flood=False)
        self.verify_bd(self.bd_list[4], learn=False, forward=True,
                       flood=True, uu_flood=True)

    def test_l2bd_inst_03(self):
        """ L2BD Multi-instance 3 - delete 2 BDs
        """
        # Config 3
        # Delete 2 BDs
        self.delete_bd(2)

        # Verify 3
        for bd_id in self.bd_deleted_list:
            self.assertEqual(self.verify_bd(bd_id), 0)
        for bd_id in self.bd_list:
            self.assertEqual(self.verify_bd(bd_id), 1)

        # Test 3
        self.run_verify_test()

    def test_l2bd_inst_04(self):
        """ L2BD Multi-instance test 4 - add 2 BDs
        """
        # Config 4
        # Create 5 BDs, put interfaces to these BDs and send MAC learning
        # packets
        self.create_bd_and_mac_learn(2)

        # Verify 4
        for bd_id in self.bd_list:
            self.assertEqual(self.verify_bd(bd_id), 1)

        # Test 4
        # self.vapi.cli("clear trace")
        self.run_verify_test()

    def test_l2bd_inst_05(self):
        """ L2BD Multi-instance 5 - delete 5 BDs
        """
        # Config 5
        # Delete 5 BDs
        self.delete_bd(5)

        # Verify 5
        for bd_id in self.bd_deleted_list:
            self.assertEqual(self.verify_bd(bd_id), 0)
        for bd_id in self.bd_list:
            self.assertEqual(self.verify_bd(bd_id), 1)


if __name__ == '__main__':
    unittest.main(testRunner=VppTestRunner)