diff options
Diffstat (limited to 'drivers/net/tap')
-rw-r--r-- | drivers/net/tap/Makefile | 61 | ||||
-rw-r--r-- | drivers/net/tap/rte_eth_tap.c | 265 | ||||
-rw-r--r-- | drivers/net/tap/rte_eth_tap.h | 50 | ||||
-rw-r--r-- | drivers/net/tap/tap_bpf.h | 117 | ||||
-rw-r--r-- | drivers/net/tap/tap_bpf_api.c | 190 | ||||
-rw-r--r-- | drivers/net/tap/tap_bpf_insns.h | 1696 | ||||
-rw-r--r-- | drivers/net/tap/tap_bpf_program.c | 224 | ||||
-rw-r--r-- | drivers/net/tap/tap_flow.c | 825 | ||||
-rw-r--r-- | drivers/net/tap/tap_flow.h | 47 | ||||
-rw-r--r-- | drivers/net/tap/tap_intr.c | 110 | ||||
-rw-r--r-- | drivers/net/tap/tap_netlink.c | 75 | ||||
-rw-r--r-- | drivers/net/tap/tap_netlink.h | 59 | ||||
-rw-r--r-- | drivers/net/tap/tap_rss.h | 34 | ||||
-rw-r--r-- | drivers/net/tap/tap_tcmsgs.c | 62 | ||||
-rw-r--r-- | drivers/net/tap/tap_tcmsgs.h | 38 |
15 files changed, 3335 insertions, 518 deletions
diff --git a/drivers/net/tap/Makefile b/drivers/net/tap/Makefile index 405b49e4..ccc5c5fc 100644 --- a/drivers/net/tap/Makefile +++ b/drivers/net/tap/Makefile @@ -1,32 +1,5 @@ -# BSD LICENSE -# -# Copyright(c) 2016 Intel Corporation. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2016 Intel Corporation include $(RTE_SDK)/mk/rte.vars.mk @@ -39,6 +12,12 @@ EXPORT_MAP := rte_pmd_tap_version.map LIBABIVER := 1 +# +# TAP_MAX_QUEUES must be a power of 2 +# +ifeq ($(TAP_MAX_QUEUES),) + TAP_MAX_QUEUES = 16 +endif CFLAGS += -O3 CFLAGS += -I$(SRCDIR) CFLAGS += -I. @@ -47,6 +26,8 @@ LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash LDLIBS += -lrte_bus_vdev +CFLAGS += -DTAP_MAX_QUEUES=$(TAP_MAX_QUEUES) + # # all source are stored in SRCS-y # @@ -54,6 +35,8 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += rte_eth_tap.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_flow.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_netlink.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_tcmsgs.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_bpf_api.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_intr.c include $(RTE_SDK)/mk/rte.lib.mk @@ -80,6 +63,26 @@ tap_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh linux/pkt_cls.h \ enum TCA_FLOWER_KEY_VLAN_PRIO \ $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_TC_BPF \ + linux/pkt_cls.h \ + enum TCA_BPF_UNSPEC \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_TC_BPF_FD \ + linux/pkt_cls.h \ + enum TCA_BPF_FD \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_TC_ACT_BPF \ + linux/tc_act/tc_bpf.h \ + enum TCA_ACT_BPF_UNSPEC \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_TC_ACT_BPF_FD \ + linux/tc_act/tc_bpf.h \ + enum TCA_ACT_BPF_FD \ + $(AUTOCONF_OUTPUT) # Create tap_autoconf.h or update it in case it differs from the new one. diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c index 6b27679a..f09db0ea 100644 --- a/drivers/net/tap/rte_eth_tap.c +++ b/drivers/net/tap/rte_eth_tap.c @@ -1,34 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #include <rte_atomic.h> @@ -36,7 +7,7 @@ #include <rte_byteorder.h> #include <rte_common.h> #include <rte_mbuf.h> -#include <rte_ethdev.h> +#include <rte_ethdev_driver.h> #include <rte_ethdev_vdev.h> #include <rte_malloc.h> #include <rte_bus_vdev.h> @@ -53,6 +24,7 @@ #include <sys/mman.h> #include <errno.h> #include <signal.h> +#include <stdbool.h> #include <stdint.h> #include <sys/uio.h> #include <unistd.h> @@ -60,7 +32,6 @@ #include <net/if.h> #include <linux/if_tun.h> #include <linux/if_ether.h> -#include <linux/version.h> #include <fcntl.h> #include <rte_eth_tap.h> @@ -73,19 +44,14 @@ #define DEFAULT_TAP_NAME "dtap" #define ETH_TAP_IFACE_ARG "iface" -#define ETH_TAP_SPEED_ARG "speed" #define ETH_TAP_REMOTE_ARG "remote" #define ETH_TAP_MAC_ARG "mac" #define ETH_TAP_MAC_FIXED "fixed" -#define FLOWER_KERNEL_VERSION KERNEL_VERSION(4, 2, 0) -#define FLOWER_VLAN_KERNEL_VERSION KERNEL_VERSION(4, 9, 0) - static struct rte_vdev_driver pmd_tap_drv; static const char *valid_arguments[] = { ETH_TAP_IFACE_ARG, - ETH_TAP_SPEED_ARG, ETH_TAP_REMOTE_ARG, ETH_TAP_MAC_ARG, NULL @@ -99,7 +65,7 @@ static struct rte_eth_link pmd_link = { .link_speed = ETH_SPEED_NUM_10G, .link_duplex = ETH_LINK_FULL_DUPLEX, .link_status = ETH_LINK_DOWN, - .link_autoneg = ETH_LINK_SPEED_AUTONEG + .link_autoneg = ETH_LINK_AUTONEG }; static void @@ -285,6 +251,45 @@ tap_verify_csum(struct rte_mbuf *mbuf) } } +static uint64_t +tap_rx_offload_get_port_capa(void) +{ + /* + * In order to support legacy apps, + * report capabilities also as port capabilities. + */ + return DEV_RX_OFFLOAD_SCATTER | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_CRC_STRIP; +} + +static uint64_t +tap_rx_offload_get_queue_capa(void) +{ + return DEV_RX_OFFLOAD_SCATTER | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_CRC_STRIP; +} + +static bool +tap_rxq_are_offloads_valid(struct rte_eth_dev *dev, uint64_t offloads) +{ + uint64_t port_offloads = dev->data->dev_conf.rxmode.offloads; + uint64_t queue_supp_offloads = tap_rx_offload_get_queue_capa(); + uint64_t port_supp_offloads = tap_rx_offload_get_port_capa(); + + if ((offloads & (queue_supp_offloads | port_supp_offloads)) != + offloads) + return false; + if ((port_offloads ^ offloads) & port_supp_offloads) + return false; + return true; +} + /* Callback to handle the rx burst of packets to the correct interface and * file descriptor(s) in a multi-queue setup. */ @@ -309,8 +314,9 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) int len; len = readv(rxq->fd, *rxq->iovecs, - 1 + (rxq->rxmode->enable_scatter ? - rxq->nb_rx_desc : 1)); + 1 + + (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ? + rxq->nb_rx_desc : 1)); if (len < (int)sizeof(struct tun_pi)) break; @@ -365,7 +371,7 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) seg->next = NULL; mbuf->packet_type = rte_net_get_ptype(mbuf, NULL, RTE_PTYPE_ALL_MASK); - if (rxq->rxmode->hw_ip_checksum) + if (rxq->rxmode->offloads & DEV_RX_OFFLOAD_CHECKSUM) tap_verify_csum(mbuf); /* account for the receive frame */ @@ -379,6 +385,44 @@ end: return num_rx; } +static uint64_t +tap_tx_offload_get_port_capa(void) +{ + /* + * In order to support legacy apps, + * report capabilities also as port capabilities. + */ + return DEV_TX_OFFLOAD_MULTI_SEGS | + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM; +} + +static uint64_t +tap_tx_offload_get_queue_capa(void) +{ + return DEV_TX_OFFLOAD_MULTI_SEGS | + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM; +} + +static bool +tap_txq_are_offloads_valid(struct rte_eth_dev *dev, uint64_t offloads) +{ + uint64_t port_offloads = dev->data->dev_conf.txmode.offloads; + uint64_t queue_supp_offloads = tap_tx_offload_get_queue_capa(); + uint64_t port_supp_offloads = tap_tx_offload_get_port_capa(); + + if ((offloads & (queue_supp_offloads | port_supp_offloads)) != + offloads) + return false; + /* Verify we have no conflict with port offloads */ + if ((port_offloads ^ offloads) & port_supp_offloads) + return false; + return true; +} + static void tap_tx_offload(char *packet, uint64_t ol_flags, unsigned int l2_len, unsigned int l3_len) @@ -465,9 +509,10 @@ pmd_tx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) rte_pktmbuf_mtod(seg, void *); seg = seg->next; } - if (mbuf->ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_IPV4) || - (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM || - (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) { + if (txq->csum && + ((mbuf->ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_IPV4) || + (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM || + (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM))) { /* Support only packets with all data in the same seg */ if (mbuf->nb_segs > 1) break; @@ -605,6 +650,17 @@ tap_dev_stop(struct rte_eth_dev *dev) static int tap_dev_configure(struct rte_eth_dev *dev) { + uint64_t supp_tx_offloads = tap_tx_offload_get_port_capa(); + uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads; + + if ((tx_offloads & supp_tx_offloads) != tx_offloads) { + rte_errno = ENOTSUP; + RTE_LOG(ERR, PMD, + "Some Tx offloads are not supported " + "requested 0x%" PRIx64 " supported 0x%" PRIx64 "\n", + tx_offloads, supp_tx_offloads); + return -rte_errno; + } if (dev->data->nb_rx_queues > RTE_PMD_TAP_MAX_QUEUES) { RTE_LOG(ERR, PMD, "%s: number of rx queues %d exceeds max num of queues %d\n", @@ -678,13 +734,12 @@ tap_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->min_rx_bufsize = 0; dev_info->pci_dev = NULL; dev_info->speed_capa = tap_dev_speed_capa(); - dev_info->rx_offload_capa = (DEV_RX_OFFLOAD_IPV4_CKSUM | - DEV_RX_OFFLOAD_UDP_CKSUM | - DEV_RX_OFFLOAD_TCP_CKSUM); - dev_info->tx_offload_capa = - (DEV_TX_OFFLOAD_IPV4_CKSUM | - DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_CKSUM); + dev_info->rx_queue_offload_capa = tap_rx_offload_get_queue_capa(); + dev_info->rx_offload_capa = tap_rx_offload_get_port_capa() | + dev_info->rx_queue_offload_capa; + dev_info->tx_queue_offload_capa = tap_tx_offload_get_queue_capa(); + dev_info->tx_offload_capa = tap_tx_offload_get_port_capa() | + dev_info->tx_queue_offload_capa; } static int @@ -1000,6 +1055,19 @@ tap_rx_queue_setup(struct rte_eth_dev *dev, return -1; } + /* Verify application offloads are valid for our port and queue. */ + if (!tap_rxq_are_offloads_valid(dev, rx_conf->offloads)) { + rte_errno = ENOTSUP; + RTE_LOG(ERR, PMD, + "%p: Rx queue offloads 0x%" PRIx64 + " don't match port offloads 0x%" PRIx64 + " or supported offloads 0x%" PRIx64 "\n", + (void *)dev, rx_conf->offloads, + dev->data->dev_conf.rxmode.offloads, + (tap_rx_offload_get_port_capa() | + tap_rx_offload_get_queue_capa())); + return -rte_errno; + } rxq->mp = mp; rxq->trigger_seen = 1; /* force initial burst */ rxq->in_port = dev->data->port_id; @@ -1058,21 +1126,46 @@ tap_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, uint16_t nb_tx_desc __rte_unused, unsigned int socket_id __rte_unused, - const struct rte_eth_txconf *tx_conf __rte_unused) + const struct rte_eth_txconf *tx_conf) { struct pmd_internals *internals = dev->data->dev_private; + struct tx_queue *txq; int ret; if (tx_queue_id >= dev->data->nb_tx_queues) return -1; - dev->data->tx_queues[tx_queue_id] = &internals->txq[tx_queue_id]; + txq = dev->data->tx_queues[tx_queue_id]; + /* + * Don't verify port offloads for application which + * use the old API. + */ + if (tx_conf != NULL && + !!(tx_conf->txq_flags & ETH_TXQ_FLAGS_IGNORE)) { + if (tap_txq_are_offloads_valid(dev, tx_conf->offloads)) { + txq->csum = !!(tx_conf->offloads & + (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM)); + } else { + rte_errno = ENOTSUP; + RTE_LOG(ERR, PMD, + "%p: Tx queue offloads 0x%" PRIx64 + " don't match port offloads 0x%" PRIx64 + " or supported offloads 0x%" PRIx64, + (void *)dev, tx_conf->offloads, + dev->data->dev_conf.txmode.offloads, + tap_tx_offload_get_port_capa()); + return -rte_errno; + } + } ret = tap_setup_queue(dev, internals, tx_queue_id, 0); if (ret == -1) return -1; - - RTE_LOG(DEBUG, PMD, " TX TAP device name %s, qid %d on fd %d\n", - internals->name, tx_queue_id, internals->txq[tx_queue_id].fd); + RTE_LOG(DEBUG, PMD, + " TX TAP device name %s, qid %d on fd %d csum %s\n", + internals->name, tx_queue_id, internals->txq[tx_queue_id].fd, + txq->csum ? "on" : "off"); return 0; } @@ -1123,35 +1216,49 @@ tap_dev_intr_handler(void *cb_arg) struct rte_eth_dev *dev = cb_arg; struct pmd_internals *pmd = dev->data->dev_private; - nl_recv(pmd->intr_handle.fd, tap_nl_msg_handler, dev); + tap_nl_recv(pmd->intr_handle.fd, tap_nl_msg_handler, dev); } static int -tap_intr_handle_set(struct rte_eth_dev *dev, int set) +tap_lsc_intr_handle_set(struct rte_eth_dev *dev, int set) { struct pmd_internals *pmd = dev->data->dev_private; /* In any case, disable interrupt if the conf is no longer there. */ if (!dev->data->dev_conf.intr_conf.lsc) { if (pmd->intr_handle.fd != -1) { - nl_final(pmd->intr_handle.fd); + tap_nl_final(pmd->intr_handle.fd); rte_intr_callback_unregister(&pmd->intr_handle, tap_dev_intr_handler, dev); } return 0; } if (set) { - pmd->intr_handle.fd = nl_init(RTMGRP_LINK); + pmd->intr_handle.fd = tap_nl_init(RTMGRP_LINK); if (unlikely(pmd->intr_handle.fd == -1)) return -EBADF; return rte_intr_callback_register( &pmd->intr_handle, tap_dev_intr_handler, dev); } - nl_final(pmd->intr_handle.fd); + tap_nl_final(pmd->intr_handle.fd); return rte_intr_callback_unregister(&pmd->intr_handle, tap_dev_intr_handler, dev); } +static int +tap_intr_handle_set(struct rte_eth_dev *dev, int set) +{ + int err; + + err = tap_lsc_intr_handle_set(dev, set); + if (err) + return err; + err = tap_rx_intr_vec_set(dev, set); + if (err && set) + tap_lsc_intr_handle_set(dev, 0); + return err; +} + static const uint32_t* tap_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) { @@ -1244,13 +1351,13 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name, data = rte_zmalloc_socket(tap_name, sizeof(*data), 0, numa_node); if (!data) { RTE_LOG(ERR, PMD, "TAP Failed to allocate data\n"); - goto error_exit; + goto error_exit_nodev; } dev = rte_eth_vdev_allocate(vdev, sizeof(*pmd)); if (!dev) { RTE_LOG(ERR, PMD, "TAP Unable to allocate device struct\n"); - goto error_exit; + goto error_exit_nodev; } pmd = dev->data->dev_private; @@ -1284,6 +1391,7 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name, pmd->intr_handle.type = RTE_INTR_HANDLE_EXT; pmd->intr_handle.fd = -1; + dev->intr_handle = &pmd->intr_handle; /* Presetup the fds to -1 as being not valid */ for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { @@ -1328,7 +1436,7 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name, * - rte_flow actual/implicit lists * - implicit rules */ - pmd->nlsk_fd = nl_init(0); + pmd->nlsk_fd = tap_nl_init(0); if (pmd->nlsk_fd == -1) { RTE_LOG(WARNING, PMD, "%s: failed to create netlink socket.\n", pmd->name); @@ -1420,6 +1528,11 @@ error_remote: tap_flow_implicit_flush(pmd, NULL); error_exit: + if (pmd->ioctl_sock > 0) + close(pmd->ioctl_sock); + rte_eth_dev_release_port(dev); + +error_exit_nodev: RTE_LOG(ERR, PMD, "TAP Unable to initialize %s\n", rte_vdev_device_name(vdev)); @@ -1444,16 +1557,6 @@ set_interface_name(const char *key __rte_unused, } static int -set_interface_speed(const char *key __rte_unused, - const char *value, - void *extra_args) -{ - *(int *)extra_args = (value) ? atoi(value) : ETH_SPEED_NUM_10G; - - return 0; -} - -static int set_remote_iface(const char *key __rte_unused, const char *value, void *extra_args) @@ -1503,15 +1606,6 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev) kvlist = rte_kvargs_parse(params, valid_arguments); if (kvlist) { - if (rte_kvargs_count(kvlist, ETH_TAP_SPEED_ARG) == 1) { - ret = rte_kvargs_process(kvlist, - ETH_TAP_SPEED_ARG, - &set_interface_speed, - &speed); - if (ret == -1) - goto leave; - } - if (rte_kvargs_count(kvlist, ETH_TAP_IFACE_ARG) == 1) { ret = rte_kvargs_process(kvlist, ETH_TAP_IFACE_ARG, @@ -1579,7 +1673,7 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev) if (internals->nlsk_fd) { tap_flow_flush(eth_dev, NULL); tap_flow_implicit_flush(internals, NULL); - nl_final(internals->nlsk_fd); + tap_nl_final(internals->nlsk_fd); } for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { if (internals->rxq[i].fd != -1) { @@ -1609,6 +1703,5 @@ RTE_PMD_REGISTER_VDEV(net_tap, pmd_tap_drv); RTE_PMD_REGISTER_ALIAS(net_tap, eth_tap); RTE_PMD_REGISTER_PARAM_STRING(net_tap, ETH_TAP_IFACE_ARG "=<string> " - ETH_TAP_SPEED_ARG "=<int> " ETH_TAP_MAC_ARG "=" ETH_TAP_MAC_FIXED " " ETH_TAP_REMOTE_ARG "=<string>"); diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h index 829f32f3..53a506ad 100644 --- a/drivers/net/tap/rte_eth_tap.h +++ b/drivers/net/tap/rte_eth_tap.h @@ -1,34 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright 2017 6WIND S.A. - * Copyright 2017 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. */ #ifndef _RTE_ETH_TAP_H_ @@ -41,11 +13,11 @@ #include <linux/if_tun.h> -#include <rte_ethdev.h> +#include <rte_ethdev_driver.h> #include <rte_ether.h> #ifdef IFF_MULTI_QUEUE -#define RTE_PMD_TAP_MAX_QUEUES 16 +#define RTE_PMD_TAP_MAX_QUEUES TAP_MAX_QUEUES #else #define RTE_PMD_TAP_MAX_QUEUES 1 #endif @@ -76,6 +48,7 @@ struct rx_queue { struct tx_queue { int fd; uint16_t *mtu; /* Pointer to MTU from dev_data */ + uint16_t csum:1; /* Enable checksum offloading */ struct pkt_stats stats; /* Stats for this TX queue */ }; @@ -90,6 +63,13 @@ struct pmd_internals { int ioctl_sock; /* socket for ioctl calls */ int nlsk_fd; /* Netlink socket fd */ int flow_isolate; /* 1 if flow isolation is enabled */ + int flower_support; /* 1 if kernel supports, else 0 */ + int flower_vlan_support; /* 1 if kernel supports, else 0 */ + int rss_enabled; /* 1 if RSS is enabled, else 0 */ + /* implicit rules set when RSS is enabled */ + int map_fd; /* BPF RSS map fd */ + int bpf_fd[RTE_PMD_TAP_MAX_QUEUES];/* List of bpf fds per queue */ + LIST_HEAD(tap_rss_flows, rte_flow) rss_flows; LIST_HEAD(tap_flows, rte_flow) flows; /* rte_flow rules */ /* implicit rte_flow rules set when a remote device is active */ LIST_HEAD(tap_implicit_flows, rte_flow) implicit_flows; @@ -98,4 +78,8 @@ struct pmd_internals { struct rte_intr_handle intr_handle; /* LSC interrupt handle. */ }; +/* tap_intr.c */ + +int tap_rx_intr_vec_set(struct rte_eth_dev *dev, int set); + #endif /* _RTE_ETH_TAP_H_ */ diff --git a/drivers/net/tap/tap_bpf.h b/drivers/net/tap/tap_bpf.h new file mode 100644 index 00000000..1a70ffe2 --- /dev/null +++ b/drivers/net/tap/tap_bpf.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 + * Copyright 2017 Mellanox Technologies, Ltd. + */ + +#ifndef __TAP_BPF_H__ +#define __TAP_BPF_H__ + +#include <tap_autoconf.h> + +/* Do not #include <linux/bpf.h> since eBPF must compile on different + * distros which may include partial definitions for eBPF (while the + * kernel itself may support eBPF). Instead define here all that is needed + */ + +/* BPF_MAP_UPDATE_ELEM command flags */ +#define BPF_ANY 0 /* create a new element or update an existing */ + +/* BPF architecture instruction struct */ +struct bpf_insn { + __u8 code; + __u8 dst_reg:4; + __u8 src_reg:4; + __s16 off; + __s32 imm; /* immediate value */ +}; + +/* BPF program types */ +enum bpf_prog_type { + BPF_PROG_TYPE_UNSPEC, + BPF_PROG_TYPE_SOCKET_FILTER, + BPF_PROG_TYPE_KPROBE, + BPF_PROG_TYPE_SCHED_CLS, + BPF_PROG_TYPE_SCHED_ACT, +}; + +/* BPF commands types */ +enum bpf_cmd { + BPF_MAP_CREATE, + BPF_MAP_LOOKUP_ELEM, + BPF_MAP_UPDATE_ELEM, + BPF_MAP_DELETE_ELEM, + BPF_MAP_GET_NEXT_KEY, + BPF_PROG_LOAD, +}; + +/* BPF maps types */ +enum bpf_map_type { + BPF_MAP_TYPE_UNSPEC, + BPF_MAP_TYPE_HASH, +}; + +/* union of anonymous structs used with TAP BPF commands */ +union bpf_attr { + /* BPF_MAP_CREATE command */ + struct { + __u32 map_type; + __u32 key_size; + __u32 value_size; + __u32 max_entries; + __u32 map_flags; + __u32 inner_map_fd; + }; + + /* BPF_MAP_UPDATE_ELEM, BPF_MAP_DELETE_ELEM commands */ + struct { + __u32 map_fd; + __aligned_u64 key; + union { + __aligned_u64 value; + __aligned_u64 next_key; + }; + __u64 flags; + }; + + /* BPF_PROG_LOAD command */ + struct { + __u32 prog_type; + __u32 insn_cnt; + __aligned_u64 insns; + __aligned_u64 license; + __u32 log_level; + __u32 log_size; + __aligned_u64 log_buf; + __u32 kern_version; + __u32 prog_flags; + }; +} __attribute__((aligned(8))); + +#ifndef __NR_bpf +# if defined(__i386__) +# define __NR_bpf 357 +# elif defined(__x86_64__) +# define __NR_bpf 321 +# elif defined(__arm__) +# define __NR_bpf 386 +# elif defined(__aarch64__) +# define __NR_bpf 280 +# elif defined(__sparc__) +# define __NR_bpf 349 +# elif defined(__s390__) +# define __NR_bpf 351 +# elif defined(__powerpc__) +# define __NR_bpf 361 +# else +# error __NR_bpf not defined +# endif +#endif + +enum { + BPF_MAP_ID_KEY, + BPF_MAP_ID_SIMPLE, +}; + +static int bpf_load(enum bpf_prog_type type, const struct bpf_insn *insns, + size_t insns_cnt, const char *license); + +#endif /* __TAP_BPF_H__ */ diff --git a/drivers/net/tap/tap_bpf_api.c b/drivers/net/tap/tap_bpf_api.c new file mode 100644 index 00000000..109a681e --- /dev/null +++ b/drivers/net/tap/tap_bpf_api.c @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 Mellanox Technologies, Ltd. + */ + +#include <errno.h> +#include <string.h> +#include <unistd.h> +#include <sys/queue.h> + +#include <rte_malloc.h> +#include <rte_eth_tap.h> +#include <tap_flow.h> +#include <tap_autoconf.h> +#include <tap_tcmsgs.h> +#include <tap_bpf.h> +#include <tap_bpf_insns.h> + +/** + * Load BPF program (section cls_q) into the kernel and return a bpf fd + * + * @param queue_idx + * Queue index matching packet cb + * + * @return + * -1 if the BPF program couldn't be loaded. An fd (int) otherwise. + */ +int tap_flow_bpf_cls_q(__u32 queue_idx) +{ + cls_q_insns[1].imm = queue_idx; + + return bpf_load(BPF_PROG_TYPE_SCHED_CLS, + (struct bpf_insn *)cls_q_insns, + RTE_DIM(cls_q_insns), + "Dual BSD/GPL"); +} + +/** + * Load BPF program (section l3_l4) into the kernel and return a bpf fd. + * + * @param[in] key_idx + * RSS MAP key index + * + * @param[in] map_fd + * BPF RSS map file descriptor + * + * @return + * -1 if the BPF program couldn't be loaded. An fd (int) otherwise. + */ +int tap_flow_bpf_calc_l3_l4_hash(__u32 key_idx, int map_fd) +{ + l3_l4_hash_insns[4].imm = key_idx; + l3_l4_hash_insns[9].imm = map_fd; + + return bpf_load(BPF_PROG_TYPE_SCHED_ACT, + (struct bpf_insn *)l3_l4_hash_insns, + RTE_DIM(l3_l4_hash_insns), + "Dual BSD/GPL"); +} + +/** + * Helper function to convert a pointer to unsigned 64 bits + * + * @param[in] ptr + * pointer to address + * + * @return + * 64 bit unsigned long type of pointer address + */ +static inline __u64 ptr_to_u64(const void *ptr) +{ + return (__u64)(unsigned long)ptr; +} + +/** + * Call BPF system call + * + * @param[in] cmd + * BPF command for program loading, map creation, map entry update, etc + * + * @param[in] attr + * System call attributes relevant to system call command + * + * @param[in] size + * size of attr parameter + * + * @return + * -1 if BPF system call failed, 0 otherwise + */ +static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, + unsigned int size) +{ + return syscall(__NR_bpf, cmd, attr, size); +} + +/** + * Load BPF instructions to kernel + * + * @param[in] type + * BPF program type: classifieir or action + * + * @param[in] insns + * Array of BPF instructions (equivalent to BPF instructions) + * + * @param[in] insns_cnt + * Number of BPF instructions (size of array) + * + * @param[in] lincense + * License string that must be acknowledged by the kernel + * + * @return + * -1 if the BPF program couldn't be loaded, fd (file descriptor) otherwise + */ +static int bpf_load(enum bpf_prog_type type, + const struct bpf_insn *insns, + size_t insns_cnt, + const char *license) +{ + union bpf_attr attr = {}; + + bzero(&attr, sizeof(attr)); + attr.prog_type = type; + attr.insn_cnt = (__u32)insns_cnt; + attr.insns = ptr_to_u64(insns); + attr.license = ptr_to_u64(license); + attr.log_buf = ptr_to_u64(NULL); + attr.log_level = 0; + attr.kern_version = 0; + + return sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); +} + +/** + * Create BPF map for RSS rules + * + * @param[in] key_size + * map RSS key size + * + * @param[in] value_size + * Map RSS value size + * + * @param[in] max_entries + * Map max number of RSS entries (limit on max RSS rules) + * + * @return + * -1 if BPF map couldn't be created, map fd otherwise + */ +int tap_flow_bpf_rss_map_create(unsigned int key_size, + unsigned int value_size, + unsigned int max_entries) +{ + union bpf_attr attr = {}; + + bzero(&attr, sizeof(attr)); + attr.map_type = BPF_MAP_TYPE_HASH; + attr.key_size = key_size; + attr.value_size = value_size; + attr.max_entries = max_entries; + + return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr)); +} + +/** + * Update RSS entry in BPF map + * + * @param[in] fd + * RSS map fd + * + * @param[in] key + * Pointer to RSS key whose entry is updated + * + * @param[in] value + * Pointer to RSS new updated value + * + * @return + * -1 if RSS entry failed to be updated, 0 otherwise + */ +int tap_flow_bpf_update_rss_elem(int fd, void *key, void *value) +{ + union bpf_attr attr = {}; + + bzero(&attr, sizeof(attr)); + + attr.map_type = BPF_MAP_TYPE_HASH; + attr.map_fd = fd; + attr.key = ptr_to_u64(key); + attr.value = ptr_to_u64(value); + attr.flags = BPF_ANY; + + return sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr)); +} diff --git a/drivers/net/tap/tap_bpf_insns.h b/drivers/net/tap/tap_bpf_insns.h new file mode 100644 index 00000000..89873b6d --- /dev/null +++ b/drivers/net/tap/tap_bpf_insns.h @@ -0,0 +1,1696 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 Mellanox Technologies, Ltd. + */ + +#include <tap_bpf.h> + +/* bpf_insn array matching cls_q section. See tap_bpf_program.c file */ +struct bpf_insn cls_q_insns[] = { + {0x61, 2, 1, 52, 0x00000000}, + {0x18, 3, 0, 0, 0xdeadbeef}, + {0x00, 0, 0, 0, 0x00000000}, + {0x63, 10, 3, -4, 0x00000000}, + {0xb7, 0, 0, 0, 0x00000000}, + {0x61, 3, 10, -4, 0x00000000}, + {0x07, 3, 0, 0, 0x7cafe800}, + {0x67, 3, 0, 0, 0x00000020}, + {0x77, 3, 0, 0, 0x00000020}, + {0x5d, 2, 3, 4, 0x00000000}, + {0xb7, 2, 0, 0, 0x00000000}, + {0x63, 1, 2, 52, 0x00000000}, + {0x18, 0, 0, 0, 0xffffffff}, + {0x00, 0, 0, 0, 0x00000000}, + {0x95, 0, 0, 0, 0x00000000}, +}; + +/* bpf_insn array matching l3_l4 section. see tap_bpf_program.c file */ +struct bpf_insn l3_l4_hash_insns[] = { + {0xbf, 7, 1, 0, 0x00000000}, + {0x61, 8, 7, 16, 0x00000000}, + {0x61, 6, 7, 76, 0x00000000}, + {0x61, 9, 7, 80, 0x00000000}, + {0x18, 1, 0, 0, 0xdeadbeef}, + {0x00, 0, 0, 0, 0x00000000}, + {0x63, 10, 1, -4, 0x00000000}, + {0xbf, 2, 10, 0, 0x00000000}, + {0x07, 2, 0, 0, 0xfffffffc}, + {0x18, 1, 1, 0, 0x0000cafe}, + {0x00, 0, 0, 0, 0x00000000}, + {0x85, 0, 0, 0, 0x00000001}, + {0x55, 0, 0, 21, 0x00000000}, + {0xb7, 1, 0, 0, 0x00000a64}, + {0x6b, 10, 1, -16, 0x00000000}, + {0x18, 1, 0, 0, 0x69666e6f}, + {0x00, 0, 0, 0, 0x65727567}, + {0x7b, 10, 1, -24, 0x00000000}, + {0x18, 1, 0, 0, 0x6e207369}, + {0x00, 0, 0, 0, 0x6320746f}, + {0x7b, 10, 1, -32, 0x00000000}, + {0x18, 1, 0, 0, 0x20737372}, + {0x00, 0, 0, 0, 0x2079656b}, + {0x7b, 10, 1, -40, 0x00000000}, + {0x18, 1, 0, 0, 0x68736168}, + {0x00, 0, 0, 0, 0x203a2928}, + {0x7b, 10, 1, -48, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000000}, + {0x73, 10, 7, -14, 0x00000000}, + {0xbf, 1, 10, 0, 0x00000000}, + {0x07, 1, 0, 0, 0xffffffd0}, + {0xb7, 2, 0, 0, 0x00000023}, + {0x85, 0, 0, 0, 0x00000006}, + {0x05, 0, 0, 1632, 0x00000000}, + {0xb7, 1, 0, 0, 0x0000000e}, + {0x61, 2, 7, 20, 0x00000000}, + {0x15, 2, 0, 10, 0x00000000}, + {0x61, 2, 7, 28, 0x00000000}, + {0x55, 2, 0, 8, 0x0000a888}, + {0xbf, 2, 7, 0, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000000}, + {0xbf, 1, 6, 0, 0x00000000}, + {0x07, 1, 0, 0, 0x00000012}, + {0x2d, 1, 9, 1622, 0x00000000}, + {0xb7, 1, 0, 0, 0x00000012}, + {0x69, 8, 6, 16, 0x00000000}, + {0xbf, 7, 2, 0, 0x00000000}, + {0x7b, 10, 7, -56, 0x00000000}, + {0x57, 8, 0, 0, 0x0000ffff}, + {0x15, 8, 0, 409, 0x0000dd86}, + {0xb7, 7, 0, 0, 0x00000003}, + {0x55, 8, 0, 1614, 0x00000008}, + {0x0f, 6, 1, 0, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000000}, + {0xbf, 1, 6, 0, 0x00000000}, + {0x07, 1, 0, 0, 0x00000018}, + {0x2d, 1, 9, 1609, 0x00000000}, + {0x71, 3, 6, 12, 0x00000000}, + {0xbf, 1, 3, 0, 0x00000000}, + {0x67, 1, 0, 0, 0x00000038}, + {0xc7, 1, 0, 0, 0x00000020}, + {0x77, 1, 0, 0, 0x0000001f}, + {0x57, 1, 0, 0, 0x2cc681d1}, + {0x67, 3, 0, 0, 0x00000018}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x40000000}, + {0xb7, 2, 0, 0, 0x00000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x598d03a2}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x20000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb31a0745}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x10000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x66340e8a}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x08000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xcc681d15}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x04000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x98d03a2b}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x02000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x31a07456}, + {0x57, 3, 0, 0, 0x01000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6340e8ad}, + {0x71, 3, 6, 13, 0x00000000}, + {0x67, 3, 0, 0, 0x00000010}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00800000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc681d15b}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00400000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8d03a2b7}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00200000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1a07456f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00100000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x340e8ade}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00080000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x681d15bd}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00040000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd03a2b7b}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00020000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa07456f6}, + {0x57, 3, 0, 0, 0x00010000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x40e8aded}, + {0x71, 3, 6, 14, 0x00000000}, + {0x67, 3, 0, 0, 0x00000008}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00008000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x81d15bdb}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00004000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x03a2b7b7}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00002000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x07456f6f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00001000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0e8adedf}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000800}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1d15bdbf}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000400}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3a2b7b7e}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000200}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7456f6fd}, + {0x57, 3, 0, 0, 0x00000100}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe8adedfa}, + {0x71, 3, 6, 15, 0x00000000}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000080}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd15bdbf4}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000040}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa2b7b7e9}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000020}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x456f6fd3}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000010}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8adedfa7}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000008}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x15bdbf4f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000004}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2b7b7e9e}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000002}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x56f6fd3d}, + {0x57, 3, 0, 0, 0x00000001}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xadedfa7b}, + {0x71, 4, 6, 16, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000038}, + {0xc7, 5, 0, 0, 0x00000020}, + {0xb7, 3, 0, 0, 0xffffffff}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x5bdbf4f7}, + {0x67, 4, 0, 0, 0x00000018}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb7b7e9ef}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6f6fd3df}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdedfa7bf}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbdbf4f7f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7b7e9eff}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf6fd3dff}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xedfa7bfe}, + {0x71, 4, 6, 17, 0x00000000}, + {0x67, 4, 0, 0, 0x00000010}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdbf4f7fc}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb7e9eff9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6fd3dff2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdfa7bfe5}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbf4f7fca}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7e9eff94}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfd3dff28}, + {0x57, 4, 0, 0, 0x00010000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfa7bfe51}, + {0x71, 4, 6, 18, 0x00000000}, + {0x67, 4, 0, 0, 0x00000008}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf4f7fca2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe9eff945}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd3dff28a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa7bfe514}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4f7fca28}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9eff9450}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3dff28a0}, + {0x57, 4, 0, 0, 0x00000100}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7bfe5141}, + {0x71, 4, 6, 19, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf7fca283}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xeff94506}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdff28a0c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbfe51418}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7fca2831}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xff945063}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xff28a0c6}, + {0x57, 4, 0, 0, 0x00000001}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfe51418c}, + {0x71, 4, 6, 20, 0x00000000}, + {0x67, 4, 0, 0, 0x00000008}, + {0x71, 5, 6, 21, 0x00000000}, + {0x4f, 4, 5, 0, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000030}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfca28319}, + {0x67, 4, 0, 0, 0x00000010}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x40000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf9450633}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x20000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf28a0c67}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x10000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe51418ce}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x08000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xca28319d}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x04000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9450633b}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x02000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x28a0c676}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x01000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x51418ced}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00800000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa28319db}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00400000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x450633b6}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00200000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8a0c676c}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00100000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1418ced8}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00080000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x28319db1}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00040000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x50633b63}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00020000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa0c676c6}, + {0x57, 4, 0, 0, 0x00010000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x418ced8d}, + {0x71, 3, 6, 22, 0x00000000}, + {0x67, 3, 0, 0, 0x00000008}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00008000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8319db1a}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00004000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0633b634}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00002000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0c676c68}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00001000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x18ced8d1}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000800}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x319db1a3}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000400}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x633b6347}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000200}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc676c68f}, + {0x57, 3, 0, 0, 0x00000100}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8ced8d1f}, + {0x71, 3, 6, 23, 0x00000000}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000080}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x19db1a3e}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000040}, + {0x79, 5, 10, -56, 0x00000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x33b6347d}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000020}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x676c68fa}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000010}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xced8d1f4}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000008}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9db1a3e9}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000004}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3b6347d2}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000002}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x76c68fa5}, + {0x57, 3, 0, 0, 0x00000001}, + {0x1d, 3, 2, 1177, 0x00000000}, + {0xa7, 1, 0, 0, 0xed8d1f4a}, + {0x05, 0, 0, 1175, 0x00000000}, + {0x0f, 6, 1, 0, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000000}, + {0xbf, 1, 6, 0, 0x00000000}, + {0x07, 1, 0, 0, 0x0000002c}, + {0x2d, 1, 9, 1202, 0x00000000}, + {0x61, 4, 6, 8, 0x00000000}, + {0xbf, 1, 4, 0, 0x00000000}, + {0x67, 1, 0, 0, 0x00000038}, + {0xc7, 1, 0, 0, 0x00000020}, + {0x77, 1, 0, 0, 0x0000001f}, + {0x57, 1, 0, 0, 0x2cc681d1}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000040}, + {0xb7, 2, 0, 0, 0x00000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x598d03a2}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000020}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb31a0745}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000010}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x66340e8a}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000008}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xcc681d15}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000004}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x98d03a2b}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000002}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x31a07456}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000001}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6340e8ad}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00008000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc681d15b}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00004000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8d03a2b7}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00002000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1a07456f}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00001000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x340e8ade}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000800}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x681d15bd}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000400}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd03a2b7b}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000200}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa07456f6}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000100}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x40e8aded}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00800000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x81d15bdb}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00400000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x03a2b7b7}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00200000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x07456f6f}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00100000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0e8adedf}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00080000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1d15bdbf}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00040000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3a2b7b7e}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00020000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7456f6fd}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00010000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe8adedfa}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0xb7, 3, 0, 0, 0xffffffff}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd15bdbf4}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa2b7b7e9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x456f6fd3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8adedfa7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x15bdbf4f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2b7b7e9e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x56f6fd3d}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xadedfa7b}, + {0x61, 4, 6, 12, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x5bdbf4f7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb7b7e9ef}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6f6fd3df}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdedfa7bf}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbdbf4f7f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7b7e9eff}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf6fd3dff}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xedfa7bfe}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdbf4f7fc}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb7e9eff9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6fd3dff2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdfa7bfe5}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbf4f7fca}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7e9eff94}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfd3dff28}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfa7bfe51}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf4f7fca2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe9eff945}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd3dff28a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa7bfe514}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4f7fca28}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9eff9450}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3dff28a0}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7bfe5141}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf7fca283}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xeff94506}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdff28a0c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbfe51418}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7fca2831}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xff945063}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xff28a0c6}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfe51418c}, + {0x61, 4, 6, 16, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfca28319}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf9450633}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf28a0c67}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe51418ce}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xca28319d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9450633b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x28a0c676}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x51418ced}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa28319db}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x450633b6}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8a0c676c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1418ced8}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x28319db1}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x50633b63}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa0c676c6}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x418ced8d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8319db1a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0633b634}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0c676c68}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x18ced8d1}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x319db1a3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x633b6347}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc676c68f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8ced8d1f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x19db1a3e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x33b6347d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x676c68fa}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xced8d1f4}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9db1a3e9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3b6347d2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x76c68fa5}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xed8d1f4a}, + {0x61, 4, 6, 20, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdb1a3e94}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb6347d28}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6c68fa51}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd8d1f4a3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb1a3e946}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6347d28d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc68fa51a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8d1f4a35}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1a3e946b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x347d28d7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x68fa51ae}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd1f4a35c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa3e946b9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x47d28d73}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8fa51ae7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1f4a35cf}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3e946b9e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7d28d73c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfa51ae78}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf4a35cf1}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe946b9e3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd28d73c7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa51ae78e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4a35cf1c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x946b9e38}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x28d73c71}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x51ae78e3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa35cf1c6}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x46b9e38d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8d73c71b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1ae78e36}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x35cf1c6c}, + {0x61, 4, 6, 24, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6b9e38d9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd73c71b2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xae78e364}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x5cf1c6c9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb9e38d92}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x73c71b25}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe78e364b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xcf1c6c96}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9e38d92c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3c71b259}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x78e364b2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf1c6c964}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe38d92c9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc71b2593}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8e364b27}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1c6c964e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x38d92c9c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x71b25938}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe364b270}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc6c964e0}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8d92c9c0}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1b259380}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x364b2700}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6c964e01}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd92c9c03}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb2593807}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x64b2700f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc964e01e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x92c9c03d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2593807a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4b2700f4}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x964e01e8}, + {0x61, 4, 6, 28, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2c9c03d1}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x593807a3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb2700f46}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x64e01e8d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc9c03d1a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x93807a35}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2700f46b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4e01e8d6}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9c03d1ad}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3807a35b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x700f46b6}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe01e8d6c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc03d1ad9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x807a35b3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x00f46b66}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x01e8d6cc}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x03d1ad99}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x07a35b32}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0f46b665}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1e8d6cca}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3d1ad994}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7a35b328}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf46b6651}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe8d6cca2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd1ad9944}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa35b3289}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x46b66512}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8d6cca25}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1ad9944a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x35b32894}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6b665129}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd6cca253}, + {0x61, 4, 6, 32, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xad9944a7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x5b32894f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb665129f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6cca253e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd9944a7d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb32894fb}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x665129f6}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xcca253ec}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9944a7d9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x32894fb2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x65129f65}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xca253eca}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x944a7d95}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2894fb2a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x5129f655}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa253ecab}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x44a7d956}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x894fb2ac}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x129f6558}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x253ecab1}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4a7d9563}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x94fb2ac7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x29f6558f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x53ecab1e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa7d9563d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4fb2ac7a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9f6558f5}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3ecab1ea}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7d9563d5}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfb2ac7ab}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf6558f56}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xecab1eac}, + {0x61, 4, 6, 36, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd9563d59}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb2ac7ab2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6558f564}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xcab1eac8}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9563d590}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2ac7ab20}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x558f5641}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xab1eac83}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x563d5906}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xac7ab20c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x58f56418}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb1eac831}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x63d59063}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc7ab20c7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8f56418f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1eac831e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3d59063c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7ab20c78}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf56418f0}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xeac831e1}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd59063c2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xab20c784}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x56418f09}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xac831e12}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x59063c25}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb20c784b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6418f097}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc831e12f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9063c25f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x20c784be}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x418f097c}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x831e12f9}, + {0x71, 4, 6, 40, 0x00000000}, + {0x67, 4, 0, 0, 0x00000008}, + {0x71, 5, 6, 41, 0x00000000}, + {0x4f, 4, 5, 0, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000030}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x063c25f3}, + {0x67, 4, 0, 0, 0x00000010}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x40000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0c784be7}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x20000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x18f097cf}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x10000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x31e12f9f}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x08000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x63c25f3f}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x04000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc784be7f}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x02000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8f097cff}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x01000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1e12f9fe}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00800000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3c25f3fc}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00400000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x784be7f8}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00200000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf097cff0}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00100000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe12f9fe0}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00080000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc25f3fc1}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00040000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x84be7f83}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00020000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x097cff07}, + {0x57, 4, 0, 0, 0x00010000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x12f9fe0f}, + {0x71, 3, 6, 42, 0x00000000}, + {0x67, 3, 0, 0, 0x00000008}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00008000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x25f3fc1f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00004000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4be7f83f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00002000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x97cff07f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00001000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2f9fe0fe}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000800}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x5f3fc1fd}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000400}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbe7f83fb}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000200}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7cff07f7}, + {0x57, 3, 0, 0, 0x00000100}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf9fe0fee}, + {0x71, 3, 6, 43, 0x00000000}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000080}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf3fc1fdc}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000040}, + {0x79, 5, 10, -56, 0x00000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe7f83fb8}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000020}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xcff07f70}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000010}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9fe0fee1}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000008}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3fc1fdc2}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000004}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7f83fb85}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000002}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xff07f70a}, + {0x57, 3, 0, 0, 0x00000001}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfe0fee15}, + {0x71, 2, 0, 201, 0x00000000}, + {0x67, 2, 0, 0, 0x00000008}, + {0x71, 3, 0, 200, 0x00000000}, + {0x4f, 2, 3, 0, 0x00000000}, + {0x71, 3, 0, 203, 0x00000000}, + {0x67, 3, 0, 0, 0x00000008}, + {0x71, 4, 0, 202, 0x00000000}, + {0x4f, 3, 4, 0, 0x00000000}, + {0x67, 3, 0, 0, 0x00000010}, + {0x4f, 3, 2, 0, 0x00000000}, + {0x67, 1, 0, 0, 0x00000020}, + {0x77, 1, 0, 0, 0x00000020}, + {0xbf, 2, 1, 0, 0x00000000}, + {0x3f, 2, 3, 0, 0x00000000}, + {0x2f, 2, 3, 0, 0x00000000}, + {0x1f, 1, 2, 0, 0x00000000}, + {0x57, 1, 0, 0, 0x0000000f}, + {0x67, 1, 0, 0, 0x00000002}, + {0x0f, 0, 1, 0, 0x00000000}, + {0x71, 1, 0, 137, 0x00000000}, + {0x67, 1, 0, 0, 0x00000008}, + {0x71, 2, 0, 136, 0x00000000}, + {0x4f, 1, 2, 0, 0x00000000}, + {0x71, 2, 0, 138, 0x00000000}, + {0x71, 3, 0, 139, 0x00000000}, + {0x67, 3, 0, 0, 0x00000008}, + {0x4f, 3, 2, 0, 0x00000000}, + {0x67, 3, 0, 0, 0x00000010}, + {0x4f, 3, 1, 0, 0x00000000}, + {0x07, 3, 0, 0, 0x7cafe800}, + {0x63, 5, 3, 52, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000001}, + {0xbf, 0, 7, 0, 0x00000000}, + {0x95, 0, 0, 0, 0x00000000}, +}; diff --git a/drivers/net/tap/tap_bpf_program.c b/drivers/net/tap/tap_bpf_program.c new file mode 100644 index 00000000..8abb3b76 --- /dev/null +++ b/drivers/net/tap/tap_bpf_program.c @@ -0,0 +1,224 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 + * Copyright 2017 Mellanox Technologies, Ltd. + */ + +#include <stdint.h> +#include <stdbool.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <asm/types.h> +#include <linux/in.h> +#include <linux/if.h> +#include <linux/if_ether.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/if_tunnel.h> +#include <linux/filter.h> +#include <linux/bpf.h> + +#include "tap_rss.h" + +/** Create IPv4 address */ +#define IPv4(a, b, c, d) ((__u32)(((a) & 0xff) << 24) | \ + (((b) & 0xff) << 16) | \ + (((c) & 0xff) << 8) | \ + ((d) & 0xff)) + +#define PORT(a, b) ((__u16)(((a) & 0xff) << 8) | \ + ((b) & 0xff)) + +/* + * The queue number is offset by a unique QUEUE_OFFSET, to distinguish + * packets that have gone through this rule (skb->cb[1] != 0) from others. + */ +#define QUEUE_OFFSET 0x7cafe800 +#define PIN_GLOBAL_NS 2 + +#define KEY_IDX 0 +#define BPF_MAP_ID_KEY 1 + +struct vlan_hdr { + __be16 proto; + __be16 tci; +}; + +struct bpf_elf_map __attribute__((section("maps"), used)) +map_keys = { + .type = BPF_MAP_TYPE_HASH, + .id = BPF_MAP_ID_KEY, + .size_key = sizeof(__u32), + .size_value = sizeof(struct rss_key), + .max_elem = 256, + .pinning = PIN_GLOBAL_NS, +}; + +__section("cls_q") int +match_q(struct __sk_buff *skb) +{ + __u32 queue = skb->cb[1]; + volatile __u32 q = 0xdeadbeef; + __u32 match_queue = QUEUE_OFFSET + q; + + /* printt("match_q$i() queue = %d\n", queue); */ + + if (queue != match_queue) + return TC_ACT_OK; + + /* queue match */ + skb->cb[1] = 0; + return TC_ACT_UNSPEC; +} + + +struct ipv4_l3_l4_tuple { + __u32 src_addr; + __u32 dst_addr; + __u16 dport; + __u16 sport; +} __attribute__((packed)); + +struct ipv6_l3_l4_tuple { + __u8 src_addr[16]; + __u8 dst_addr[16]; + __u16 dport; + __u16 sport; +} __attribute__((packed)); + +static const __u8 def_rss_key[] = { + 0xd1, 0x81, 0xc6, 0x2c, + 0xf7, 0xf4, 0xdb, 0x5b, + 0x19, 0x83, 0xa2, 0xfc, + 0x94, 0x3e, 0x1a, 0xdb, + 0xd9, 0x38, 0x9e, 0x6b, + 0xd1, 0x03, 0x9c, 0x2c, + 0xa7, 0x44, 0x99, 0xad, + 0x59, 0x3d, 0x56, 0xd9, + 0xf3, 0x25, 0x3c, 0x06, + 0x2a, 0xdc, 0x1f, 0xfc, +}; + +static __u32 __attribute__((always_inline)) +rte_softrss_be(const __u32 *input_tuple, const uint8_t *rss_key, + __u8 input_len) +{ + __u32 i, j, hash = 0; +#pragma unroll + for (j = 0; j < input_len; j++) { +#pragma unroll + for (i = 0; i < 32; i++) { + if (input_tuple[j] & (1 << (31 - i))) { + hash ^= ((const __u32 *)def_rss_key)[j] << i | + (__u32)((uint64_t) + (((const __u32 *)def_rss_key)[j + 1]) + >> (32 - i)); + } + } + } + return hash; +} + +static int __attribute__((always_inline)) +rss_l3_l4(struct __sk_buff *skb) +{ + void *data_end = (void *)(long)skb->data_end; + void *data = (void *)(long)skb->data; + __u16 proto = (__u16)skb->protocol; + __u32 key_idx = 0xdeadbeef; + __u32 hash; + struct rss_key *rsskey; + __u64 off = ETH_HLEN; + int j; + __u8 *key = 0; + __u32 len; + __u32 queue = 0; + + rsskey = map_lookup_elem(&map_keys, &key_idx); + if (!rsskey) { + printt("hash(): rss key is not configured\n"); + return TC_ACT_OK; + } + key = (__u8 *)rsskey->key; + + /* Get correct proto for 802.1ad */ + if (skb->vlan_present && skb->vlan_proto == htons(ETH_P_8021AD)) { + if (data + ETH_ALEN * 2 + sizeof(struct vlan_hdr) + + sizeof(proto) > data_end) + return TC_ACT_OK; + proto = *(__u16 *)(data + ETH_ALEN * 2 + + sizeof(struct vlan_hdr)); + off += sizeof(struct vlan_hdr); + } + + if (proto == htons(ETH_P_IP)) { + if (data + off + sizeof(struct iphdr) + sizeof(__u32) + > data_end) + return TC_ACT_OK; + + __u8 *src_dst_addr = data + off + offsetof(struct iphdr, saddr); + __u8 *src_dst_port = data + off + sizeof(struct iphdr); + struct ipv4_l3_l4_tuple v4_tuple = { + .src_addr = IPv4(*(src_dst_addr + 0), + *(src_dst_addr + 1), + *(src_dst_addr + 2), + *(src_dst_addr + 3)), + .dst_addr = IPv4(*(src_dst_addr + 4), + *(src_dst_addr + 5), + *(src_dst_addr + 6), + *(src_dst_addr + 7)), + .sport = PORT(*(src_dst_port + 0), + *(src_dst_port + 1)), + .dport = PORT(*(src_dst_port + 2), + *(src_dst_port + 3)), + }; + __u8 input_len = sizeof(v4_tuple) / sizeof(__u32); + if (rsskey->hash_fields & (1 << HASH_FIELD_IPV4_L3)) + input_len--; + hash = rte_softrss_be((__u32 *)&v4_tuple, key, 3); + } else if (proto == htons(ETH_P_IPV6)) { + if (data + off + sizeof(struct ipv6hdr) + + sizeof(__u32) > data_end) + return TC_ACT_OK; + __u8 *src_dst_addr = data + off + + offsetof(struct ipv6hdr, saddr); + __u8 *src_dst_port = data + off + + sizeof(struct ipv6hdr); + struct ipv6_l3_l4_tuple v6_tuple; + for (j = 0; j < 4; j++) + *((uint32_t *)&v6_tuple.src_addr + j) = + __builtin_bswap32(*((uint32_t *) + src_dst_addr + j)); + for (j = 0; j < 4; j++) + *((uint32_t *)&v6_tuple.dst_addr + j) = + __builtin_bswap32(*((uint32_t *) + src_dst_addr + 4 + j)); + v6_tuple.sport = PORT(*(src_dst_port + 0), + *(src_dst_port + 1)); + v6_tuple.dport = PORT(*(src_dst_port + 2), + *(src_dst_port + 3)); + + __u8 input_len = sizeof(v6_tuple) / sizeof(__u32); + if (rsskey->hash_fields & (1 << HASH_FIELD_IPV6_L3)) + input_len--; + hash = rte_softrss_be((__u32 *)&v6_tuple, key, 9); + } else { + return TC_ACT_PIPE; + } + + queue = rsskey->queues[(hash % rsskey->nb_queues) & + (TAP_MAX_QUEUES - 1)]; + skb->cb[1] = QUEUE_OFFSET + queue; + /* printt(">>>>> rss_l3_l4 hash=0x%x queue=%u\n", hash, queue); */ + + return TC_ACT_RECLASSIFY; +} + +#define RSS(L) \ + __section(#L) int \ + L ## _hash(struct __sk_buff *skb) \ + { \ + return rss_ ## L (skb); \ + } + +RSS(l3_l4) + +BPF_LICENSE("Dual BSD/GPL"); diff --git a/drivers/net/tap/tap_flow.c b/drivers/net/tap/tap_flow.c index ffc0b85b..551b2d83 100644 --- a/drivers/net/tap/tap_flow.c +++ b/drivers/net/tap/tap_flow.c @@ -1,39 +1,13 @@ -/*- - * BSD LICENSE - * - * Copyright 2017 6WIND S.A. - * Copyright 2017 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. */ #include <errno.h> #include <string.h> +#include <unistd.h> #include <sys/queue.h> +#include <sys/resource.h> #include <rte_byteorder.h> #include <rte_jhash.h> @@ -42,6 +16,7 @@ #include <tap_flow.h> #include <tap_autoconf.h> #include <tap_tcmsgs.h> +#include <tap_rss.h> #ifndef HAVE_TC_FLOWER /* @@ -81,12 +56,80 @@ enum { TCA_FLOWER_KEY_VLAN_ETH_TYPE, /* be16 */ }; #endif +/* + * For kernels < 4.2 BPF related enums may not be defined. + * Runtime checks will be carried out to gracefully report on TC messages that + * are rejected by the kernel. Rejection reasons may be due to: + * 1. enum is not defined + * 2. enum is defined but kernel is not configured to support BPF system calls, + * BPF classifications or BPF actions. + */ +#ifndef HAVE_TC_BPF +enum { + TCA_BPF_UNSPEC, + TCA_BPF_ACT, + TCA_BPF_POLICE, + TCA_BPF_CLASSID, + TCA_BPF_OPS_LEN, + TCA_BPF_OPS, +}; +#endif +#ifndef HAVE_TC_BPF_FD +enum { + TCA_BPF_FD = TCA_BPF_OPS + 1, + TCA_BPF_NAME, +}; +#endif +#ifndef HAVE_TC_ACT_BPF +#define tc_gen \ + __u32 index; \ + __u32 capab; \ + int action; \ + int refcnt; \ + int bindcnt + +struct tc_act_bpf { + tc_gen; +}; + +enum { + TCA_ACT_BPF_UNSPEC, + TCA_ACT_BPF_TM, + TCA_ACT_BPF_PARMS, + TCA_ACT_BPF_OPS_LEN, + TCA_ACT_BPF_OPS, +}; + +#endif +#ifndef HAVE_TC_ACT_BPF_FD +enum { + TCA_ACT_BPF_FD = TCA_ACT_BPF_OPS + 1, + TCA_ACT_BPF_NAME, +}; +#endif + +/* RSS key management */ +enum bpf_rss_key_e { + KEY_CMD_GET = 1, + KEY_CMD_RELEASE, + KEY_CMD_INIT, + KEY_CMD_DEINIT, +}; + +enum key_status_e { + KEY_STAT_UNSPEC, + KEY_STAT_USED, + KEY_STAT_AVAILABLE, +}; #define ISOLATE_HANDLE 1 +#define REMOTE_PROMISCUOUS_HANDLE 2 struct rte_flow { LIST_ENTRY(rte_flow) next; /* Pointer to the next rte_flow structure */ struct rte_flow *remote_flow; /* associated remote flow */ + int bpf_fd[SEC_MAX]; /* list of bfs fds per ELF section */ + uint32_t key_idx; /* RSS rule key index into BPF map */ struct nlmsg msg; }; @@ -104,6 +147,24 @@ struct remote_rule { int mirred; }; +struct action_data { + char id[16]; + + union { + struct tc_gact gact; + struct tc_mirred mirred; + struct skbedit { + struct tc_skbedit skbedit; + uint16_t queue; + } skbedit; + struct bpf { + struct tc_act_bpf bpf; + int bpf_fd; + const char *annotation; + } bpf; + }; +}; + static int tap_flow_create_eth(const struct rte_flow_item *item, void *data); static int tap_flow_create_vlan(const struct rte_flow_item *item, void *data); static int tap_flow_create_ipv4(const struct rte_flow_item *item, void *data); @@ -124,6 +185,10 @@ tap_flow_create(struct rte_eth_dev *dev, const struct rte_flow_action actions[], struct rte_flow_error *error); +static void +tap_flow_free(struct pmd_internals *pmd, + struct rte_flow *flow); + static int tap_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, @@ -134,6 +199,14 @@ tap_flow_isolate(struct rte_eth_dev *dev, int set, struct rte_flow_error *error); +static int bpf_rss_key(enum bpf_rss_key_e cmd, __u32 *key_idx); +static int rss_enable(struct pmd_internals *pmd, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); +static int rss_add_actions(struct rte_flow *flow, struct pmd_internals *pmd, + const struct rte_flow_action_rss *rss, + struct rte_flow_error *error); + static const struct rte_flow_ops tap_flow_ops = { .validate = tap_flow_validate, .create = tap_flow_create, @@ -465,16 +538,16 @@ tap_flow_create_eth(const struct rte_flow_item *item, void *data) return 0; msg = &flow->msg; if (!is_zero_ether_addr(&spec->dst)) { - nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_DST, ETHER_ADDR_LEN, + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_DST, ETHER_ADDR_LEN, &spec->dst.addr_bytes); - nlattr_add(&msg->nh, + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_DST_MASK, ETHER_ADDR_LEN, &mask->dst.addr_bytes); } if (!is_zero_ether_addr(&mask->src)) { - nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_SRC, ETHER_ADDR_LEN, + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_SRC, ETHER_ADDR_LEN, &spec->src.addr_bytes); - nlattr_add(&msg->nh, + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_SRC_MASK, ETHER_ADDR_LEN, &mask->src.addr_bytes); } @@ -526,9 +599,11 @@ tap_flow_create_vlan(const struct rte_flow_item *item, void *data) uint8_t vid = VLAN_ID(tci); if (prio) - nlattr_add8(&msg->nh, TCA_FLOWER_KEY_VLAN_PRIO, prio); + tap_nlattr_add8(&msg->nh, + TCA_FLOWER_KEY_VLAN_PRIO, prio); if (vid) - nlattr_add16(&msg->nh, TCA_FLOWER_KEY_VLAN_ID, vid); + tap_nlattr_add16(&msg->nh, + TCA_FLOWER_KEY_VLAN_ID, vid); } return 0; } @@ -571,19 +646,19 @@ tap_flow_create_ipv4(const struct rte_flow_item *item, void *data) if (!spec) return 0; if (spec->hdr.dst_addr) { - nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST, + tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST, spec->hdr.dst_addr); - nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST_MASK, + tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST_MASK, mask->hdr.dst_addr); } if (spec->hdr.src_addr) { - nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC, + tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC, spec->hdr.src_addr); - nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC_MASK, + tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC_MASK, mask->hdr.src_addr); } if (spec->hdr.next_proto_id) - nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, + tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, spec->hdr.next_proto_id); return 0; } @@ -627,19 +702,20 @@ tap_flow_create_ipv6(const struct rte_flow_item *item, void *data) if (!spec) return 0; if (memcmp(spec->hdr.dst_addr, empty_addr, 16)) { - nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST, + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST, sizeof(spec->hdr.dst_addr), &spec->hdr.dst_addr); - nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST_MASK, + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST_MASK, sizeof(mask->hdr.dst_addr), &mask->hdr.dst_addr); } if (memcmp(spec->hdr.src_addr, empty_addr, 16)) { - nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC, + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC, sizeof(spec->hdr.src_addr), &spec->hdr.src_addr); - nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC_MASK, + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC_MASK, sizeof(mask->hdr.src_addr), &mask->hdr.src_addr); } if (spec->hdr.proto) - nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, spec->hdr.proto); + tap_nlattr_add8(&msg->nh, + TCA_FLOWER_KEY_IP_PROTO, spec->hdr.proto); return 0; } @@ -677,14 +753,14 @@ tap_flow_create_udp(const struct rte_flow_item *item, void *data) if (!flow) return 0; msg = &flow->msg; - nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_UDP); + tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_UDP); if (!spec) return 0; if (spec->hdr.dst_port & mask->hdr.dst_port) - nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_DST, + tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_DST, spec->hdr.dst_port); if (spec->hdr.src_port & mask->hdr.src_port) - nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_SRC, + tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_SRC, spec->hdr.src_port); return 0; } @@ -723,14 +799,14 @@ tap_flow_create_tcp(const struct rte_flow_item *item, void *data) if (!flow) return 0; msg = &flow->msg; - nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_TCP); + tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_TCP); if (!spec) return 0; if (spec->hdr.dst_port & mask->hdr.dst_port) - nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_DST, + tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_DST, spec->hdr.dst_port); if (spec->hdr.src_port & mask->hdr.src_port) - nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_SRC, + tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_SRC, spec->hdr.src_port); return 0; } @@ -816,112 +892,98 @@ tap_flow_item_validate(const struct rte_flow_item *item, } /** - * Transform a DROP/PASSTHRU action item in the provided flow for TC. + * Configure the kernel with a TC action and its configured parameters + * Handled actions: "gact", "mirred", "skbedit", "bpf" * - * @param[in, out] flow - * Flow to be filled. - * @param[in] action - * Appropriate action to be set in the TCA_GACT_PARMS structure. + * @param[in] flow + * Pointer to rte flow containing the netlink message * - * @return - * 0 if checks are alright, -1 otherwise. - */ -static int -add_action_gact(struct rte_flow *flow, int action) -{ - struct nlmsg *msg = &flow->msg; - size_t act_index = 1; - struct tc_gact p = { - .action = action - }; - - if (nlattr_nested_start(msg, TCA_FLOWER_ACT) < 0) - return -1; - if (nlattr_nested_start(msg, act_index++) < 0) - return -1; - nlattr_add(&msg->nh, TCA_ACT_KIND, sizeof("gact"), "gact"); - if (nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0) - return -1; - nlattr_add(&msg->nh, TCA_GACT_PARMS, sizeof(p), &p); - nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */ - nlattr_nested_finish(msg); /* nested act_index */ - nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */ - return 0; -} - -/** - * Transform a MIRRED action item in the provided flow for TC. + * @param[in, out] act_index + * Pointer to action sequence number in the TC command * - * @param[in, out] flow - * Flow to be filled. - * @param[in] ifindex - * Netdevice ifindex, where to mirror/redirect packet to. - * @param[in] action_type - * Either TCA_EGRESS_REDIR for redirection or TCA_EGRESS_MIRROR for mirroring. + * @param[in] adata + * Pointer to struct holding the action parameters * * @return - * 0 if checks are alright, -1 otherwise. + * -1 on failure, 0 on success */ static int -add_action_mirred(struct rte_flow *flow, uint16_t ifindex, uint16_t action_type) +add_action(struct rte_flow *flow, size_t *act_index, struct action_data *adata) { struct nlmsg *msg = &flow->msg; - size_t act_index = 1; - struct tc_mirred p = { - .eaction = action_type, - .ifindex = ifindex, - }; - if (nlattr_nested_start(msg, TCA_FLOWER_ACT) < 0) + if (tap_nlattr_nested_start(msg, (*act_index)++) < 0) return -1; - if (nlattr_nested_start(msg, act_index++) < 0) + + tap_nlattr_add(&msg->nh, TCA_ACT_KIND, + strlen(adata->id) + 1, adata->id); + if (tap_nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0) return -1; - nlattr_add(&msg->nh, TCA_ACT_KIND, sizeof("mirred"), "mirred"); - if (nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0) + if (strcmp("gact", adata->id) == 0) { + tap_nlattr_add(&msg->nh, TCA_GACT_PARMS, sizeof(adata->gact), + &adata->gact); + } else if (strcmp("mirred", adata->id) == 0) { + if (adata->mirred.eaction == TCA_EGRESS_MIRROR) + adata->mirred.action = TC_ACT_PIPE; + else /* REDIRECT */ + adata->mirred.action = TC_ACT_STOLEN; + tap_nlattr_add(&msg->nh, TCA_MIRRED_PARMS, + sizeof(adata->mirred), + &adata->mirred); + } else if (strcmp("skbedit", adata->id) == 0) { + tap_nlattr_add(&msg->nh, TCA_SKBEDIT_PARMS, + sizeof(adata->skbedit.skbedit), + &adata->skbedit.skbedit); + tap_nlattr_add16(&msg->nh, TCA_SKBEDIT_QUEUE_MAPPING, + adata->skbedit.queue); + } else if (strcmp("bpf", adata->id) == 0) { + tap_nlattr_add32(&msg->nh, TCA_ACT_BPF_FD, adata->bpf.bpf_fd); + tap_nlattr_add(&msg->nh, TCA_ACT_BPF_NAME, + strlen(adata->bpf.annotation) + 1, + adata->bpf.annotation); + tap_nlattr_add(&msg->nh, TCA_ACT_BPF_PARMS, + sizeof(adata->bpf.bpf), + &adata->bpf.bpf); + } else { return -1; - if (action_type == TCA_EGRESS_MIRROR) - p.action = TC_ACT_PIPE; - else /* REDIRECT */ - p.action = TC_ACT_STOLEN; - nlattr_add(&msg->nh, TCA_MIRRED_PARMS, sizeof(p), &p); - nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */ - nlattr_nested_finish(msg); /* nested act_index */ - nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */ + } + tap_nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */ + tap_nlattr_nested_finish(msg); /* nested act_index */ return 0; } /** - * Transform a QUEUE action item in the provided flow for TC. + * Helper function to send a serie of TC actions to the kernel * - * @param[in, out] flow - * Flow to be filled. - * @param[in] queue - * Queue id to use. + * @param[in] flow + * Pointer to rte flow containing the netlink message + * + * @param[in] nb_actions + * Number of actions in an array of action structs + * + * @param[in] data + * Pointer to an array of action structs + * + * @param[in] classifier_actions + * The classifier on behave of which the actions are configured * * @return - * 0 if checks are alright, -1 otherwise. + * -1 on failure, 0 on success */ static int -add_action_skbedit(struct rte_flow *flow, uint16_t queue) +add_actions(struct rte_flow *flow, int nb_actions, struct action_data *data, + int classifier_action) { struct nlmsg *msg = &flow->msg; size_t act_index = 1; - struct tc_skbedit p = { - .action = TC_ACT_PIPE - }; + int i; - if (nlattr_nested_start(msg, TCA_FLOWER_ACT) < 0) - return -1; - if (nlattr_nested_start(msg, act_index++) < 0) + if (tap_nlattr_nested_start(msg, classifier_action) < 0) return -1; - nlattr_add(&msg->nh, TCA_ACT_KIND, sizeof("skbedit"), "skbedit"); - if (nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0) - return -1; - nlattr_add(&msg->nh, TCA_SKBEDIT_PARMS, sizeof(p), &p); - nlattr_add16(&msg->nh, TCA_SKBEDIT_QUEUE_MAPPING, queue); - nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */ - nlattr_nested_finish(msg); /* nested act_index */ - nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */ + for (i = 0; i < nb_actions; i++) + if (add_action(flow, &act_index, data + i) < 0) + return -1; + tap_nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */ return 0; } @@ -984,7 +1046,8 @@ priv_flow_process(struct pmd_internals *pmd, return -rte_errno; } else if (flow) { uint16_t group = attr->group << GROUP_SHIFT; - uint16_t prio = group | (attr->priority + PRIORITY_OFFSET); + uint16_t prio = group | (attr->priority + + RSS_PRIORITY_OFFSET + PRIORITY_OFFSET); flow->msg.t.tcm_info = TC_H_MAKE(prio << 16, flow->msg.t.tcm_info); } @@ -1004,8 +1067,8 @@ priv_flow_process(struct pmd_internals *pmd, TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0); } /* use flower filter type */ - nlattr_add(&flow->msg.nh, TCA_KIND, sizeof("flower"), "flower"); - if (nlattr_nested_start(&flow->msg, TCA_OPTIONS) < 0) + tap_nlattr_add(&flow->msg.nh, TCA_KIND, sizeof("flower"), "flower"); + if (tap_nlattr_nested_start(&flow->msg, TCA_OPTIONS) < 0) goto exit_item_not_supported; } for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { @@ -1041,19 +1104,24 @@ priv_flow_process(struct pmd_internals *pmd, } if (flow) { if (data.vlan) { - nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE, + tap_nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE, htons(ETH_P_8021Q)); - nlattr_add16(&flow->msg.nh, + tap_nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_VLAN_ETH_TYPE, data.eth_type ? data.eth_type : htons(ETH_P_ALL)); } else if (data.eth_type) { - nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE, + tap_nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE, data.eth_type); } } if (mirred && flow) { - uint16_t if_index = pmd->if_index; + struct action_data adata = { + .id = "mirred", + .mirred = { + .eaction = mirred, + }, + }; /* * If attr->egress && mirred, then this is a special @@ -1061,9 +1129,13 @@ priv_flow_process(struct pmd_internals *pmd, * redirect packets coming from the DPDK App, out * through the remote netdevice. */ - if (attr->egress) - if_index = pmd->remote_if_index; - if (add_action_mirred(flow, if_index, mirred) < 0) + adata.mirred.ifindex = attr->ingress ? pmd->if_index : + pmd->remote_if_index; + if (mirred == TCA_EGRESS_MIRROR) + adata.mirred.action = TC_ACT_PIPE; + else + adata.mirred.action = TC_ACT_STOLEN; + if (add_actions(flow, 1, &adata, TCA_FLOWER_ACT) < 0) goto exit_action_not_supported; else goto end; @@ -1077,14 +1149,33 @@ priv_flow_process(struct pmd_internals *pmd, if (action) goto exit_action_not_supported; action = 1; - if (flow) - err = add_action_gact(flow, TC_ACT_SHOT); + if (flow) { + struct action_data adata = { + .id = "gact", + .gact = { + .action = TC_ACT_SHOT, + }, + }; + + err = add_actions(flow, 1, &adata, + TCA_FLOWER_ACT); + } } else if (actions->type == RTE_FLOW_ACTION_TYPE_PASSTHRU) { if (action) goto exit_action_not_supported; action = 1; - if (flow) - err = add_action_gact(flow, TC_ACT_UNSPEC); + if (flow) { + struct action_data adata = { + .id = "gact", + .gact = { + /* continue */ + .action = TC_ACT_UNSPEC, + }, + }; + + err = add_actions(flow, 1, &adata, + TCA_FLOWER_ACT); + } } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { const struct rte_flow_action_queue *queue = (const struct rte_flow_action_queue *) @@ -1096,22 +1187,35 @@ priv_flow_process(struct pmd_internals *pmd, if (!queue || (queue->index > pmd->dev->data->nb_rx_queues - 1)) goto exit_action_not_supported; - if (flow) - err = add_action_skbedit(flow, queue->index); + if (flow) { + struct action_data adata = { + .id = "skbedit", + .skbedit = { + .skbedit = { + .action = TC_ACT_PIPE, + }, + .queue = queue->index, + }, + }; + + err = add_actions(flow, 1, &adata, + TCA_FLOWER_ACT); + } } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { - /* Fake RSS support. */ const struct rte_flow_action_rss *rss = (const struct rte_flow_action_rss *) actions->conf; - if (action) + if (action++) goto exit_action_not_supported; - action = 1; - if (!rss || rss->num < 1 || - (rss->queue[0] > pmd->dev->data->nb_rx_queues - 1)) - goto exit_action_not_supported; - if (flow) - err = add_action_skbedit(flow, rss->queue[0]); + + if (!pmd->rss_enabled) { + err = rss_enable(pmd, attr, error); + if (err) + goto exit_action_not_supported; + } + if (flow && rss) + err = rss_add_actions(flow, pmd, rss, error); } else { goto exit_action_not_supported; } @@ -1120,7 +1224,7 @@ priv_flow_process(struct pmd_internals *pmd, } end: if (flow) - nlattr_nested_finish(&flow->msg); /* nested TCA_OPTIONS */ + tap_nlattr_nested_finish(&flow->msg); /* nested TCA_OPTIONS */ return 0; exit_item_not_supported: rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, @@ -1184,6 +1288,38 @@ tap_flow_set_handle(struct rte_flow *flow) } /** + * Free the flow opened file descriptors and allocated memory + * + * @param[in] flow + * Pointer to the flow to free + * + */ +static void +tap_flow_free(struct pmd_internals *pmd, struct rte_flow *flow) +{ + int i; + + if (!flow) + return; + + if (pmd->rss_enabled) { + /* Close flow BPF file descriptors */ + for (i = 0; i < SEC_MAX; i++) + if (flow->bpf_fd[i] != 0) { + close(flow->bpf_fd[i]); + flow->bpf_fd[i] = 0; + } + + /* Release the map key for this RSS rule */ + bpf_rss_key(KEY_CMD_RELEASE, &flow->key_idx); + flow->key_idx = 0; + } + + /* Free flow allocated memory */ + rte_free(flow); +} + +/** * Create a flow. * * @see rte_flow_create() @@ -1232,13 +1368,13 @@ tap_flow_create(struct rte_eth_dev *dev, tap_flow_set_handle(flow); if (priv_flow_process(pmd, attr, items, actions, error, flow, 0)) goto fail; - err = nl_send(pmd->nlsk_fd, &msg->nh); + err = tap_nl_send(pmd->nlsk_fd, &msg->nh); if (err < 0) { rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "couldn't send request to kernel"); goto fail; } - err = nl_recv_ack(pmd->nlsk_fd); + err = tap_nl_recv_ack(pmd->nlsk_fd); if (err < 0) { RTE_LOG(ERR, PMD, "Kernel refused TC filter rule creation (%d): %s\n", @@ -1276,14 +1412,14 @@ tap_flow_create(struct rte_eth_dev *dev, NULL, "rte flow rule validation failed"); goto fail; } - err = nl_send(pmd->nlsk_fd, &msg->nh); + err = tap_nl_send(pmd->nlsk_fd, &msg->nh); if (err < 0) { rte_flow_error_set( error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "Failure sending nl request"); goto fail; } - err = nl_recv_ack(pmd->nlsk_fd); + err = tap_nl_recv_ack(pmd->nlsk_fd); if (err < 0) { RTE_LOG(ERR, PMD, "Kernel refused TC filter rule creation (%d): %s\n", @@ -1301,7 +1437,7 @@ fail: if (remote_flow) rte_free(remote_flow); if (flow) - rte_free(flow); + tap_flow_free(pmd, flow); return NULL; } @@ -1329,13 +1465,13 @@ tap_flow_destroy_pmd(struct pmd_internals *pmd, flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; flow->msg.nh.nlmsg_type = RTM_DELTFILTER; - ret = nl_send(pmd->nlsk_fd, &flow->msg.nh); + ret = tap_nl_send(pmd->nlsk_fd, &flow->msg.nh); if (ret < 0) { rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "couldn't send request to kernel"); goto end; } - ret = nl_recv_ack(pmd->nlsk_fd); + ret = tap_nl_recv_ack(pmd->nlsk_fd); /* If errno is ENOENT, the rule is already no longer in the kernel. */ if (ret < 0 && errno == ENOENT) ret = 0; @@ -1348,18 +1484,19 @@ tap_flow_destroy_pmd(struct pmd_internals *pmd, "couldn't receive kernel ack to our request"); goto end; } + if (remote_flow) { remote_flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; remote_flow->msg.nh.nlmsg_type = RTM_DELTFILTER; - ret = nl_send(pmd->nlsk_fd, &remote_flow->msg.nh); + ret = tap_nl_send(pmd->nlsk_fd, &remote_flow->msg.nh); if (ret < 0) { rte_flow_error_set( error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "Failure sending nl request"); goto end; } - ret = nl_recv_ack(pmd->nlsk_fd); + ret = tap_nl_recv_ack(pmd->nlsk_fd); if (ret < 0 && errno == ENOENT) ret = 0; if (ret < 0) { @@ -1375,7 +1512,7 @@ tap_flow_destroy_pmd(struct pmd_internals *pmd, end: if (remote_flow) rte_free(remote_flow); - rte_free(flow); + tap_flow_free(pmd, flow); return ret; } @@ -1556,9 +1693,15 @@ int tap_flow_implicit_create(struct pmd_internals *pmd, * The ISOLATE rule is always present and must have a static handle, as * the action is changed whether the feature is enabled (DROP) or * disabled (PASSTHRU). + * There is just one REMOTE_PROMISCUOUS rule in all cases. It should + * have a static handle such that adding it twice will fail with EEXIST + * with any kernel version. Remark: old kernels may falsely accept the + * same REMOTE_PROMISCUOUS rules if they had different handles. */ if (idx == TAP_ISOLATE) remote_flow->msg.t.tcm_handle = ISOLATE_HANDLE; + else if (idx == TAP_REMOTE_PROMISC) + remote_flow->msg.t.tcm_handle = REMOTE_PROMISCUOUS_HANDLE; else tap_flow_set_handle(remote_flow); if (priv_flow_process(pmd, attr, items, actions, NULL, @@ -1566,19 +1709,23 @@ int tap_flow_implicit_create(struct pmd_internals *pmd, RTE_LOG(ERR, PMD, "rte flow rule validation failed\n"); goto fail; } - err = nl_send(pmd->nlsk_fd, &msg->nh); + err = tap_nl_send(pmd->nlsk_fd, &msg->nh); if (err < 0) { RTE_LOG(ERR, PMD, "Failure sending nl request\n"); goto fail; } - err = nl_recv_ack(pmd->nlsk_fd); + err = tap_nl_recv_ack(pmd->nlsk_fd); if (err < 0) { + /* Silently ignore re-entering remote promiscuous rule */ + if (errno == EEXIST && idx == TAP_REMOTE_PROMISC) + goto success; RTE_LOG(ERR, PMD, "Kernel refused TC filter rule creation (%d): %s\n", errno, strerror(errno)); goto fail; } LIST_INSERT_HEAD(&pmd->implicit_flows, remote_flow, next); +success: return 0; fail: if (remote_flow) @@ -1632,6 +1779,346 @@ tap_flow_implicit_flush(struct pmd_internals *pmd, struct rte_flow_error *error) return 0; } +#define MAX_RSS_KEYS 256 +#define KEY_IDX_OFFSET (3 * MAX_RSS_KEYS) +#define SEC_NAME_CLS_Q "cls_q" + +const char *sec_name[SEC_MAX] = { + [SEC_L3_L4] = "l3_l4", +}; + +/** + * Enable RSS on tap: create TC rules for queuing. + * + * @param[in, out] pmd + * Pointer to private structure. + * + * @param[in] attr + * Pointer to rte_flow to get flow group + * + * @param[out] error + * Pointer to error reporting if not NULL. + * + * @return 0 on success, negative value on failure. + */ +static int rss_enable(struct pmd_internals *pmd, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + struct rte_flow *rss_flow = NULL; + struct nlmsg *msg = NULL; + /* 4096 is the maximum number of instructions for a BPF program */ + char annotation[64]; + int i; + int err = 0; + + /* unlimit locked memory */ + struct rlimit memlock_limit = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, + }; + setrlimit(RLIMIT_MEMLOCK, &memlock_limit); + + /* Get a new map key for a new RSS rule */ + err = bpf_rss_key(KEY_CMD_INIT, NULL); + if (err < 0) { + rte_flow_error_set( + error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to initialize BPF RSS keys"); + + return -1; + } + + /* + * Create BPF RSS MAP + */ + pmd->map_fd = tap_flow_bpf_rss_map_create(sizeof(__u32), /* key size */ + sizeof(struct rss_key), + MAX_RSS_KEYS); + if (pmd->map_fd < 0) { + RTE_LOG(ERR, PMD, + "Failed to create BPF map (%d): %s\n", + errno, strerror(errno)); + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Kernel too old or not configured " + "to support BPF maps"); + + return -ENOTSUP; + } + + /* + * Add a rule per queue to match reclassified packets and direct them to + * the correct queue. + */ + for (i = 0; i < pmd->dev->data->nb_rx_queues; i++) { + pmd->bpf_fd[i] = tap_flow_bpf_cls_q(i); + if (pmd->bpf_fd[i] < 0) { + RTE_LOG(ERR, PMD, + "Failed to load BPF section %s for queue %d", + SEC_NAME_CLS_Q, i); + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "Kernel too old or not configured " + "to support BPF programs loading"); + + return -ENOTSUP; + } + + rss_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0); + if (!rss_flow) { + RTE_LOG(ERR, PMD, + "Cannot allocate memory for rte_flow"); + return -1; + } + msg = &rss_flow->msg; + tc_init_msg(msg, pmd->if_index, RTM_NEWTFILTER, NLM_F_REQUEST | + NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); + msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL)); + tap_flow_set_handle(rss_flow); + uint16_t group = attr->group << GROUP_SHIFT; + uint16_t prio = group | (i + PRIORITY_OFFSET); + msg->t.tcm_info = TC_H_MAKE(prio << 16, msg->t.tcm_info); + msg->t.tcm_parent = TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0); + + tap_nlattr_add(&msg->nh, TCA_KIND, sizeof("bpf"), "bpf"); + if (tap_nlattr_nested_start(msg, TCA_OPTIONS) < 0) + return -1; + tap_nlattr_add32(&msg->nh, TCA_BPF_FD, pmd->bpf_fd[i]); + snprintf(annotation, sizeof(annotation), "[%s%d]", + SEC_NAME_CLS_Q, i); + tap_nlattr_add(&msg->nh, TCA_BPF_NAME, strlen(annotation) + 1, + annotation); + /* Actions */ + { + struct action_data adata = { + .id = "skbedit", + .skbedit = { + .skbedit = { + .action = TC_ACT_PIPE, + }, + .queue = i, + }, + }; + if (add_actions(rss_flow, 1, &adata, TCA_BPF_ACT) < 0) + return -1; + } + tap_nlattr_nested_finish(msg); /* nested TCA_OPTIONS */ + + /* Netlink message is now ready to be sent */ + if (tap_nl_send(pmd->nlsk_fd, &msg->nh) < 0) + return -1; + err = tap_nl_recv_ack(pmd->nlsk_fd); + if (err < 0) { + RTE_LOG(ERR, PMD, + "Kernel refused TC filter rule creation (%d): %s\n", + errno, strerror(errno)); + return err; + } + LIST_INSERT_HEAD(&pmd->rss_flows, rss_flow, next); + } + + pmd->rss_enabled = 1; + return err; +} + +/** + * Manage bpf RSS keys repository with operations: init, get, release + * + * @param[in] cmd + * Command on RSS keys: init, get, release + * + * @param[in, out] key_idx + * Pointer to RSS Key index (out for get command, in for release command) + * + * @return -1 if couldn't get, release or init the RSS keys, 0 otherwise. + */ +static int bpf_rss_key(enum bpf_rss_key_e cmd, __u32 *key_idx) +{ + __u32 i; + int err = 0; + static __u32 num_used_keys; + static __u32 rss_keys[MAX_RSS_KEYS] = {KEY_STAT_UNSPEC}; + static __u32 rss_keys_initialized; + __u32 key; + + switch (cmd) { + case KEY_CMD_GET: + if (!rss_keys_initialized) { + err = -1; + break; + } + + if (num_used_keys == RTE_DIM(rss_keys)) { + err = -1; + break; + } + + *key_idx = num_used_keys % RTE_DIM(rss_keys); + while (rss_keys[*key_idx] == KEY_STAT_USED) + *key_idx = (*key_idx + 1) % RTE_DIM(rss_keys); + + rss_keys[*key_idx] = KEY_STAT_USED; + + /* + * Add an offset to key_idx in order to handle a case of + * RSS and non RSS flows mixture. + * If a non RSS flow is destroyed it has an eBPF map + * index 0 (initialized on flow creation) and might + * unintentionally remove RSS entry 0 from eBPF map. + * To avoid this issue, add an offset to the real index + * during a KEY_CMD_GET operation and subtract this offset + * during a KEY_CMD_RELEASE operation in order to restore + * the real index. + */ + *key_idx += KEY_IDX_OFFSET; + num_used_keys++; + break; + + case KEY_CMD_RELEASE: + if (!rss_keys_initialized) + break; + + /* + * Subtract offest to restore real key index + * If a non RSS flow is falsely trying to release map + * entry 0 - the offset subtraction will calculate the real + * map index as an out-of-range value and the release operation + * will be silently ignored. + */ + key = *key_idx - KEY_IDX_OFFSET; + if (key >= RTE_DIM(rss_keys)) + break; + + if (rss_keys[key] == KEY_STAT_USED) { + rss_keys[key] = KEY_STAT_AVAILABLE; + num_used_keys--; + } + break; + + case KEY_CMD_INIT: + for (i = 0; i < RTE_DIM(rss_keys); i++) + rss_keys[i] = KEY_STAT_AVAILABLE; + + rss_keys_initialized = 1; + num_used_keys = 0; + break; + + case KEY_CMD_DEINIT: + for (i = 0; i < RTE_DIM(rss_keys); i++) + rss_keys[i] = KEY_STAT_UNSPEC; + + rss_keys_initialized = 0; + num_used_keys = 0; + break; + + default: + break; + } + + return err; +} + +/** + * Add RSS hash calculations and queue selection + * + * @param[in, out] pmd + * Pointer to internal structure. Used to set/get RSS map fd + * + * @param[in] rss + * Pointer to RSS flow actions + * + * @param[out] error + * Pointer to error reporting if not NULL. + * + * @return 0 on success, negative value on failure + */ +static int rss_add_actions(struct rte_flow *flow, struct pmd_internals *pmd, + const struct rte_flow_action_rss *rss, + struct rte_flow_error *error) +{ + /* 4096 is the maximum number of instructions for a BPF program */ + int i; + int err; + struct rss_key rss_entry = { .hash_fields = 0, + .key_size = 0 }; + + /* Get a new map key for a new RSS rule */ + err = bpf_rss_key(KEY_CMD_GET, &flow->key_idx); + if (err < 0) { + rte_flow_error_set( + error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to get BPF RSS key"); + + return -1; + } + + /* Update RSS map entry with queues */ + rss_entry.nb_queues = rss->num; + for (i = 0; i < rss->num; i++) + rss_entry.queues[i] = rss->queue[i]; + rss_entry.hash_fields = + (1 << HASH_FIELD_IPV4_L3_L4) | (1 << HASH_FIELD_IPV6_L3_L4); + + /* Add this RSS entry to map */ + err = tap_flow_bpf_update_rss_elem(pmd->map_fd, + &flow->key_idx, &rss_entry); + + if (err) { + RTE_LOG(ERR, PMD, + "Failed to update BPF map entry #%u (%d): %s\n", + flow->key_idx, errno, strerror(errno)); + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Kernel too old or not configured " + "to support BPF maps updates"); + + return -ENOTSUP; + } + + + /* + * Load bpf rules to calculate hash for this key_idx + */ + + flow->bpf_fd[SEC_L3_L4] = + tap_flow_bpf_calc_l3_l4_hash(flow->key_idx, pmd->map_fd); + if (flow->bpf_fd[SEC_L3_L4] < 0) { + RTE_LOG(ERR, PMD, + "Failed to load BPF section %s (%d): %s\n", + sec_name[SEC_L3_L4], errno, strerror(errno)); + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Kernel too old or not configured " + "to support BPF program loading"); + + return -ENOTSUP; + } + + /* Actions */ + { + struct action_data adata[] = { + { + .id = "bpf", + .bpf = { + .bpf_fd = flow->bpf_fd[SEC_L3_L4], + .annotation = sec_name[SEC_L3_L4], + .bpf = { + .action = TC_ACT_PIPE, + }, + }, + }, + }; + + if (add_actions(flow, RTE_DIM(adata), adata, + TCA_FLOWER_ACT) < 0) + return -1; + } + + return 0; +} + /** * Manage filter operations. * diff --git a/drivers/net/tap/tap_flow.h b/drivers/net/tap/tap_flow.h index 9e332b03..ac6a952d 100644 --- a/drivers/net/tap/tap_flow.h +++ b/drivers/net/tap/tap_flow.h @@ -1,34 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright 2017 6WIND S.A. - * Copyright 2017 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. */ #ifndef _TAP_FLOW_H_ @@ -37,6 +9,7 @@ #include <rte_flow.h> #include <rte_flow_driver.h> #include <rte_eth_tap.h> +#include <tap_autoconf.h> /** * In TC, priority 0 means we require the kernel to allocate one for us. @@ -49,6 +22,7 @@ #define GROUP_MASK (0xf) #define GROUP_SHIFT 12 #define MAX_GROUP GROUP_MASK +#define RSS_PRIORITY_OFFSET RTE_PMD_TAP_MAX_QUEUES /** * These index are actually in reversed order: their priority is processed @@ -67,6 +41,11 @@ enum implicit_rule_index { TAP_REMOTE_MAX_IDX, }; +enum bpf_fd_idx { + SEC_L3_L4, + SEC_MAX, +}; + int tap_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_type filter_type, enum rte_filter_op filter_op, @@ -80,4 +59,10 @@ int tap_flow_implicit_destroy(struct pmd_internals *pmd, int tap_flow_implicit_flush(struct pmd_internals *pmd, struct rte_flow_error *error); +int tap_flow_bpf_cls_q(__u32 queue_idx); +int tap_flow_bpf_calc_l3_l4_hash(__u32 key_idx, int map_fd); +int tap_flow_bpf_rss_map_create(unsigned int key_size, unsigned int value_size, + unsigned int max_entries); +int tap_flow_bpf_update_rss_elem(int fd, void *key, void *value); + #endif /* _TAP_FLOW_H_ */ diff --git a/drivers/net/tap/tap_intr.c b/drivers/net/tap/tap_intr.c new file mode 100644 index 00000000..b0e19914 --- /dev/null +++ b/drivers/net/tap/tap_intr.c @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018 Mellanox Technologies, Ltd. + */ + +/** + * @file + * Interrupts handling for tap driver. + */ + +#include <assert.h> +#include <errno.h> +#include <fcntl.h> +#include <signal.h> +#include <stdint.h> +#include <stdlib.h> +#include <unistd.h> + +#include <rte_eth_tap.h> +#include <rte_errno.h> +#include <rte_interrupts.h> + + +/** + * Unregister Rx interrupts free the queue interrupt vector. + * + * @param dev + * Pointer to the tap rte_eth_dev structure. + */ +static void +tap_rx_intr_vec_uninstall(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct rte_intr_handle *intr_handle = &pmd->intr_handle; + + rte_intr_free_epoll_fd(intr_handle); + free(intr_handle->intr_vec); + intr_handle->intr_vec = NULL; + intr_handle->nb_efd = 0; +} + +/** + * Allocate Rx queue interrupt vector and register Rx interrupts. + * + * @param dev + * Pointer to the tap rte_eth_dev device structure. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +static int +tap_rx_intr_vec_install(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + unsigned int rxqs_n = pmd->dev->data->nb_rx_queues; + struct rte_intr_handle *intr_handle = &pmd->intr_handle; + unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); + unsigned int i; + unsigned int count = 0; + + if (!dev->data->dev_conf.intr_conf.rxq) + return 0; + intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n])); + if (intr_handle->intr_vec == NULL) { + rte_errno = ENOMEM; + RTE_LOG(ERR, PMD, + "failed to allocate memory for interrupt vector," + " Rx interrupts will not be supported"); + return -rte_errno; + } + for (i = 0; i < n; i++) { + struct rx_queue *rxq = pmd->dev->data->rx_queues[i]; + + /* Skip queues that cannot request interrupts. */ + if (!rxq || rxq->fd <= 0) { + /* Use invalid intr_vec[] index to disable entry. */ + intr_handle->intr_vec[i] = + RTE_INTR_VEC_RXTX_OFFSET + + RTE_MAX_RXTX_INTR_VEC_ID; + continue; + } + intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count; + intr_handle->efds[count] = rxq->fd; + count++; + } + if (!count) + tap_rx_intr_vec_uninstall(dev); + else + intr_handle->nb_efd = count; + return 0; +} + +/** + * Register or unregister the Rx interrupts. + * + * @param dev + * Pointer to the tap rte_eth_dev device structure. + * @param set + * should the operation be register or unregister the interrupts. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +tap_rx_intr_vec_set(struct rte_eth_dev *dev, int set) +{ + tap_rx_intr_vec_uninstall(dev); + if (set) + return tap_rx_intr_vec_install(dev); + return 0; +} diff --git a/drivers/net/tap/tap_netlink.c b/drivers/net/tap/tap_netlink.c index ee92e2e7..82c8dc0e 100644 --- a/drivers/net/tap/tap_netlink.c +++ b/drivers/net/tap/tap_netlink.c @@ -1,34 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright 2017 6WIND S.A. - * Copyright 2017 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. */ #include <errno.h> @@ -63,7 +35,7 @@ struct nested_tail { * netlink socket file descriptor on success, -1 otherwise. */ int -nl_init(uint32_t nl_groups) +tap_nl_init(uint32_t nl_groups) { int fd, sndbuf_size = SNDBUF_SIZE, rcvbuf_size = RCVBUF_SIZE; struct sockaddr_nl local = { @@ -101,7 +73,7 @@ nl_init(uint32_t nl_groups) * 0 on success, -1 otherwise. */ int -nl_final(int nlsk_fd) +tap_nl_final(int nlsk_fd) { if (close(nlsk_fd)) { RTE_LOG(ERR, PMD, "Failed to close netlink socket: %s (%d)\n", @@ -123,7 +95,7 @@ nl_final(int nlsk_fd) * the number of sent bytes on success, -1 otherwise. */ int -nl_send(int nlsk_fd, struct nlmsghdr *nh) +tap_nl_send(int nlsk_fd, struct nlmsghdr *nh) { /* man 7 netlink EXAMPLE */ struct sockaddr_nl sa = { @@ -153,7 +125,8 @@ nl_send(int nlsk_fd, struct nlmsghdr *nh) } /** - * Check that the kernel sends an appropriate ACK in response to an nl_send(). + * Check that the kernel sends an appropriate ACK in response + * to an tap_nl_send(). * * @param[in] nlsk_fd * The netlink socket file descriptor used for communication. @@ -162,14 +135,14 @@ nl_send(int nlsk_fd, struct nlmsghdr *nh) * 0 on success, -1 otherwise with errno set. */ int -nl_recv_ack(int nlsk_fd) +tap_nl_recv_ack(int nlsk_fd) { - return nl_recv(nlsk_fd, NULL, NULL); + return tap_nl_recv(nlsk_fd, NULL, NULL); } /** * Receive a message from the kernel on the netlink socket, following an - * nl_send(). + * tap_nl_send(). * * @param[in] nlsk_fd * The netlink socket file descriptor used for communication. @@ -182,7 +155,7 @@ nl_recv_ack(int nlsk_fd) * 0 on success, -1 otherwise with errno set. */ int -nl_recv(int nlsk_fd, int (*cb)(struct nlmsghdr *, void *arg), void *arg) +tap_nl_recv(int nlsk_fd, int (*cb)(struct nlmsghdr *, void *arg), void *arg) { /* man 7 netlink EXAMPLE */ struct sockaddr_nl sa; @@ -247,7 +220,7 @@ nl_recv(int nlsk_fd, int (*cb)(struct nlmsghdr *, void *arg), void *arg) * The data to append. */ void -nlattr_add(struct nlmsghdr *nh, unsigned short type, +tap_nlattr_add(struct nlmsghdr *nh, unsigned short type, unsigned int data_len, const void *data) { /* see man 3 rtnetlink */ @@ -271,9 +244,9 @@ nlattr_add(struct nlmsghdr *nh, unsigned short type, * The data to append. */ void -nlattr_add8(struct nlmsghdr *nh, unsigned short type, uint8_t data) +tap_nlattr_add8(struct nlmsghdr *nh, unsigned short type, uint8_t data) { - nlattr_add(nh, type, sizeof(uint8_t), &data); + tap_nlattr_add(nh, type, sizeof(uint8_t), &data); } /** @@ -287,9 +260,9 @@ nlattr_add8(struct nlmsghdr *nh, unsigned short type, uint8_t data) * The data to append. */ void -nlattr_add16(struct nlmsghdr *nh, unsigned short type, uint16_t data) +tap_nlattr_add16(struct nlmsghdr *nh, unsigned short type, uint16_t data) { - nlattr_add(nh, type, sizeof(uint16_t), &data); + tap_nlattr_add(nh, type, sizeof(uint16_t), &data); } /** @@ -303,14 +276,14 @@ nlattr_add16(struct nlmsghdr *nh, unsigned short type, uint16_t data) * The data to append. */ void -nlattr_add32(struct nlmsghdr *nh, unsigned short type, uint32_t data) +tap_nlattr_add32(struct nlmsghdr *nh, unsigned short type, uint32_t data) { - nlattr_add(nh, type, sizeof(uint32_t), &data); + tap_nlattr_add(nh, type, sizeof(uint32_t), &data); } /** * Start a nested netlink attribute. - * It must be followed later by a call to nlattr_nested_finish(). + * It must be followed later by a call to tap_nlattr_nested_finish(). * * @param[in, out] msg * The netlink message where to edit the nested_tails metadata. @@ -321,7 +294,7 @@ nlattr_add32(struct nlmsghdr *nh, unsigned short type, uint32_t data) * -1 if adding a nested netlink attribute failed, 0 otherwise. */ int -nlattr_nested_start(struct nlmsg *msg, uint16_t type) +tap_nlattr_nested_start(struct nlmsg *msg, uint16_t type) { struct nested_tail *tail; @@ -335,7 +308,7 @@ nlattr_nested_start(struct nlmsg *msg, uint16_t type) tail->tail = (struct rtattr *)NLMSG_TAIL(&msg->nh); - nlattr_add(&msg->nh, type, 0, NULL); + tap_nlattr_add(&msg->nh, type, 0, NULL); tail->prev = msg->nested_tails; @@ -346,7 +319,7 @@ nlattr_nested_start(struct nlmsg *msg, uint16_t type) /** * End a nested netlink attribute. - * It follows a call to nlattr_nested_start(). + * It follows a call to tap_nlattr_nested_start(). * In effect, it will modify the nested attribute length to include every bytes * from the nested attribute start, up to here. * @@ -354,7 +327,7 @@ nlattr_nested_start(struct nlmsg *msg, uint16_t type) * The netlink message where to edit the nested_tails metadata. */ void -nlattr_nested_finish(struct nlmsg *msg) +tap_nlattr_nested_finish(struct nlmsg *msg) { struct nested_tail *tail = msg->nested_tails; diff --git a/drivers/net/tap/tap_netlink.h b/drivers/net/tap/tap_netlink.h index 98e13902..fafef840 100644 --- a/drivers/net/tap/tap_netlink.h +++ b/drivers/net/tap/tap_netlink.h @@ -1,34 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright 2017 6WIND S.A. - * Copyright 2017 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. */ #ifndef _TAP_NETLINK_H_ @@ -53,17 +25,18 @@ struct nlmsg { #define NLMSG_TAIL(nlh) (void *)((char *)(nlh) + NLMSG_ALIGN((nlh)->nlmsg_len)) -int nl_init(uint32_t nl_groups); -int nl_final(int nlsk_fd); -int nl_send(int nlsk_fd, struct nlmsghdr *nh); -int nl_recv(int nlsk_fd, int (*callback)(struct nlmsghdr *, void *), void *arg); -int nl_recv_ack(int nlsk_fd); -void nlattr_add(struct nlmsghdr *nh, unsigned short type, - unsigned int data_len, const void *data); -void nlattr_add8(struct nlmsghdr *nh, unsigned short type, uint8_t data); -void nlattr_add16(struct nlmsghdr *nh, unsigned short type, uint16_t data); -void nlattr_add32(struct nlmsghdr *nh, unsigned short type, uint32_t data); -int nlattr_nested_start(struct nlmsg *msg, uint16_t type); -void nlattr_nested_finish(struct nlmsg *msg); +int tap_nl_init(uint32_t nl_groups); +int tap_nl_final(int nlsk_fd); +int tap_nl_send(int nlsk_fd, struct nlmsghdr *nh); +int tap_nl_recv(int nlsk_fd, int (*callback)(struct nlmsghdr *, void *), + void *arg); +int tap_nl_recv_ack(int nlsk_fd); +void tap_nlattr_add(struct nlmsghdr *nh, unsigned short type, + unsigned int data_len, const void *data); +void tap_nlattr_add8(struct nlmsghdr *nh, unsigned short type, uint8_t data); +void tap_nlattr_add16(struct nlmsghdr *nh, unsigned short type, uint16_t data); +void tap_nlattr_add32(struct nlmsghdr *nh, unsigned short type, uint32_t data); +int tap_nlattr_nested_start(struct nlmsg *msg, uint16_t type); +void tap_nlattr_nested_finish(struct nlmsg *msg); #endif /* _TAP_NETLINK_H_ */ diff --git a/drivers/net/tap/tap_rss.h b/drivers/net/tap/tap_rss.h new file mode 100644 index 00000000..3bb0d140 --- /dev/null +++ b/drivers/net/tap/tap_rss.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 Mellanox Technologies, Ltd. + */ + +#ifndef _TAP_RSS_H_ +#define _TAP_RSS_H_ + +#ifndef TAP_MAX_QUEUES +#define TAP_MAX_QUEUES 16 +#endif + +/* hashed fields for RSS */ +enum hash_field { + HASH_FIELD_IPV4_L3, /* IPv4 src/dst addr */ + HASH_FIELD_IPV4_L3_L4, /* IPv4 src/dst addr + L4 src/dst ports */ + HASH_FIELD_IPV6_L3, /* IPv6 src/dst addr */ + HASH_FIELD_IPV6_L3_L4, /* IPv6 src/dst addr + L4 src/dst ports */ + HASH_FIELD_L2_SRC, /* Ethernet src addr */ + HASH_FIELD_L2_DST, /* Ethernet dst addr */ + HASH_FIELD_L3_SRC, /* L3 src addr */ + HASH_FIELD_L3_DST, /* L3 dst addr */ + HASH_FIELD_L4_SRC, /* TCP/UDP src ports */ + HASH_FIELD_L4_DST, /* TCP/UDP dst ports */ +}; + +struct rss_key { + __u8 key[128]; + __u32 hash_fields; + __u32 key_size; + __u32 queues[TAP_MAX_QUEUES]; + __u32 nb_queues; +} __attribute__((packed)); + +#endif /* _TAP_RSS_H_ */ diff --git a/drivers/net/tap/tap_tcmsgs.c b/drivers/net/tap/tap_tcmsgs.c index d74ac805..954f13eb 100644 --- a/drivers/net/tap/tap_tcmsgs.c +++ b/drivers/net/tap/tap_tcmsgs.c @@ -1,34 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright 2017 6WIND S.A. - * Copyright 2017 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. */ #include <inttypes.h> @@ -107,7 +79,7 @@ qdisc_del(int nlsk_fd, uint16_t ifindex, struct qdisc *qinfo) msg.t.tcm_parent = qinfo->parent; /* if no netlink socket is provided, create one */ if (!nlsk_fd) { - fd = nl_init(0); + fd = tap_nl_init(0); if (fd < 0) { RTE_LOG(ERR, PMD, "Could not delete QDISC: null netlink socket\n"); @@ -116,16 +88,16 @@ qdisc_del(int nlsk_fd, uint16_t ifindex, struct qdisc *qinfo) } else { fd = nlsk_fd; } - if (nl_send(fd, &msg.nh) < 0) + if (tap_nl_send(fd, &msg.nh) < 0) goto error; - if (nl_recv_ack(fd) < 0) + if (tap_nl_recv_ack(fd) < 0) goto error; if (!nlsk_fd) - return nl_final(fd); + return tap_nl_final(fd); return 0; error: if (!nlsk_fd) - nl_final(fd); + tap_nl_final(fd); return -1; } @@ -150,11 +122,11 @@ qdisc_add_multiq(int nlsk_fd, uint16_t ifindex) NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); msg.t.tcm_handle = TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0); msg.t.tcm_parent = TC_H_ROOT; - nlattr_add(&msg.nh, TCA_KIND, sizeof("multiq"), "multiq"); - nlattr_add(&msg.nh, TCA_OPTIONS, sizeof(opt), &opt); - if (nl_send(nlsk_fd, &msg.nh) < 0) + tap_nlattr_add(&msg.nh, TCA_KIND, sizeof("multiq"), "multiq"); + tap_nlattr_add(&msg.nh, TCA_OPTIONS, sizeof(opt), &opt); + if (tap_nl_send(nlsk_fd, &msg.nh) < 0) return -1; - if (nl_recv_ack(nlsk_fd) < 0) + if (tap_nl_recv_ack(nlsk_fd) < 0) return -1; return 0; } @@ -179,10 +151,10 @@ qdisc_add_ingress(int nlsk_fd, uint16_t ifindex) NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); msg.t.tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0); msg.t.tcm_parent = TC_H_INGRESS; - nlattr_add(&msg.nh, TCA_KIND, sizeof("ingress"), "ingress"); - if (nl_send(nlsk_fd, &msg.nh) < 0) + tap_nlattr_add(&msg.nh, TCA_KIND, sizeof("ingress"), "ingress"); + if (tap_nl_send(nlsk_fd, &msg.nh) < 0) return -1; - if (nl_recv_ack(nlsk_fd) < 0) + if (tap_nl_recv_ack(nlsk_fd) < 0) return -1; return 0; } @@ -246,9 +218,9 @@ qdisc_iterate(int nlsk_fd, uint16_t ifindex, }; tc_init_msg(&msg, ifindex, RTM_GETQDISC, NLM_F_REQUEST | NLM_F_DUMP); - if (nl_send(nlsk_fd, &msg.nh) < 0) + if (tap_nl_send(nlsk_fd, &msg.nh) < 0) return -1; - if (nl_recv(nlsk_fd, callback, &args) < 0) + if (tap_nl_recv(nlsk_fd, callback, &args) < 0) return -1; return 0; } diff --git a/drivers/net/tap/tap_tcmsgs.h b/drivers/net/tap/tap_tcmsgs.h index 78959577..f72f8c5c 100644 --- a/drivers/net/tap/tap_tcmsgs.h +++ b/drivers/net/tap/tap_tcmsgs.h @@ -1,39 +1,12 @@ -/*- - * BSD LICENSE - * - * Copyright 2017 6WIND S.A. - * Copyright 2017 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. */ #ifndef _TAP_TCMSGS_H_ #define _TAP_TCMSGS_H_ +#include <tap_autoconf.h> #include <linux/if_ether.h> #include <linux/rtnetlink.h> #include <linux/pkt_sched.h> @@ -41,6 +14,9 @@ #include <linux/tc_act/tc_mirred.h> #include <linux/tc_act/tc_gact.h> #include <linux/tc_act/tc_skbedit.h> +#ifdef HAVE_TC_ACT_BPF +#include <linux/tc_act/tc_bpf.h> +#endif #include <inttypes.h> #include <rte_ether.h> |