diff options
39 files changed, 1117 insertions, 622 deletions
@@ -4,11 +4,11 @@ project_creation_date: '2016-05-19' project_category: '' lifecycle_state: 'Incubation' project_lead: &tldk_ptl - name: 'n/a' - email: '' - id: '' - company: '' - timezone: '' + name: 'Konstantin Ananyev' + email: 'konstantin.ananyev@intel.com' + id: 'konstantin_ananyev' + company: 'intel' + timezone: 'Atlantis/Shangri-la' primary_contact: *tldk_ptl issue_tracking: type: 'jira' @@ -23,13 +23,13 @@ realtime_discussion: server: 'freenode' channel: 'fdio-tldk' meetings: - - type: 'n/a' + - type: '' agenda: '' url: 'https://wiki.fd.io/view/TLDK/Meeting' - server: 'n/a' - channel: 'n/a' - repeats: 'n/a' - time: 'n/a' + server: '' + channel: '' + repeats: '' + time: '' repositories: - 'tldk' committers: @@ -49,18 +49,9 @@ committers: email: 'keith.wiles@intel.com' id: 'keithwiles' timezone: 'Atlantis/Shangri-la' - - name: 'Konstantin Ananyev' - company: 'intel' - email: 'konstantin.ananyev@intel.com' - id: 'konstantin_ananyev' - timezone: 'Atlantis/Shangri-la' tsc: - # yamllint disable rule:line-length - approval: '' + approval: 'https://wiki.fd.io/view/TSC/Meeting_Minutes' changes: - - type: 'removal' - name: '' - link: '' - - type: 'promotion' + - type: '' name: '' link: '' diff --git a/app/nginx/auto/modules b/app/nginx/auto/modules index f1791c9..cdba3a8 100644 --- a/app/nginx/auto/modules +++ b/app/nginx/auto/modules @@ -1239,9 +1239,10 @@ if [ $USE_TLDK = YES ]; then src/tldk/tldk_event.c src/tldk/parse.c" ngx_module_libs="-L${TLDK_ROOT}/${RTE_TARGET}/lib -Wl,--whole-archive - -ltle_l4p -ltle_dring -ltle_timer -Wl,--no-whole-archive - -L${RTE_SDK}/${RTE_TARGET}/lib -Wl,--whole-archive -ldpdk - -lm -lpcap -lnuma -Wl,--no-whole-archive" + -ltle_l4p -ltle_dring -ltle_memtank -ltle_timer + -Wl,--no-whole-archive -L${RTE_SDK}/${RTE_TARGET}/lib + -Wl,--whole-archive -ldpdk -lm -lpcap -lnuma + -Wl,--no-whole-archive" ngx_module_link=YES ngx_module_order= diff --git a/app/nginx/src/tldk/be.c b/app/nginx/src/tldk/be.c index 1b7b496..b226c26 100644 --- a/app/nginx/src/tldk/be.c +++ b/app/nginx/src/tldk/be.c @@ -204,8 +204,8 @@ port_init(const struct tldk_port_conf *pcf) port_conf.rxmode.offloads |= pcf->rx_offload & RX_CSUM_OFFLOAD; } - port_conf.rxmode.max_rx_pkt_len = pcf->mtu + ETHER_CRC_LEN; - if (port_conf.rxmode.max_rx_pkt_len > ETHER_MAX_LEN) + port_conf.rxmode.max_rx_pkt_len = pcf->mtu + RTE_ETHER_CRC_LEN; + if (port_conf.rxmode.max_rx_pkt_len > RTE_ETHER_MAX_LEN) port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; port_conf.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_IP | ETH_RSS_TCP; @@ -240,8 +240,7 @@ be_check_lcore(uint32_t lid) if (rte_get_master_lcore() != lid && rte_eal_get_lcore_state(lid) == RUNNING) { - RTE_LOG(ERR, USER1, "lcore %u already running %p\n", - lid, lcore_config[lid].f); + RTE_LOG(ERR, USER1, "lcore %u already in use\n", lid); return -EINVAL; } @@ -424,31 +423,31 @@ fill_dst(struct tle_dest *dst, const struct tldk_dev *td, const struct tldk_port_conf *pcf, const struct tldk_dest_conf *dest, uint16_t l3_type, struct rte_mempool *mp) { - struct ether_hdr *eth; - struct ipv4_hdr *ip4h; - struct ipv6_hdr *ip6h; + struct rte_ether_hdr *eth; + struct rte_ipv4_hdr *ip4h; + struct rte_ipv6_hdr *ip6h; dst->dev = td->dev; dst->head_mp = mp; dst->mtu = RTE_MIN(dest->mtu, pcf->mtu); dst->l2_len = sizeof(*eth); - eth = (struct ether_hdr *)dst->hdr; + eth = (struct rte_ether_hdr *)dst->hdr; - ether_addr_copy(&pcf->mac, ð->s_addr); - ether_addr_copy(&dest->mac, ð->d_addr); + rte_ether_addr_copy(&pcf->mac, ð->s_addr); + rte_ether_addr_copy(&dest->mac, ð->d_addr); eth->ether_type = rte_cpu_to_be_16(l3_type); - if (l3_type == ETHER_TYPE_IPv4) { + if (l3_type == RTE_ETHER_TYPE_IPV4) { dst->l3_len = sizeof(*ip4h); - ip4h = (struct ipv4_hdr *)(eth + 1); + ip4h = (struct rte_ipv4_hdr *)(eth + 1); ip4h->version_ihl = 4 << 4 | - sizeof(*ip4h) / IPV4_IHL_MULTIPLIER; + sizeof(*ip4h) / RTE_IPV4_IHL_MULTIPLIER; ip4h->time_to_live = 64; ip4h->next_proto_id = IPPROTO_TCP; - } else if (l3_type == ETHER_TYPE_IPv6) { + } else if (l3_type == RTE_ETHER_TYPE_IPV6) { dst->l3_len = sizeof(*ip6h); - ip6h = (struct ipv6_hdr *)(eth + 1); + ip6h = (struct rte_ipv6_hdr *)(eth + 1); ip6h->vtc_flow = 6 << 4; ip6h->proto = IPPROTO_TCP; ip6h->hop_limits = 64; @@ -469,12 +468,12 @@ be_add_dest(const struct tldk_dest_conf *dcf, struct tldk_ctx *tcx, n = tcx->dst4_num; dp = tcx->dst4 + n; m = RTE_DIM(tcx->dst4); - l3_type = ETHER_TYPE_IPv4; + l3_type = RTE_ETHER_TYPE_IPV4; } else { n = tcx->dst6_num; dp = tcx->dst6 + n; m = RTE_DIM(tcx->dst6); - l3_type = ETHER_TYPE_IPv6; + l3_type = RTE_ETHER_TYPE_IPV6; } if (n + dnum >= m) { @@ -650,9 +649,9 @@ fill_pkt_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t l3, uint32_t l4) } static inline int -is_ipv4_frag(const struct ipv4_hdr *iph) +is_ipv4_frag(const struct rte_ipv4_hdr *iph) { - const uint16_t mask = rte_cpu_to_be_16(~IPV4_HDR_DF_FLAG); + const uint16_t mask = rte_cpu_to_be_16(~RTE_IPV4_HDR_DF_FLAG); return ((mask & iph->fragment_offset) != 0); } @@ -660,9 +659,9 @@ is_ipv4_frag(const struct ipv4_hdr *iph) static inline uint32_t get_tcp_header_size(struct rte_mbuf *m, uint32_t l2_len, uint32_t l3_len) { - const struct tcp_hdr *tcp; + const struct rte_tcp_hdr *tcp; - tcp = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, l2_len + l3_len); + tcp = rte_pktmbuf_mtod_offset(m, struct rte_tcp_hdr *, l2_len + l3_len); return (tcp->data_off >> 4) * 4; } @@ -670,9 +669,9 @@ static inline void adjust_ipv4_pktlen(struct rte_mbuf *m, uint32_t l2_len) { uint32_t plen, trim; - const struct ipv4_hdr *iph; + const struct rte_ipv4_hdr *iph; - iph = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, l2_len); + iph = rte_pktmbuf_mtod_offset(m, const struct rte_ipv4_hdr *, l2_len); plen = rte_be_to_cpu_16(iph->total_length) + l2_len; if (plen < m->pkt_len) { trim = m->pkt_len - plen; @@ -684,9 +683,9 @@ static inline void adjust_ipv6_pktlen(struct rte_mbuf *m, uint32_t l2_len) { uint32_t plen, trim; - const struct ipv6_hdr *iph; + const struct rte_ipv6_hdr *iph; - iph = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *, l2_len); + iph = rte_pktmbuf_mtod_offset(m, const struct rte_ipv6_hdr *, l2_len); plen = rte_be_to_cpu_16(iph->payload_len) + sizeof(*iph) + l2_len; if (plen < m->pkt_len) { trim = m->pkt_len - plen; @@ -698,23 +697,24 @@ static inline void tcp_stat_update(struct tldk_ctx *lc, const struct rte_mbuf *m, uint32_t l2_len, uint32_t l3_len) { - const struct tcp_hdr *th; + const struct rte_tcp_hdr *th; - th = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, l2_len + l3_len); + th = rte_pktmbuf_mtod_offset(m, struct rte_tcp_hdr *, l2_len + l3_len); lc->tcp_stat.flags[th->tcp_flags]++; } static inline uint32_t get_ipv4_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t proto, uint32_t frag) { - const struct ipv4_hdr *iph; + const struct rte_ipv4_hdr *iph; int32_t dlen, len; dlen = rte_pktmbuf_data_len(m); dlen -= l2; - iph = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, l2); - len = (iph->version_ihl & IPV4_HDR_IHL_MASK) * IPV4_IHL_MULTIPLIER; + iph = rte_pktmbuf_mtod_offset(m, const struct rte_ipv4_hdr *, l2); + len = (iph->version_ihl & RTE_IPV4_HDR_IHL_MASK) * + RTE_IPV4_IHL_MULTIPLIER; if (frag != 0 && is_ipv4_frag(iph)) { m->packet_type &= ~RTE_PTYPE_L4_MASK; @@ -745,7 +745,7 @@ get_ipv6x_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t nproto, const struct ip6_ext *ipx; int32_t dlen, len, ofs; - len = sizeof(struct ipv6_hdr); + len = sizeof(struct rte_ipv6_hdr); dlen = rte_pktmbuf_data_len(m); dlen -= l2; @@ -795,13 +795,13 @@ get_ipv6x_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t nproto, static inline uint32_t get_ipv6_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t fproto) { - const struct ipv6_hdr *iph; + const struct rte_ipv6_hdr *iph; - iph = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *, - sizeof(struct ether_hdr)); + iph = rte_pktmbuf_mtod_offset(m, const struct rte_ipv6_hdr *, + sizeof(struct rte_ether_hdr)); if (iph->proto == fproto) - return sizeof(struct ipv6_hdr); + return sizeof(struct rte_ipv6_hdr); else if (ipv6x_hdr(iph->proto) != 0) return get_ipv6x_hdr_len(m, l2, iph->proto, fproto); @@ -814,25 +814,25 @@ fill_eth_tcp_hdr_len(struct rte_mbuf *m) { uint32_t dlen, l2_len, l3_len, l4_len; uint16_t etp; - const struct ether_hdr *eth; + const struct rte_ether_hdr *eth; dlen = rte_pktmbuf_data_len(m); /* check that first segment is at least 54B long. */ - if (dlen < sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr) + - sizeof(struct tcp_hdr)) { + if (dlen < sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + + sizeof(struct rte_tcp_hdr)) { m->packet_type = RTE_PTYPE_UNKNOWN; return; } l2_len = sizeof(*eth); - eth = rte_pktmbuf_mtod(m, const struct ether_hdr *); + eth = rte_pktmbuf_mtod(m, const struct rte_ether_hdr *); etp = eth->ether_type; - if (etp == rte_be_to_cpu_16(ETHER_TYPE_VLAN)) - l2_len += sizeof(struct vlan_hdr); + if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_VLAN)) + l2_len += sizeof(struct rte_vlan_hdr); - if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv4)) { + if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_IPV4)) { m->packet_type = RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER; @@ -840,9 +840,9 @@ fill_eth_tcp_hdr_len(struct rte_mbuf *m) l4_len = get_tcp_header_size(m, l2_len, l3_len); fill_pkt_hdr_len(m, l2_len, l3_len, l4_len); adjust_ipv4_pktlen(m, l2_len); - } else if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv6) && - dlen >= l2_len + sizeof(struct ipv6_hdr) + - sizeof(struct tcp_hdr)) { + } else if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_IPV6) && + dlen >= l2_len + sizeof(struct rte_ipv6_hdr) + + sizeof(struct rte_tcp_hdr)) { m->packet_type = RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER; @@ -865,7 +865,7 @@ type0_tcp_rx_callback(__rte_unused dpdk_port_t port, { uint32_t j, tp; uint32_t l4_len, l3_len, l2_len; - const struct ether_hdr *eth; + const struct rte_ether_hdr *eth; l2_len = sizeof(*eth); @@ -881,17 +881,17 @@ type0_tcp_rx_callback(__rte_unused dpdk_port_t port, case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L2_ETHER): l4_len = get_tcp_header_size(pkt[j], l2_len, - sizeof(struct ipv4_hdr)); + sizeof(struct rte_ipv4_hdr)); fill_pkt_hdr_len(pkt[j], l2_len, - sizeof(struct ipv4_hdr), l4_len); + sizeof(struct rte_ipv4_hdr), l4_len); adjust_ipv4_pktlen(pkt[j], l2_len); break; case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L2_ETHER): l4_len = get_tcp_header_size(pkt[j], l2_len, - sizeof(struct ipv6_hdr)); + sizeof(struct rte_ipv6_hdr)); fill_pkt_hdr_len(pkt[j], l2_len, - sizeof(struct ipv6_hdr), l4_len); + sizeof(struct rte_ipv6_hdr), l4_len); adjust_ipv6_pktlen(pkt[j], l2_len); break; case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4_EXT | @@ -931,7 +931,7 @@ type1_tcp_rx_callback(__rte_unused dpdk_port_t port, uint32_t j, tp; struct tldk_ctx *tcx; uint32_t l4_len, l3_len, l2_len; - const struct ether_hdr *eth; + const struct rte_ether_hdr *eth; tcx = user_param; l2_len = sizeof(*eth); diff --git a/app/nginx/src/tldk/module.c b/app/nginx/src/tldk/module.c index 67d9746..8736529 100644 --- a/app/nginx/src/tldk/module.c +++ b/app/nginx/src/tldk/module.c @@ -95,6 +95,8 @@ init_context(struct tldk_ctx *tcx, const struct tldk_ctx_conf *cf, cprm.socket_id = sid; cprm.proto = TLE_PROTO_TCP; cprm.max_streams = cf->nb_stream; + cprm.free_streams.min = cf->free_streams.nb_min; + cprm.free_streams.max = cf->free_streams.nb_max; cprm.max_stream_rbufs = cf->nb_rbuf; cprm.max_stream_sbufs = cf->nb_sbuf; if (cf->be_in_worker != 0) diff --git a/app/nginx/src/tldk/ngx_tldk.h b/app/nginx/src/tldk/ngx_tldk.h index ffd479b..592a555 100644 --- a/app/nginx/src/tldk/ngx_tldk.h +++ b/app/nginx/src/tldk/ngx_tldk.h @@ -58,7 +58,7 @@ struct tldk_port_conf { uint64_t tx_offload; uint32_t ipv4; struct in6_addr ipv6; - struct ether_addr mac; + struct rte_ether_addr mac; }; struct tldk_dev_conf { @@ -76,7 +76,7 @@ struct tldk_dest_conf { struct in_addr ipv4; struct in6_addr ipv6; }; - struct ether_addr mac; + struct rte_ether_addr mac; }; #define TLDK_MAX_DEST 0x10 @@ -86,6 +86,10 @@ struct tldk_ctx_conf { uint32_t lcore; uint32_t nb_mbuf; uint32_t nb_stream; + struct { + uint32_t nb_min; + uint32_t nb_max; + } free_streams; uint32_t nb_rbuf; uint32_t nb_sbuf; uint32_t nb_dev; diff --git a/app/nginx/src/tldk/parse.c b/app/nginx/src/tldk/parse.c index 5d71d9e..455d276 100644 --- a/app/nginx/src/tldk/parse.c +++ b/app/nginx/src/tldk/parse.c @@ -38,7 +38,7 @@ union parse_val { struct in6_addr addr6; }; } in; - struct ether_addr mac; + struct rte_ether_addr mac; rte_cpuset_t cpuset; }; @@ -163,7 +163,7 @@ tldk_port_parse(ngx_conf_t *cf, struct tldk_port_conf *prt) union parse_val pvl[RTE_DIM(kh)]; memset(pvl, 0, sizeof(pvl)); - pvl[1].u64 = ETHER_MAX_LEN - ETHER_CRC_LEN; + pvl[1].u64 = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; if (cf->args->nelts % 2 != 0) return NGX_CONF_ERROR; @@ -286,7 +286,7 @@ tldk_dest_parse(ngx_conf_t *cf, struct tldk_dest_conf *dst) union parse_val pvl[RTE_DIM(kh)]; memset(pvl, 0, sizeof(pvl)); - pvl[1].u64 = ETHER_MAX_LEN - ETHER_CRC_LEN; + pvl[1].u64 = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; if (cf->args->nelts % 2 != 1 || cf->args->nelts == 1) return NGX_CONF_ERROR; @@ -413,6 +413,18 @@ tldk_ctx_parse(ngx_conf_t *cf, ngx_command_t *dummy, void *conf) &pvl) < 0) return NGX_CONF_ERROR; tcx->nb_stream = pvl.u64; + } else if (ngx_strcmp(v[0].data, "min_free_streams") == 0) { + if (cf->args->nelts != 2 || + parse_uint_val((const char *)v[1].data, + &pvl) < 0) + return NGX_CONF_ERROR; + tcx->free_streams.nb_min = pvl.u64; + } else if (ngx_strcmp(v[0].data, "max_free_streams") == 0) { + if (cf->args->nelts != 2 || + parse_uint_val((const char *)v[1].data, + &pvl) < 0) + return NGX_CONF_ERROR; + tcx->free_streams.nb_max = pvl.u64; } else if (ngx_strcmp(v[0].data, "rbufs") == 0) { if (cf->args->nelts != 2 || parse_uint_val((const char *)v[1].data, diff --git a/dpdk/Makefile b/dpdk/Makefile index 15204fa..c474433 100644 --- a/dpdk/Makefile +++ b/dpdk/Makefile @@ -14,7 +14,7 @@ # Scripts require non-POSIX parts of bash SHELL := /bin/bash -DPDK_VERSION ?= v18.11 +DPDK_VERSION ?= v20.05 DPDK_BUILD_DIR ?= $(CURDIR)/_build DPDK_INSTALL_DIR ?= $(DPDK_BUILD_DIR)/dpdk/$(RTE_TARGET) DPDK_PKTMBUF_HEADROOM ?= 128 diff --git a/examples/l4fwd/Makefile b/examples/l4fwd/Makefile index f18b622..a6e0de3 100644 --- a/examples/l4fwd/Makefile +++ b/examples/l4fwd/Makefile @@ -38,6 +38,7 @@ CFLAGS += -I$(RTE_OUTPUT)/include LDLIBS += -L$(RTE_OUTPUT)/lib LDLIBS += -ltle_l4p +LDLIBS += -ltle_memtank LDLIBS += -ltle_timer EXTRA_CFLAGS += -O3 diff --git a/examples/l4fwd/README b/examples/l4fwd/README index dc3dc67..328f7ef 100644 --- a/examples/l4fwd/README +++ b/examples/l4fwd/README @@ -139,6 +139,7 @@ -K | --seckey <string> /* 16 character long secret key used by */ \ /* hash algorithms to generate the */ \ /* sequence number. */ \ + -M | --mbuf-num <num> /* other than default number of mbufs per pool. */ \ <port0_params> <port1_params> ... <portN_params> Note that: options -U and -T cannot be used together. diff --git a/examples/l4fwd/common.h b/examples/l4fwd/common.h index b7750d7..a2cd5f6 100644 --- a/examples/l4fwd/common.h +++ b/examples/l4fwd/common.h @@ -357,31 +357,31 @@ fill_dst(struct tle_dest *dst, struct netbe_dev *bed, const struct netbe_dest *bdp, uint16_t l3_type, int32_t sid, uint8_t proto_id) { - struct ether_hdr *eth; - struct ipv4_hdr *ip4h; - struct ipv6_hdr *ip6h; + struct rte_ether_hdr *eth; + struct rte_ipv4_hdr *ip4h; + struct rte_ipv6_hdr *ip6h; dst->dev = bed->dev; dst->head_mp = frag_mpool[sid + 1]; dst->mtu = RTE_MIN(bdp->mtu, bed->port.mtu); dst->l2_len = sizeof(*eth); - eth = (struct ether_hdr *)dst->hdr; + eth = (struct rte_ether_hdr *)dst->hdr; - ether_addr_copy(&bed->port.mac, ð->s_addr); - ether_addr_copy(&bdp->mac, ð->d_addr); + rte_ether_addr_copy(&bed->port.mac, ð->s_addr); + rte_ether_addr_copy(&bdp->mac, ð->d_addr); eth->ether_type = rte_cpu_to_be_16(l3_type); - if (l3_type == ETHER_TYPE_IPv4) { + if (l3_type == RTE_ETHER_TYPE_IPV4) { dst->l3_len = sizeof(*ip4h); - ip4h = (struct ipv4_hdr *)(eth + 1); + ip4h = (struct rte_ipv4_hdr *)(eth + 1); ip4h->version_ihl = 4 << 4 | - sizeof(*ip4h) / IPV4_IHL_MULTIPLIER; + sizeof(*ip4h) / RTE_IPV4_IHL_MULTIPLIER; ip4h->time_to_live = 64; ip4h->next_proto_id = proto_id; - } else if (l3_type == ETHER_TYPE_IPv6) { + } else if (l3_type == RTE_ETHER_TYPE_IPV6) { dst->l3_len = sizeof(*ip6h); - ip6h = (struct ipv6_hdr *)(eth + 1); + ip6h = (struct rte_ipv6_hdr *)(eth + 1); ip6h->vtc_flow = 6 << 4; ip6h->proto = proto_id; ip6h->hop_limits = 64; @@ -402,12 +402,12 @@ netbe_add_dest(struct netbe_lcore *lc, uint32_t dev_idx, uint16_t family, n = lc->dst4_num; dp = lc->dst4 + n; m = RTE_DIM(lc->dst4); - l3_type = ETHER_TYPE_IPv4; + l3_type = RTE_ETHER_TYPE_IPV4; } else { n = lc->dst6_num; dp = lc->dst6 + n; m = RTE_DIM(lc->dst6); - l3_type = ETHER_TYPE_IPv6; + l3_type = RTE_ETHER_TYPE_IPV6; } if (n + dnum >= m) { @@ -441,21 +441,21 @@ netbe_add_dest(struct netbe_lcore *lc, uint32_t dev_idx, uint16_t family, static inline void fill_arp_reply(struct netbe_dev *dev, struct rte_mbuf *m) { - struct ether_hdr *eth; - struct arp_hdr *ahdr; - struct arp_ipv4 *adata; + struct rte_ether_hdr *eth; + struct rte_arp_hdr *ahdr; + struct rte_arp_ipv4 *adata; uint32_t tip; /* set up the ethernet data */ - eth = rte_pktmbuf_mtod(m, struct ether_hdr *); + eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); eth->d_addr = eth->s_addr; eth->s_addr = dev->port.mac; /* set up the arp data */ - ahdr = rte_pktmbuf_mtod_offset(m, struct arp_hdr *, m->l2_len); + ahdr = rte_pktmbuf_mtod_offset(m, struct rte_arp_hdr *, m->l2_len); adata = &ahdr->arp_data; - ahdr->arp_op = rte_cpu_to_be_16(ARP_OP_REPLY); + ahdr->arp_opcode = rte_cpu_to_be_16(RTE_ARP_OP_REPLY); tip = adata->arp_tip; adata->arp_tip = adata->arp_sip; diff --git a/examples/l4fwd/main.c b/examples/l4fwd/main.c index 9396403..57e9cfd 100644 --- a/examples/l4fwd/main.c +++ b/examples/l4fwd/main.c @@ -61,16 +61,12 @@ RTE_DEFINE_PER_LCORE(struct netfe_lcore *, _fe); static volatile int force_quit; -static struct netbe_cfg becfg; +static struct netbe_cfg becfg = {.mpool_buf_num=MPOOL_NB_BUF}; static struct rte_mempool *mpool[RTE_MAX_NUMA_NODES + 1]; static struct rte_mempool *frag_mpool[RTE_MAX_NUMA_NODES + 1]; static char proto_name[3][10] = {"udp", "tcp", ""}; -static const struct rte_eth_conf port_conf_default = { - .rxmode = { - .offloads = DEV_RX_OFFLOAD_VLAN_STRIP, - }, -}; +static const struct rte_eth_conf port_conf_default; struct tx_content tx_content = { .sz = 0, diff --git a/examples/l4fwd/netbe.h b/examples/l4fwd/netbe.h index 6d9aa35..430bc28 100644 --- a/examples/l4fwd/netbe.h +++ b/examples/l4fwd/netbe.h @@ -75,7 +75,7 @@ struct netbe_port { uint64_t tx_offload; uint32_t ipv4; struct in6_addr ipv6; - struct ether_addr mac; + struct rte_ether_addr mac; uint32_t hash_key_size; uint8_t hash_key[RSS_HASH_KEY_LENGTH]; }; @@ -90,7 +90,7 @@ struct netbe_dest { struct in_addr ipv4; struct in6_addr ipv6; }; - struct ether_addr mac; + struct rte_ether_addr mac; }; struct netbe_dest_prm { @@ -151,6 +151,7 @@ struct netbe_cfg { uint32_t arp; uint32_t prt_num; uint32_t cpu_num; + uint32_t mpool_buf_num; struct netbe_port *prt; struct netbe_lcore *cpu; }; diff --git a/examples/l4fwd/parse.c b/examples/l4fwd/parse.c index 40adee4..b936bab 100644 --- a/examples/l4fwd/parse.c +++ b/examples/l4fwd/parse.c @@ -45,6 +45,9 @@ static const struct { #define OPT_SHORT_PROMISC 'P' #define OPT_LONG_PROMISC "promisc" +#define OPT_SHORT_MBUFNUM 'M' +#define OPT_LONG_MBUFNUM "mbuf-num" + #define OPT_SHORT_RBUFS 'R' #define OPT_LONG_RBUFS "rbufs" @@ -95,6 +98,7 @@ static const struct option long_opt[] = { {OPT_LONG_SBULK, 1, 0, OPT_SHORT_SBULK}, {OPT_LONG_CTXFLAGS, 1, 0, OPT_SHORT_CTXFLAGS}, {OPT_LONG_PROMISC, 0, 0, OPT_SHORT_PROMISC}, + {OPT_LONG_MBUFNUM, 1, 0, OPT_SHORT_MBUFNUM}, {OPT_LONG_RBUFS, 1, 0, OPT_SHORT_RBUFS}, {OPT_LONG_SBUFS, 1, 0, OPT_SHORT_SBUFS}, {OPT_LONG_BECFG, 1, 0, OPT_SHORT_BECFG}, @@ -330,7 +334,7 @@ parse_netbe_arg(struct netbe_port *prt, const char *arg, rte_cpuset_t *pcpu) union parse_val val[RTE_DIM(hndl)]; memset(val, 0, sizeof(val)); - val[2].u64 = ETHER_MAX_LEN - ETHER_CRC_LEN; + val[2].u64 = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; rc = parse_kvargs(arg, keys_man, RTE_DIM(keys_man), keys_opt, RTE_DIM(keys_opt), hndl, val); @@ -373,7 +377,8 @@ check_netbe_dest(const struct netbe_dest *dst) RTE_LOG(ERR, USER1, "%s(line=%u) invalid masklen=%u", __func__, dst->line, dst->prfx); return -EINVAL; - } else if (dst->mtu > ETHER_MAX_JUMBO_FRAME_LEN - ETHER_CRC_LEN) { + } else if (dst->mtu > + RTE_ETHER_MAX_JUMBO_FRAME_LEN - RTE_ETHER_CRC_LEN) { RTE_LOG(ERR, USER1, "%s(line=%u) invalid mtu=%u", __func__, dst->line, dst->mtu); return -EINVAL; @@ -409,7 +414,7 @@ parse_netbe_dest(struct netbe_dest *dst, const char *arg) /* set default values. */ memset(val, 0, sizeof(val)); - val[4].u64 = ETHER_MAX_JUMBO_FRAME_LEN - ETHER_CRC_LEN; + val[4].u64 = RTE_ETHER_MAX_JUMBO_FRAME_LEN - RTE_ETHER_CRC_LEN; rc = parse_kvargs(arg, keys_man, RTE_DIM(keys_man), keys_opt, RTE_DIM(keys_opt), hndl, val); @@ -816,7 +821,7 @@ parse_app_options(int argc, char **argv, struct netbe_cfg *cfg, optind = 0; optarg = NULL; - while ((opt = getopt_long(argc, argv, "aB:C:c:LPR:S:TUb:f:s:v:H:K:W:w:", + while ((opt = getopt_long(argc, argv, "aB:C:c:LPR:S:M:TUb:f:s:v:H:K:W:w:", long_opt, &opt_idx)) != EOF) { if (opt == OPT_SHORT_ARP) { cfg->arp = 1; @@ -834,7 +839,13 @@ parse_app_options(int argc, char **argv, struct netbe_cfg *cfg, "for option: \'%c\'\n", __func__, optarg, opt); ctx_prm->flags = v; - } else if (opt == OPT_SHORT_PROMISC) { + } else if (opt == OPT_SHORT_MBUFNUM) { + rc = parse_uint_val(NULL, optarg, &v); + if (rc < 0) + rte_exit(EXIT_FAILURE, "%s: invalid value: %s " + "for option: \'%c\'\n", + __func__, optarg, opt); + cfg->mpool_buf_num = v; } else if (opt == OPT_SHORT_PROMISC) { cfg->promisc = 1; } else if (opt == OPT_SHORT_RBUFS) { diff --git a/examples/l4fwd/parse.h b/examples/l4fwd/parse.h index 4303623..4634d60 100644 --- a/examples/l4fwd/parse.h +++ b/examples/l4fwd/parse.h @@ -29,7 +29,7 @@ union parse_val { struct in6_addr addr6; }; } in; - struct ether_addr mac; + struct rte_ether_addr mac; rte_cpuset_t cpuset; }; diff --git a/examples/l4fwd/pkt.c b/examples/l4fwd/pkt.c index 43aa9c8..6694e81 100644 --- a/examples/l4fwd/pkt.c +++ b/examples/l4fwd/pkt.c @@ -49,9 +49,9 @@ fill_pkt_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t l3, uint32_t l4) } static inline int -is_ipv4_frag(const struct ipv4_hdr *iph) +is_ipv4_frag(const struct rte_ipv4_hdr *iph) { - const uint16_t mask = rte_cpu_to_be_16(~IPV4_HDR_DF_FLAG); + const uint16_t mask = rte_cpu_to_be_16(~RTE_IPV4_HDR_DF_FLAG); return ((mask & iph->fragment_offset) != 0); } @@ -59,9 +59,9 @@ is_ipv4_frag(const struct ipv4_hdr *iph) static inline uint32_t get_tcp_header_size(struct rte_mbuf *m, uint32_t l2_len, uint32_t l3_len) { - const struct tcp_hdr *tcp; + const struct rte_tcp_hdr *tcp; - tcp = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, l2_len + l3_len); + tcp = rte_pktmbuf_mtod_offset(m, struct rte_tcp_hdr *, l2_len + l3_len); return (tcp->data_off >> 4) * 4; } @@ -69,9 +69,9 @@ static inline void adjust_ipv4_pktlen(struct rte_mbuf *m, uint32_t l2_len) { uint32_t plen, trim; - const struct ipv4_hdr *iph; + const struct rte_ipv4_hdr *iph; - iph = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, l2_len); + iph = rte_pktmbuf_mtod_offset(m, const struct rte_ipv4_hdr *, l2_len); plen = rte_be_to_cpu_16(iph->total_length) + l2_len; if (plen < m->pkt_len) { trim = m->pkt_len - plen; @@ -83,9 +83,9 @@ static inline void adjust_ipv6_pktlen(struct rte_mbuf *m, uint32_t l2_len) { uint32_t plen, trim; - const struct ipv6_hdr *iph; + const struct rte_ipv6_hdr *iph; - iph = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *, l2_len); + iph = rte_pktmbuf_mtod_offset(m, const struct rte_ipv6_hdr *, l2_len); plen = rte_be_to_cpu_16(iph->payload_len) + sizeof(*iph) + l2_len; if (plen < m->pkt_len) { trim = m->pkt_len - plen; @@ -97,23 +97,24 @@ static inline void tcp_stat_update(struct netbe_lcore *lc, const struct rte_mbuf *m, uint32_t l2_len, uint32_t l3_len) { - const struct tcp_hdr *th; + const struct rte_tcp_hdr *th; - th = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, l2_len + l3_len); + th = rte_pktmbuf_mtod_offset(m, struct rte_tcp_hdr *, l2_len + l3_len); lc->tcp_stat.flags[th->tcp_flags]++; } static inline uint32_t get_ipv4_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t proto, uint32_t frag) { - const struct ipv4_hdr *iph; + const struct rte_ipv4_hdr *iph; int32_t dlen, len; dlen = rte_pktmbuf_data_len(m); dlen -= l2; - iph = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, l2); - len = (iph->version_ihl & IPV4_HDR_IHL_MASK) * IPV4_IHL_MULTIPLIER; + iph = rte_pktmbuf_mtod_offset(m, const struct rte_ipv4_hdr *, l2); + len = (iph->version_ihl & RTE_IPV4_HDR_IHL_MASK) * + RTE_IPV4_IHL_MULTIPLIER; if (frag != 0 && is_ipv4_frag(iph)) { m->packet_type &= ~RTE_PTYPE_L4_MASK; @@ -155,7 +156,7 @@ get_ipv6x_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t nproto, const struct ip6_ext *ipx; int32_t dlen, len, ofs; - len = sizeof(struct ipv6_hdr); + len = sizeof(struct rte_ipv6_hdr); dlen = rte_pktmbuf_data_len(m); dlen -= l2; @@ -205,13 +206,13 @@ get_ipv6x_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t nproto, static inline uint32_t get_ipv6_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t fproto) { - const struct ipv6_hdr *iph; + const struct rte_ipv6_hdr *iph; - iph = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *, - sizeof(struct ether_hdr)); + iph = rte_pktmbuf_mtod_offset(m, const struct rte_ipv6_hdr *, + sizeof(struct rte_ether_hdr)); if (iph->proto == fproto) - return sizeof(struct ipv6_hdr); + return sizeof(struct rte_ipv6_hdr); else if (ipv6x_hdr(iph->proto) != 0) return get_ipv6x_hdr_len(m, l2, iph->proto, fproto); @@ -234,14 +235,14 @@ static inline struct rte_mbuf * handle_arp(struct rte_mbuf *m, struct netbe_lcore *lc, dpdk_port_t port, uint32_t l2len) { - const struct arp_hdr *ahdr; + const struct rte_arp_hdr *ahdr; struct pkt_buf *abuf; - ahdr = rte_pktmbuf_mtod_offset(m, const struct arp_hdr *, l2len); + ahdr = rte_pktmbuf_mtod_offset(m, const struct rte_arp_hdr *, l2len); - if (ahdr->arp_hrd != rte_be_to_cpu_16(ARP_HRD_ETHER) || - ahdr->arp_pro != rte_be_to_cpu_16(ETHER_TYPE_IPv4) || - ahdr->arp_op != rte_be_to_cpu_16(ARP_OP_REQUEST)) { + if (ahdr->arp_hardware != rte_be_to_cpu_16(RTE_ARP_HRD_ETHER) || + ahdr->arp_protocol != rte_be_to_cpu_16(RTE_ETHER_TYPE_IPV4) || + ahdr->arp_opcode != rte_be_to_cpu_16(RTE_ARP_OP_REQUEST)) { m->packet_type = RTE_PTYPE_UNKNOWN; return m; @@ -263,28 +264,28 @@ fill_eth_tcp_arp_hdr_len(struct rte_mbuf *m, struct netbe_lcore *lc, { uint32_t dlen, l2_len, l3_len, l4_len; uint16_t etp; - const struct ether_hdr *eth; + const struct rte_ether_hdr *eth; dlen = rte_pktmbuf_data_len(m); /* check that first segment is at least 54B long. */ - if (dlen < sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr) + - sizeof(struct tcp_hdr)) { + if (dlen < sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + + sizeof(struct rte_tcp_hdr)) { m->packet_type = RTE_PTYPE_UNKNOWN; return m; } l2_len = sizeof(*eth); - eth = rte_pktmbuf_mtod(m, const struct ether_hdr *); + eth = rte_pktmbuf_mtod(m, const struct rte_ether_hdr *); etp = eth->ether_type; - if (etp == rte_be_to_cpu_16(ETHER_TYPE_VLAN)) - l2_len += sizeof(struct vlan_hdr); + if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_VLAN)) + l2_len += sizeof(struct rte_vlan_hdr); - if (etp == rte_be_to_cpu_16(ETHER_TYPE_ARP)) + if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_ARP)) return handle_arp(m, lc, port, l2_len); - if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv4)) { + if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_IPV4)) { m->packet_type = RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER; @@ -292,9 +293,9 @@ fill_eth_tcp_arp_hdr_len(struct rte_mbuf *m, struct netbe_lcore *lc, l4_len = get_tcp_header_size(m, l2_len, l3_len); fill_pkt_hdr_len(m, l2_len, l3_len, l4_len); adjust_ipv4_pktlen(m, l2_len); - } else if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv6) && - dlen >= l2_len + sizeof(struct ipv6_hdr) + - sizeof(struct tcp_hdr)) { + } else if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_IPV6) && + dlen >= l2_len + sizeof(struct rte_ipv6_hdr) + + sizeof(struct rte_tcp_hdr)) { m->packet_type = RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER; @@ -313,25 +314,25 @@ fill_eth_tcp_hdr_len(struct rte_mbuf *m) { uint32_t dlen, l2_len, l3_len, l4_len; uint16_t etp; - const struct ether_hdr *eth; + const struct rte_ether_hdr *eth; dlen = rte_pktmbuf_data_len(m); /* check that first segment is at least 54B long. */ - if (dlen < sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr) + - sizeof(struct tcp_hdr)) { + if (dlen < sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + + sizeof(struct rte_tcp_hdr)) { m->packet_type = RTE_PTYPE_UNKNOWN; return; } l2_len = sizeof(*eth); - eth = rte_pktmbuf_mtod(m, const struct ether_hdr *); + eth = rte_pktmbuf_mtod(m, const struct rte_ether_hdr *); etp = eth->ether_type; - if (etp == rte_be_to_cpu_16(ETHER_TYPE_VLAN)) - l2_len += sizeof(struct vlan_hdr); + if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_VLAN)) + l2_len += sizeof(struct rte_vlan_hdr); - if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv4)) { + if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_IPV4)) { m->packet_type = RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER; @@ -339,9 +340,9 @@ fill_eth_tcp_hdr_len(struct rte_mbuf *m) l4_len = get_tcp_header_size(m, l2_len, l3_len); fill_pkt_hdr_len(m, l2_len, l3_len, l4_len); adjust_ipv4_pktlen(m, l2_len); - } else if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv6) && - dlen >= l2_len + sizeof(struct ipv6_hdr) + - sizeof(struct tcp_hdr)) { + } else if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_IPV6) && + dlen >= l2_len + sizeof(struct rte_ipv6_hdr) + + sizeof(struct rte_tcp_hdr)) { m->packet_type = RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER; @@ -358,38 +359,38 @@ fill_eth_udp_hdr_len(struct rte_mbuf *m) { uint32_t dlen, l2_len; uint16_t etp; - const struct ether_hdr *eth; + const struct rte_ether_hdr *eth; dlen = rte_pktmbuf_data_len(m); /* check that first segment is at least 42B long. */ - if (dlen < sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr) + - sizeof(struct udp_hdr)) { + if (dlen < sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + + sizeof(struct rte_udp_hdr)) { m->packet_type = RTE_PTYPE_UNKNOWN; return; } l2_len = sizeof(*eth); - eth = rte_pktmbuf_mtod(m, const struct ether_hdr *); + eth = rte_pktmbuf_mtod(m, const struct rte_ether_hdr *); etp = eth->ether_type; - if (etp == rte_be_to_cpu_16(ETHER_TYPE_VLAN)) - l2_len += sizeof(struct vlan_hdr); + if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_VLAN)) + l2_len += sizeof(struct rte_vlan_hdr); - if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv4)) { + if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_IPV4)) { m->packet_type = RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER; fill_ipv4_hdr_len(m, l2_len, IPPROTO_UDP, 1, - sizeof(struct udp_hdr)); - } else if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv6) && - dlen >= l2_len + sizeof(struct ipv6_hdr) + - sizeof(struct udp_hdr)) { + sizeof(struct rte_udp_hdr)); + } else if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_IPV6) && + dlen >= l2_len + sizeof(struct rte_ipv6_hdr) + + sizeof(struct rte_udp_hdr)) { m->packet_type = RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER; fill_ipv6_hdr_len(m, l2_len, IPPROTO_UDP, - sizeof(struct udp_hdr)); + sizeof(struct rte_udp_hdr)); } else m->packet_type = RTE_PTYPE_UNKNOWN; } @@ -406,7 +407,7 @@ ipv4x_cksum(const void *iph, size_t len) static inline void fix_reassembled(struct rte_mbuf *m, int32_t hwcsum, uint32_t proto) { - struct ipv4_hdr *iph; + struct rte_ipv4_hdr *iph; /* update packet type. */ m->packet_type &= ~RTE_PTYPE_L4_MASK; @@ -425,7 +426,8 @@ fix_reassembled(struct rte_mbuf *m, int32_t hwcsum, uint32_t proto) /* recalculate ipv4 cksum after reassemble. */ else if (hwcsum == 0 && RTE_ETH_IS_IPV4_HDR(m->packet_type)) { - iph = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, m->l2_len); + iph = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, + m->l2_len); iph->hdr_checksum = ipv4x_cksum(iph, m->l3_len); } } @@ -444,19 +446,21 @@ reassemble(struct rte_mbuf *m, struct netbe_lcore *lc, uint64_t tms, if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) { - struct ipv4_hdr *iph; + struct rte_ipv4_hdr *iph; - iph = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, m->l2_len); + iph = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, + m->l2_len); /* process this fragment. */ m = rte_ipv4_frag_reassemble_packet(tbl, dr, m, tms, iph); } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) { - struct ipv6_hdr *iph; + struct rte_ipv6_hdr *iph; struct ipv6_extension_fragment *fhdr; - iph = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *, m->l2_len); + iph = rte_pktmbuf_mtod_offset(m, struct rte_ipv6_hdr *, + m->l2_len); /* * we store fragment header offset in tso_segsz before @@ -535,7 +539,7 @@ type0_tcp_rx_callback(__rte_unused dpdk_port_t port, uint32_t j, tp; struct netbe_lcore *lc; uint32_t l4_len, l3_len, l2_len; - const struct ether_hdr *eth; + const struct rte_ether_hdr *eth; lc = user_param; l2_len = sizeof(*eth); @@ -554,17 +558,17 @@ type0_tcp_rx_callback(__rte_unused dpdk_port_t port, case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L2_ETHER): l4_len = get_tcp_header_size(pkt[j], l2_len, - sizeof(struct ipv4_hdr)); + sizeof(struct rte_ipv4_hdr)); fill_pkt_hdr_len(pkt[j], l2_len, - sizeof(struct ipv4_hdr), l4_len); + sizeof(struct rte_ipv4_hdr), l4_len); adjust_ipv4_pktlen(pkt[j], l2_len); break; case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L2_ETHER): l4_len = get_tcp_header_size(pkt[j], l2_len, - sizeof(struct ipv6_hdr)); + sizeof(struct rte_ipv6_hdr)); fill_pkt_hdr_len(pkt[j], l2_len, - sizeof(struct ipv6_hdr), l4_len); + sizeof(struct rte_ipv6_hdr), l4_len); adjust_ipv6_pktlen(pkt[j], l2_len); break; case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4_EXT | @@ -604,7 +608,7 @@ type0_udp_rx_callback(dpdk_port_t port, __rte_unused uint16_t queue, uint64_t cts; struct netbe_lcore *lc; uint32_t l2_len; - const struct ether_hdr *eth; + const struct rte_ether_hdr *eth; lc = user_param; cts = 0; @@ -623,37 +627,37 @@ type0_udp_rx_callback(dpdk_port_t port, __rte_unused uint16_t queue, case (RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L2_ETHER): fill_pkt_hdr_len(pkt[j], l2_len, - sizeof(struct ipv4_hdr), - sizeof(struct udp_hdr)); + sizeof(struct rte_ipv4_hdr), + sizeof(struct rte_udp_hdr)); adjust_ipv4_pktlen(pkt[j], l2_len); break; case (RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L2_ETHER): fill_pkt_hdr_len(pkt[j], l2_len, - sizeof(struct ipv6_hdr), - sizeof(struct udp_hdr)); + sizeof(struct rte_ipv6_hdr), + sizeof(struct rte_udp_hdr)); adjust_ipv6_pktlen(pkt[j], l2_len); break; case (RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L2_ETHER): fill_ipv4_hdr_len(pkt[j], l2_len, - UINT32_MAX, 0, sizeof(struct udp_hdr)); + UINT32_MAX, 0, sizeof(struct rte_udp_hdr)); break; case (RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L2_ETHER): fill_ipv6_hdr_len(pkt[j], l2_len, - IPPROTO_UDP, sizeof(struct udp_hdr)); + IPPROTO_UDP, sizeof(struct rte_udp_hdr)); break; /* possibly fragmented udp packets. */ case (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L2_ETHER): case (RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L2_ETHER): fill_ipv4_hdr_len(pkt[j], l2_len, - IPPROTO_UDP, 1, sizeof(struct udp_hdr)); + IPPROTO_UDP, 1, sizeof(struct rte_udp_hdr)); break; case (RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L2_ETHER): case (RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L2_ETHER): fill_ipv6_hdr_len(pkt[j], l2_len, - IPPROTO_UDP, sizeof(struct udp_hdr)); + IPPROTO_UDP, sizeof(struct rte_udp_hdr)); break; default: /* treat packet types as invalid. */ @@ -690,7 +694,7 @@ type1_tcp_rx_callback(__rte_unused dpdk_port_t port, uint32_t j, tp; struct netbe_lcore *lc; uint32_t l4_len, l3_len, l2_len; - const struct ether_hdr *eth; + const struct rte_ether_hdr *eth; lc = user_param; l2_len = sizeof(*eth); @@ -745,7 +749,7 @@ type1_udp_rx_callback(dpdk_port_t port, __rte_unused uint16_t queue, uint64_t cts; struct netbe_lcore *lc; uint32_t l2_len; - const struct ether_hdr *eth; + const struct rte_ether_hdr *eth; lc = user_param; cts = 0; @@ -763,22 +767,22 @@ type1_udp_rx_callback(dpdk_port_t port, __rte_unused uint16_t queue, case (RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER): fill_ipv4_hdr_len(pkt[j], l2_len, - UINT32_MAX, 0, sizeof(struct udp_hdr)); + UINT32_MAX, 0, sizeof(struct rte_udp_hdr)); break; case (RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER): fill_ipv6_hdr_len(pkt[j], l2_len, - IPPROTO_UDP, sizeof(struct udp_hdr)); + IPPROTO_UDP, sizeof(struct rte_udp_hdr)); break; case (RTE_PTYPE_L4_FRAG | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER): fill_ipv4_hdr_len(pkt[j], l2_len, - IPPROTO_UDP, 0, sizeof(struct udp_hdr)); + IPPROTO_UDP, 0, sizeof(struct rte_udp_hdr)); break; case (RTE_PTYPE_L4_FRAG | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER): fill_ipv6_hdr_len(pkt[j], l2_len, - IPPROTO_UDP, sizeof(struct udp_hdr)); + IPPROTO_UDP, sizeof(struct rte_udp_hdr)); break; default: /* treat packet types as invalid. */ diff --git a/examples/l4fwd/port.h b/examples/l4fwd/port.h index a154844..ce730dd 100644 --- a/examples/l4fwd/port.h +++ b/examples/l4fwd/port.h @@ -182,8 +182,8 @@ port_init(struct netbe_port *uprt, uint32_t proto) __func__, uprt->id); port_conf.rxmode.offloads |= uprt->rx_offload & RX_CSUM_OFFLOAD; } - port_conf.rxmode.max_rx_pkt_len = uprt->mtu + ETHER_CRC_LEN; - if (port_conf.rxmode.max_rx_pkt_len > ETHER_MAX_LEN) + port_conf.rxmode.max_rx_pkt_len = uprt->mtu + RTE_ETHER_CRC_LEN; + if (port_conf.rxmode.max_rx_pkt_len > RTE_ETHER_MAX_LEN) port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; rc = update_rss_conf(uprt, &dev_info, &port_conf, proto); @@ -258,8 +258,7 @@ check_lcore(uint32_t lc) return -EINVAL; } if (rte_eal_get_lcore_state(lc) == RUNNING) { - RTE_LOG(ERR, USER1, "lcore %u already running %p\n", - lc, lcore_config[lc].f); + RTE_LOG(ERR, USER1, "lcore %u already in use\n", lc); return -EINVAL; } return 0; @@ -316,14 +315,14 @@ log_netbe_cfg(const struct netbe_cfg *ucfg) } static int -pool_init(uint32_t sid) +pool_init(uint32_t sid, uint32_t mpool_buf_num) { int32_t rc; struct rte_mempool *mp; char name[RTE_MEMPOOL_NAMESIZE]; snprintf(name, sizeof(name), "MP%u", sid); - mp = rte_pktmbuf_pool_create(name, MPOOL_NB_BUF, MPOOL_CACHE_SIZE, 0, + mp = rte_pktmbuf_pool_create(name, mpool_buf_num, MPOOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, sid - 1); if (mp == NULL) { rc = -rte_errno; @@ -337,14 +336,14 @@ pool_init(uint32_t sid) } static int -frag_pool_init(uint32_t sid) +frag_pool_init(uint32_t sid, uint32_t mpool_buf_num) { int32_t rc; struct rte_mempool *frag_mp; char frag_name[RTE_MEMPOOL_NAMESIZE]; snprintf(frag_name, sizeof(frag_name), "frag_MP%u", sid); - frag_mp = rte_pktmbuf_pool_create(frag_name, MPOOL_NB_BUF, + frag_mp = rte_pktmbuf_pool_create(frag_name, mpool_buf_num, MPOOL_CACHE_SIZE, 0, FRAG_MBUF_BUF_SIZE, sid - 1); if (frag_mp == NULL) { rc = -rte_errno; @@ -406,13 +405,13 @@ netbe_port_init(struct netbe_cfg *cfg) assert(sid < RTE_DIM(mpool)); if (mpool[sid] == NULL) { - rc = pool_init(sid); + rc = pool_init(sid, cfg->mpool_buf_num); if (rc != 0) return rc; } if (frag_mpool[sid] == NULL) { - rc = frag_pool_init(sid); + rc = frag_pool_init(sid, cfg->mpool_buf_num); if (rc != 0) return rc; } diff --git a/examples/l4fwd/test/config.sh b/examples/l4fwd/test/config.sh index fef8fda..ca62e3d 100644 --- a/examples/l4fwd/test/config.sh +++ b/examples/l4fwd/test/config.sh @@ -9,17 +9,28 @@ DPDK_PORT=0 TCP_PORT=6000 # local interface addresses to set -LOCAL_IPV4=192.168.1.60 -LOCAL_IPV6=fd12:3456:789a:0001:0000:0000:0000:0060 +L4FWD_IPV4=192.168.2.60 +L4FWD_IPV6=fd12:3456:789a:0002:0000:0000:0000:0060 # remote interface addresses to set -REMOTE_IPV4=192.168.1.64 -REMOTE_IPV6=fd12:3456:789a:0001:0000:0000:0000:0064 +LINUX_IPV4=192.168.2.64 +LINUX_IPV6=fd12:3456:789a:0002:0000:0000:0000:0064 # mask length for addresses of each IP version MASK_IPV4=24 MASK_IPV6=64 +# Interface tap/remote +IFACE="" + +# should tap mode be used (1 - use tap interface, 0 - use real NIC) +USE_TAP=0 + +# MAC address for tap interface - filled when tap is created +LINUX_MAC="00:64:74:61:70:30" +# fake MAC address to provide in neighbours +FAKE_MAC="00:64:74:61:70:33" + # name of the config files for backend and frontend of l4fwd app L4FWD_BE_CFG_FILE=$(mktemp) L4FWD_FE_CFG_FILE=$(mktemp) @@ -40,6 +51,15 @@ then exit 127 fi +# set interface based on mode used +if [[ "${ETH_DEV}" == "tap" ]] +then + IFACE=l4fwd_tap0 + USE_TAP=1 +else + IFACE=${REMOTE_IFACE} +fi + # check if L4FWD_PATH points to an executable if [[ ! -x ${L4FWD_PATH} ]] then @@ -47,23 +67,27 @@ then exit 127 fi -# check if REMOTE_HOST is reachable -ssh ${REMOTE_HOST} echo -st=$? -if [[ $st -ne 0 ]] +# neccesary check for real NIC mode +if [[ ${USE_TAP} -eq 0 ]] then - echo "host ${REMOTE_HOST} is not reachable" - exit $st -fi + # check if REMOTE_HOST is reachable + ssh ${REMOTE_HOST} echo + st=$? + if [[ $st -ne 0 ]] + then + echo "host ${REMOTE_HOST} is not reachable" + exit $st + fi -# get ethernet address of REMOTE_HOST -REMOTE_MAC=$(ssh ${REMOTE_HOST} ip addr show dev ${REMOTE_IFACE}) -st=$? -REMOTE_MAC=$(echo ${REMOTE_MAC} | sed -e 's/^.*ether //' -e 's/ brd.*$//') -if [[ $st -ne 0 || -z "${REMOTE_MAC}" ]] -then - echo "could not retrive ethernet address from ${REMOTE_IFACE}" - exit 127 + # get ethernet address of REMOTE_HOST + LINUX_MAC=$(ssh ${REMOTE_HOST} ip addr show dev ${IFACE}) + st=$? + LINUX_MAC=$(echo ${LINUX_MAC} | sed -e 's/^.*ether //' -e 's/ brd.*$//') + if [[ $st -ne 0 || -z "${LINUX_MAC}" ]] + then + echo "could not retrive ethernet address from ${IFACE}" + exit 127 + fi fi # check if FECORE is set - default 0 @@ -77,7 +101,7 @@ L4FWD_BECORE=${L4FWD_BECORE:-${L4FWD_FECORE}} # set file for l4fwd app output L4FWD_OUT_FILE=./l4fwd.out # set rbufs/sbufs/streams to open for l4fwd -L4FWD_STREAMS='--rbufs 0x100 --sbufs 0x100 --streams 0x100' +L4FWD_STREAMS='--mbuf-num 0x2000 --rbufs 0x100 --sbufs 0x100 --streams 0x100' # set lcores for DPDK to start if [[ ${L4FWD_FECORE} -ne ${L4FWD_BECORE} ]] @@ -87,8 +111,22 @@ else L4FWD_LCORE="${L4FWD_FECORE}" fi +L4FWD_TAP="" + +# set eal parameters specific for mode used +if [[ ${USE_TAP} -eq 0 ]] +then + L4FWD_DEV="${ETH_DEV}" +else + L4FWD_DEV="--no-pci --vdev=\"net_tap0,iface=${IFACE},\ +mac=\"${LINUX_MAC}\"\"" +fi + # set EAL parameters -L4FWD_CMD_EAL_PRM="--lcores='${L4FWD_LCORE}' -n 4 ${ETH_DEV}" +L4FWD_CMD_EAL_PRM="--lcores='${L4FWD_LCORE}' -n 4 ${L4FWD_DEV}" + +# interface to wait for until it is set up properly +L4FWD_WAIT_VDEV="${IFACE}" # l4fwd parameters (listen, TCP only, enable arp, promiscuous) L4FWD_CMD_PRM="--listen --tcp --enable-arp --promisc ${L4FWD_STREAMS}" @@ -100,21 +138,31 @@ L4FWD_CONFIG="--fecfg ${L4FWD_FE_CFG_FILE} --becfg ${L4FWD_BE_CFG_FILE}" if [[ ${ipv4} -eq 1 ]] then L4FWD_PORT_PRM="port=${DPDK_PORT},lcore=${L4FWD_BECORE},rx_offload=0x0\ -,tx_offload=0x0,ipv4=${LOCAL_IPV4}" +,tx_offload=0x0,ipv4=${L4FWD_IPV4}" elif [[ ${ipv6} -eq 1 ]] then L4FWD_PORT_PRM="port=${DPDK_PORT},lcore=${L4FWD_BECORE},rx_offload=0x0\ -,tx_offload=0x0,ipv6=${LOCAL_IPV6}" +,tx_offload=0x0,ipv6=${L4FWD_IPV6}" fi # other variables-------------------------------------------------------------- +# function to run command with ssh <remote> if needed +use_ssh() +{ + if [[ ${USE_TAP} -eq 1 ]] + then + "$@" + else + ssh ${REMOTE_HOST} "$*" + fi +} + # check if directories on remote are set, if not make one -ssh ${REMOTE_HOST} mkdir -p {${REMOTE_OUTDIR},${REMOTE_RESDIR}} +use_ssh mkdir -p {${REMOTE_OUTDIR},${REMOTE_RESDIR}} # <tc qdisc ... netem ...> instruction to set -netem="ssh ${REMOTE_HOST} tc qdisc add dev ${REMOTE_IFACE} \ -root netem limit 100000" +netem="tc qdisc add dev ${IFACE} root netem limit 100000" # setting for scp which suppresses output of scp when not in verbose mode if [[ ${verbose} -eq 1 ]] @@ -135,12 +183,13 @@ fi # set address to use by netcat if [[ ${ipv4} -eq 1 ]] then - nc_addr=${LOCAL_IPV4} + nc_addr=${L4FWD_IPV4} elif [[ ${ipv6} -eq 1 ]] then - nc_addr=${LOCAL_IPV6} + nc_addr=${L4FWD_IPV6} fi +# calculate network address let "ipv4_elem=(${MASK_IPV4}/8)" let "ipv6_elem=(${MASK_IPV6}/16)" let "ipv4_elem_rev=4-${ipv4_elem}" @@ -151,9 +200,18 @@ while [[ ${ipv4_elem_rev} -ne 0 ]]; do let "ipv4_elem_rev=${ipv4_elem_rev}-1" done -ipv4_network=$(echo ${REMOTE_IPV4} | cut -d. -f-${ipv4_elem} | \ +ipv4_network=$(echo ${LINUX_IPV4} | cut -d. -f-${ipv4_elem} | \ sed 's#.*#&'"${ipv4_append}"'#') -ipv6_network=$(echo ${REMOTE_IPV6} | cut -d: -f-${ipv6_elem} | sed 's#.*#&::#') +ipv6_network=$(echo ${LINUX_IPV6} | cut -d: -f-${ipv6_elem} | sed 's#.*#&::#') + +# create temporary result file for tap mode, and/or set common file name +if [[ ${USE_TAP} -eq 0 ]] +then + common_result_file="${REMOTE_RESDIR}/results.out" +else + > ${local_result_file} + common_result_file=${local_result_file} +fi # helper functions------------------------------------------------------------- @@ -174,18 +232,33 @@ update_results() it=$3 # get only 'real' time in results file - $(ssh ${REMOTE_HOST} "awk '/real/{print \$2}' \ - ${REMOTE_RESDIR}/${file}.result.${it} \ - >> ${REMOTE_RESDIR}/results.out") + if [[ ${USE_TAP} -eq 0 ]] + then + $(ssh ${REMOTE_HOST} "awk '/real/{print \$2}' \ +${REMOTE_RESDIR}/${file}.result.${it} >> ${common_result_file}") + else + awk '/real/{print $2}' ${REMOTE_RESDIR}/${file}.result.${it} \ + >> ${common_result_file} + fi # add file and status of test to results if [[ ${status} -ne 0 ]] then - $(ssh ${REMOTE_HOST} "sed -i '$ s_.*_[FAIL]\t&_' \ - ${REMOTE_RESDIR}/results.out") + if [[ ${USE_TAP} -eq 0 ]] + then + $(ssh ${REMOTE_HOST} "sed -i '$ s_.*_[FAIL]\t&_' \ +${common_result_file}") + else + sed -i '$ s_.*_[FAIL]\t&_' ${common_result_file} + fi else - $(ssh ${REMOTE_HOST} "sed -i '$ s_.*_[OK]\t&_' \ - ${REMOTE_RESDIR}/results.out") + if [[ ${USE_TAP} -eq 0 ]] + then + $(ssh ${REMOTE_HOST} "sed -i '$ s_.*_[OK]\t&_' \ +${common_result_file}") + else + sed -i '$ s_.*_[OK]\t&_' ${common_result_file} + fi fi length=$(expr length "${file}") @@ -196,13 +269,21 @@ update_results() tab="\t" fi - $(ssh ${REMOTE_HOST} "sed -i '$ s_.*_${file}${tab}&_' \ - ${REMOTE_RESDIR}/results.out") + if [[ ${USE_TAP} -eq 0 ]] + then + $(ssh ${REMOTE_HOST} "sed -i '$ s_.*_${file}${tab}&_' \ +${common_result_file}") + else + sed -i "$ s_.*_${file}${tab}&_" ${common_result_file} + fi } # start l4fwd app l4fwd_start() { + # make configuration files for be/fe + configure_be_fe + # create temporary file for command running l4fwd L4FWD_EXEC_FILE=$(mktemp) @@ -229,6 +310,20 @@ EOF rm -f ${L4FWD_EXEC_FILE} exit 127 fi + + if [[ ${USE_TAP} -eq 1 ]] + then + # check if tap interface is up + i=0 + st=1 + while [[ ${i} -ne 5 && ${st} -ne 0 ]] + do + sleep 1 + ip link show dev ${L4FWD_WAIT_VDEV} > /dev/null 2>&1 + st=$? + let i++ + done + fi } # stop l4fwd app @@ -236,7 +331,7 @@ l4fwd_stop() { # kill runnning l4fwd app kill -s SIGINT ${L4FWD_PID} - + sleep 1 # remove temporary files rm -f ${L4FWD_EXEC_FILE} rm -f ${L4FWD_FE_CFG_FILE} @@ -246,13 +341,8 @@ l4fwd_stop() # helper function to set netem on remote setup_netem() { - # remove netem settings from remote interface if any - check_netem=$(ssh ${REMOTE_HOST} "tc qdisc show dev \ - ${REMOTE_IFACE} | grep netem") - if [[ -n ${check_netem} ]] - then - ssh ${REMOTE_HOST} tc qdisc del dev ${REMOTE_IFACE} root - fi + # remove netem settings from interface + use_ssh tc qdisc del dev ${IFACE} root # set default delay for reorder if [[ ${reorder} -ne 0 && ${delay} -eq 0 ]] @@ -284,35 +374,42 @@ setup_netem() netem="${netem} reorder 100% gap ${reorder}" fi - # set netem on remote - ${netem} + # set netem + use_ssh ${netem} # visual break of the output if_verbose echo -e "\nNetwork rules on remote set to:" # print current netem settings - if_verbose ssh ${REMOTE_HOST} tc qdisc show dev ${REMOTE_IFACE} + if_verbose use_ssh tc qdisc show dev ${IFACE} } -# configure IPv4 remote machine -configure_ip4_remote() +# configure IPv4 interface +configure_l4fwd_ip4() { # visual break of the output - if_verbose echo "Setting interface on remote" + if_verbose echo "Setting IPv4 interface" # set remote interface with correct IP address - ssh ${REMOTE_HOST} ip link set ${REMOTE_IFACE} down - ssh ${REMOTE_HOST} ip addr flush dev ${REMOTE_IFACE} - ssh ${REMOTE_HOST} ip addr add ${REMOTE_IPV4}/${MASK_IPV4} \ - dev ${REMOTE_IFACE} - ssh ${REMOTE_HOST} ip link set ${REMOTE_IFACE} up - if_verbose ssh ${REMOTE_HOST} ip addr show dev ${REMOTE_IFACE} - - ssh ${REMOTE_HOST} ip neigh flush dev ${REMOTE_IFACE} - ssh ${REMOTE_HOST} iptables --flush + if [[ ${USE_TAP} -eq 0 ]] + then + ssh ${REMOTE_HOST} ip link set ${IFACE} down + ssh ${REMOTE_HOST} ip addr flush dev ${IFACE} + ssh ${REMOTE_HOST} ip addr add ${LINUX_IPV4}/${MASK_IPV4} \ +dev ${IFACE} + ssh ${REMOTE_HOST} ip link set ${IFACE} up + ssh ${REMOTE_HOST} ip neigh flush dev ${IFACE} + else + ip addr add ${LINUX_IPV4}/${MASK_IPV4} dev ${IFACE} + ip link set ${IFACE} up + ip neigh flush dev ${IFACE} + ip neigh add ${L4FWD_IPV4} dev ${IFACE} lladdr ${FAKE_MAC} + fi - ssh ${REMOTE_HOST} ip route change ${ipv4_network}/${MASK_IPV4} dev \ - ${REMOTE_IFACE} rto_min 30ms + use_ssh iptables --flush + use_ssh ip route change ${ipv4_network}/${MASK_IPV4} dev ${IFACE} \ +rto_min 30ms + if_verbose use_ssh ip addr show dev ${IFACE} # construct <tc qdisc ... nete ...> instruction if [[ set_netem -eq 1 ]] @@ -324,32 +421,40 @@ configure_ip4_remote() sleep 1 } -# configure IPv6 remote machine -configure_ip6_remote() +# configure IPv6 interface +configure_l4fwd_ip6() { # visual break of the output - if_verbose echo "Setting interface on remote" + if_verbose echo "Setting IPv6 interface" # set remote interface with correct IP address - ssh ${REMOTE_HOST} ip link set ${REMOTE_IFACE} down - ssh ${REMOTE_HOST} sysctl -q -w \ - net.ipv6.conf.${REMOTE_IFACE}.disable_ipv6=0 - ssh ${REMOTE_HOST} ip addr flush dev ${REMOTE_IFACE} - ssh ${REMOTE_HOST} ip -6 addr add ${REMOTE_IPV6}/${MASK_IPV6} \ - dev ${REMOTE_IFACE} - ssh ${REMOTE_HOST} ip -6 link set ${REMOTE_IFACE} up - if_verbose ssh ${REMOTE_HOST} ip addr show dev ${REMOTE_IFACE} - - ssh ${REMOTE_HOST} ip neigh flush dev ${REMOTE_IFACE} - ssh ${REMOTE_HOST} ip -6 neigh add ${LOCAL_IPV6} dev ${REMOTE_IFACE} \ - lladdr ${LOCAL_MAC} - ssh ${REMOTE_HOST} iptables --flush - ssh ${REMOTE_HOST} ip6tables --flush - - ssh ${REMOTE_HOST} ip route change ${ipv6_network}/${MASK_IPV6} dev \ - ${REMOTE_IFACE} proto kernel metric 256 rto_min 30ms - - ssh ${REMOTE_HOST} ip -6 route show + if [[ ${USE_TAP} -eq 0 ]] + then + ssh ${REMOTE_HOST} ip link set ${IFACE} down + ssh ${REMOTE_HOST} sysctl -q -w \ +net.ipv6.conf.${IFACE}.disable_ipv6=0 + ssh ${REMOTE_HOST} ip addr flush dev ${IFACE} + ssh ${REMOTE_HOST} ip -6 addr add ${LINUX_IPV6}/${MASK_IPV6} \ +dev ${IFACE} + ssh ${REMOTE_HOST} ip -6 link set ${IFACE} up + ssh ${REMOTE_HOST} ip neigh flush dev ${IFACE} + ssh ${REMOTE_HOST} ip -6 neigh add ${L4FWD_IPV6} dev ${IFACE} \ +lladdr ${LOCAL_MAC} + else + sysctl -q -w net.ipv6.conf.${IFACE}.disable_ipv6=0 + ip addr flush dev ${IFACE} + ip -6 addr add ${LINUX_IPV6}/${MASK_IPV6} dev ${IFACE} + ip -6 link set ${IFACE} up + ip neigh flush dev ${IFACE} + ip -6 neigh add ${L4FWD_IPV6} dev ${IFACE} lladdr ${FAKE_MAC} + fi + + use_ssh iptables --flush + use_ssh ip6tables --flush + + use_ssh ip route change ${ipv6_network}/${MASK_IPV6} dev \ +${IFACE} proto kernel metric 256 rto_min 30ms + if_verbose use_ssh ip addr show dev ${IFACE} # construct <tc qdisc ... nete ...> instruction if [[ set_netem -eq 1 ]] @@ -357,36 +462,28 @@ configure_ip6_remote() setup_netem fi - # give linux 1 sec to handle all network settings - sleep 1 + # give linux 3 sec to handle all network settings + sleep 3 } -# configure remote -configure_remote() + +# configure tap interfaces +configure_interfaces() { # call proper configuration if [[ ${ipv4} -eq 1 ]] then - configure_ip4_remote - - if_verbose echo -e "\nBE configuration:" - config4_be - - if_verbose echo -e "\nFE configuration:" - config4_fe + configure_l4fwd_ip4 elif [[ ${ipv6} -eq 1 ]] then - configure_ip6_remote - - if_verbose echo -e "\nBE configuration:" - config6_be - - if_verbose echo -e "\nFE configuration:" - config6_fe + configure_l4fwd_ip6 fi # create empty results file on remote - $(ssh ${REMOTE_HOST} "> ${REMOTE_RESDIR}/results.out") + if [[ ${USE_TAP} -eq 0 ]] + then + $(ssh ${REMOTE_HOST} "> ${common_result_file}") + fi } # restore netem settings to default @@ -394,21 +491,41 @@ restore_netem() { if [[ ${set_netem} -eq 1 ]] then - ssh ${REMOTE_HOST} tc qdisc del dev ${REMOTE_IFACE} root + use_ssh tc qdisc del dev ${IFACE} root fi } # remove created directories after test is done remove_directories() { - ssh ${REMOTE_HOST} rm -fr ${REMOTE_DIR} + use_ssh rm -fr ${REMOTE_DIR} } # configuration of be/fe config------------------------------------------------ +configure_be_fe() +{ + # call proper configuration + if [[ ${ipv4} -eq 1 ]] + then + if_verbose echo -e "\nBE configuration:" + config4_be + + if_verbose echo -e "\nFE configuration:" + config4_fe + elif [[ ${ipv6} -eq 1 ]] + then + if_verbose echo -e "\nBE configuration:" + config6_be + + if_verbose echo -e "\nFE configuration:" + config6_fe + fi +} + config4_be() { - cat <<EOF > ${L4FWD_BE_CFG_FILE} -port=${DPDK_PORT},masklen=${MASK_IPV4},addr=${REMOTE_IPV4},mac=${REMOTE_MAC} + cat <<EOF > ${L4FWD_BE_CFG_FILE} +port=${DPDK_PORT},masklen=${MASK_IPV4},addr=${LINUX_IPV4},mac=${LINUX_MAC} EOF if_verbose cat ${L4FWD_BE_CFG_FILE} @@ -416,8 +533,8 @@ EOF config6_be() { - cat <<EOF > ${L4FWD_BE_CFG_FILE} -port=${DPDK_PORT},masklen=${MASK_IPV6},addr=${REMOTE_IPV6},mac=${REMOTE_MAC} + cat <<EOF > ${L4FWD_BE_CFG_FILE} +port=${DPDK_PORT},masklen=${MASK_IPV6},addr=${LINUX_IPV6},mac=${LINUX_MAC} EOF if_verbose cat ${L4FWD_BE_CFG_FILE} @@ -426,8 +543,8 @@ EOF config4_fe() { cat <<EOF > ${L4FWD_FE_CFG_FILE} -lcore=${L4FWD_FECORE},belcore=${L4FWD_BECORE},op=echo,laddr=${LOCAL_IPV4}\ -,lport=${TCP_PORT},raddr=${REMOTE_IPV4},rport=0 +lcore=${L4FWD_FECORE},belcore=${L4FWD_BECORE},op=echo,laddr=${L4FWD_IPV4}\ +,lport=${TCP_PORT},raddr=${LINUX_IPV4},rport=0 EOF if_verbose cat ${L4FWD_FE_CFG_FILE} @@ -436,8 +553,8 @@ EOF config6_fe() { cat <<EOF > ${L4FWD_FE_CFG_FILE} -lcore=${L4FWD_FECORE},belcore=${L4FWD_BECORE},op=echo,laddr=${LOCAL_IPV6}\ -,lport=${TCP_PORT},raddr=${REMOTE_IPV6},rport=0 +lcore=${L4FWD_FECORE},belcore=${L4FWD_BECORE},op=echo,laddr=${L4FWD_IPV6}\ +,lport=${TCP_PORT},raddr=${LINUX_IPV6},rport=0 EOF if_verbose cat ${L4FWD_FE_CFG_FILE} diff --git a/examples/l4fwd/test/example_env_vars b/examples/l4fwd/test/example_env_vars index 9877db8..82e0e85 100644 --- a/examples/l4fwd/test/example_env_vars +++ b/examples/l4fwd/test/example_env_vars @@ -2,12 +2,19 @@ # ENV VARIABLES -export REMOTE_HOST=root@10.237.214.104 -export REMOTE_IFACE=enp138s0f0 -export LOCAL_MAC="68:05:ca:04:47:02" +# configuration for tap interface +export ETH_DEV="tap" export L4FWD_PATH=/opt/home/md/Projects/tldk/BuildForTLDK/app/l4fwd export L4FWD_FECORE=5 #optional export L4FWD_BECORE=6 #optional -export ETH_DEV="-w 8a:00.0" + +# configuration for real NIC +#export ETH_DEV="-w 8a:00.0" +#export L4FWD_PATH=/opt/home/md/Projects/tldk/BuildForTLDK/app/l4fwd +#export REMOTE_HOST=root@10.237.214.104 #required for real NIC only +#export REMOTE_IFACE=enp138s0f0 #required for real NIC only +#export LOCAL_MAC="68:05:ca:04:47:02" #required for real NIC only +#export L4FWD_FECORE=5 #optional +#export L4FWD_BECORE=6 #optional # ENV VARIABLES end diff --git a/examples/l4fwd/test/nctxrx.sh b/examples/l4fwd/test/nctxrx.sh index 6a016e0..98f84c2 100644 --- a/examples/l4fwd/test/nctxrx.sh +++ b/examples/l4fwd/test/nctxrx.sh @@ -8,41 +8,54 @@ # script with -h (help) # # User needs to specify following environment variables: -# ETH_DEV - ethernet device to be used on SUT by DPDK +# L4FWD_PATH - path to l4fwd app binary +# ETH_DEV - for real NIC usage - ethernet device to be used on SUT by DPDK +# - for tap interface - tap +# +# User needs to set following enviroment variables in case of real NIC usage: # REMOTE_HOST - ip/hostname of DUT # REMOTE_IFACE - interface name for the test-port on DUT # LOCAL_MAC - MAC address used by DPDK -# L4FWD_PATH - path to l4fwd app binary +# # Optional envirenment variables: # L4FWD_FECORE - core on which l4fwd frontend should run # L4FWD_BECORE - core on which l4fwd backend should run # -# The purpose of the script is to automate validation tests for l4fwd app -# where packets are out of order/lost. It expects l4fwd application being -# run on local linux system (SUT). Script is operating on remote linux -# machine (DUT) with use of ssh. SUT and DUT are connected via NIC. On SUT -# network traffic is managed by DPDK and on DUT by linux. On DUT netcat is -# used to send test data via TCP to TLDK on SUT, which is set to echo mode +# The purpose of the script is to automate validation tests for l4fwd app where +# packets are out of order/lost. Script is operating on local linux machine only +# or on local and remote machine (depending on enviroment variables). +# +# For local machine only, l4fwd application is being run by the script, which +# sets up the tap interface. Created interface is serving a connection for l4fwd +# and netcat within the same OS. +# +# For local/remote linux machine mode, script uses real NIC specified in +# enviroment variable. Connection with remote machine is made via ssh. L4fwd app +# is being run on local machine, while interface and netcat are being set on +# remote side (operated by linux). +# +# Netcat is used to send test data via TCP to l4fwd, which is set to echo mode # (sends back the same data). Depending on test specified, TCP segments are -# artificially changed in sending buffer of DUT, so they are lost in some -# percentage or sent out of order. If specified, report is sent from DUT -# to SUT after all tests were performed. +# artificially changed inside sending buffer, so they are lost in some +# percentage or sent out of order. Report is printed after all tests were +# performed. # -# Example traffic visualisation: -# DUT --(TCP out of order)--> SUT --(TCP with correct order)--> DUT(validation) +# Example of traffic visualisation +# Netcat(TAP/NIC) --(TCP out of order)--> (TAP/NIC)L4FWD(TAP/NIC) -- +# --(TCP with correct order)--> (TAP/NIC)Netcat(validation) # options which can be changed by the user if needed--------------------------- # timeout in [s] for calling nc (in case traffic stuck) timeout=600 -# delay for netem (50 [ms] is default value when reorder option used) +# delay for netem (20 [ms] is default value when reorder option used) delay=0 # default loss of packets [%] value loss=0 -# default probability [%] of not losing burst of packets +# default probability [%] of not loosing burst of packets loss_burst=80 # variables used by script----------------------------------------------------- @@ -55,7 +68,7 @@ rmresults="" set_netem=0 # flag to check if default files should to be used (default 1) -# default files are generated with urandom (couple of sizes) +# default files are generated with urandom default_file=1 # IP protocol version @@ -160,7 +173,7 @@ done # load configuration . $(dirname $0)/config.sh -# send file with results to local machine +# send file with results to local machine when in real NIC mode send_results() { if_verbose echo -e "Sending result file to local" @@ -185,7 +198,7 @@ run_test() # -q 0 -> wait 0 seconds after EOF and quit # timeout to deal with hanging connection when sth went wrong # feed netcat with {of} file to send - # receiving end is redirected to out/...out files + # receiving end is redirected to out/...out file # 'exec' for redirecting nc err output to not mess result cmd="exec 4>&2 \$({ time timeout ${timeout} nc ${nc_ipv6} -q 0 ${nc_addr} ${TCP_PORT} \ @@ -195,13 +208,18 @@ run_test() exec 4>&-" # create temporary file for nc command to execute - xf=$(ssh ${REMOTE_HOST} mktemp -p ${REMOTE_DIR}) + xf=$(use_ssh mktemp -p ${REMOTE_DIR}) # store command from {cmd} into temporaty file - echo "${cmd}" | ssh ${REMOTE_HOST} "cat > ${xf}" + if [[ ${USE_TAP} -eq 0 ]] + then + echo "${cmd}" | ssh ${REMOTE_HOST} "cat > ${xf}" + else + echo "${cmd}" | cat > ${xf} + fi # execute nc command in the background - ssh ${REMOTE_HOST} /bin/bash ${xf} & + use_ssh /bin/bash ${xf} & pids="${pids} $!" @@ -219,7 +237,7 @@ exec 4>&-" wait ${pids} # remove temporary files - ssh ${REMOTE_HOST} rm -f ${rmxf} + use_ssh rm -f ${rmxf} # visual break if_verbose echo -e "\nNetstat:" @@ -227,13 +245,13 @@ exec 4>&-" # prints network information for given {TCP_PORT} number # -n -> show numeric addresses # -a -> show all (both listening and non-listening sockets) - if_verbose ssh ${REMOTE_HOST} netstat -na | grep ${TCP_PORT} + if_verbose use_ssh netstat -na | grep ${TCP_PORT} # visual break if_verbose echo -e "\nJobs:" # display status of jobs in the current session (this bash script) - if_verbose ssh ${REMOTE_HOST} jobs -l + if_verbose use_ssh jobs -l # visual break if_verbose echo -e "\nNetcat processes:" @@ -242,7 +260,7 @@ exec 4>&-" # -e -> show all processes # -f -> do full format listing (more info) # grep -v -> get rid of the following word match from grep output - if_verbose ssh ${REMOTE_HOST} ps -ef | grep "nc " | grep -v grep + if_verbose use_ssh ps -ef | grep "nc " | grep -v grep # visual break if_verbose echo -e "\nRunning validation" @@ -252,13 +270,12 @@ exec 4>&-" while [[ ${i} -lt ${num} ]] do # prints checksum of sent and received file - if_verbose ssh ${REMOTE_HOST} cksum ${REMOTE_DIR}/${of} \ + if_verbose use_ssh cksum ${REMOTE_DIR}/${of} \ ${REMOTE_OUTDIR}/${of}.out.${i} # compares sent and received files if they match # compare {of} and {out/of.out.i} line by line - ssh ${REMOTE_HOST} diff ${REMOTE_DIR}/${of} \ - ${REMOTE_OUTDIR}/${of}.out.${i} + use_ssh diff ${REMOTE_DIR}/${of} ${REMOTE_OUTDIR}/${of}.out.${i} # capture the result of diff command above rc=$? @@ -280,15 +297,15 @@ exec 4>&-" fi # remove received file from out/ directory - ssh ${REMOTE_HOST} rm -f ${REMOTE_OUTDIR}/${of}.out.${i} + use_ssh rm -f ${REMOTE_OUTDIR}/${of}.out.${i} i=$(expr $i + 1) done # remove temporary results - ssh ${REMOTE_HOST} rm -f ${rmresults} + use_ssh rm -f ${rmresults} - if [[ flag_error -eq 1 ]] + if [[ flag_error -ne 0 ]] then return ${flag_error} fi @@ -296,13 +313,17 @@ exec 4>&-" if_verbose echo "" echo -e "TEST SUCCESSFUL - ${of}" if_verbose echo "" + return 0 } # clean up after error or end of tests cleanup() { - send_results + if [[ ${USE_TAP} -eq 0 ]] + then + send_results + fi restore_netem l4fwd_stop remove_directories @@ -310,17 +331,20 @@ cleanup() # script start----------------------------------------------------------------- -#configure remote machine -configure_remote - # start l4fwd app l4fwd_start +#configure configure tap interfaces +configure_interfaces + # check if default files should be used if [[ ${default_file} -eq 0 ]] then if_verbose echo -e "Sending test file to remote" - scp ${scp_suppress} ${file} ${REMOTE_HOST}:${REMOTE_DIR} + if [[ ${USE_TAP} -eq 0 ]] + then + scp ${scp_suppress} ${file} ${REMOTE_HOST}:${REMOTE_DIR} + fi run_test ${file} # check test outcome @@ -330,17 +354,17 @@ then cleanup exit ${ret} fi - ssh ${REMOTE_HOST} rm -f ${REMOTE_DIR}/${file} + use_ssh rm -f ${REMOTE_DIR}/${file} else - # use default files with size 16MB - for size in 16 + # use default files with size 8MB + for size in 8 do # generate file - if_verbose echo -e "Generating ${size}MB file for test" - x=$(ssh ${REMOTE_HOST} mktemp $(basename $0).${size}MB.XXX \ + if_verbose echo -e "\nGenerating ${size}MB file for test" + x=$(use_ssh mktemp $(basename $0).${size}MB.XXX \ -p ${REMOTE_DIR}) - ssh ${REMOTE_HOST} dd if=/dev/urandom of=${x} bs=1M \ + use_ssh dd if=/dev/urandom of=${x} bs=1M \ count=${size} ${dd_suppress} # run test over generated file @@ -355,7 +379,7 @@ else fi # remove generated file only if test successful - ssh ${REMOTE_HOST} rm -f ${x} + use_ssh rm -f ${x} done fi diff --git a/examples/l4fwd/test/run_test.sh b/examples/l4fwd/test/run_test.sh index 690651a..7e892e6 100644 --- a/examples/l4fwd/test/run_test.sh +++ b/examples/l4fwd/test/run_test.sh @@ -2,18 +2,22 @@ # readme section--------------------------------------------------------------- -# usage: /bin/bash run_test.sh [-46lrh] +# usage: /bin/bash run_test.sh [-46alrh] # # Run all tests using nctxrx.sh. Report stored and printed # after tests were done. For details about options run # script with -h (help) # # User needs to specify following environment variables: -# ETH_DEV - ethernet device to be used on SUT by DPDK +# L4FWD_PATH - path to l4fwd app binary +# ETH_DEV - for real NIC usage - ethernet device to be used on SUT by DPDK +# - for tap interface - tap +# +# User needs to set following enviroment variables in case of real NIC usage: # REMOTE_HOST - ip/hostname of DUT # REMOTE_IFACE - interface name for the test-port on DUT # LOCAL_MAC - MAC address used by DPDK -# L4FWD_PATH - path to l4fwd app binary +# # Optional envirenment variables: # L4FWD_FECORE - core on which l4fwd frontend should run # L4FWD_BECORE - core on which l4fwd backend should run @@ -203,7 +207,7 @@ do then echo -e "\nTest for reorder: ${reorder}\t[OK]" else - echo -e "\nTest for reorder: $reorder}\t[FAIL]" + echo -e "\nTest for reorder: ${reorder}\t[FAIL]" error_count=$(expr ${error_count} + 1) fi @@ -274,5 +278,12 @@ else fi # print report after all tests were done -echo -e "Report\n" +echo -e "Report:\n" cat ${result} + +if [[ ${error_count} -eq 0 ]] +then + exit 0 +else + exit 1 +fi diff --git a/examples/l4fwd/udp.h b/examples/l4fwd/udp.h index c079e9c..2465f08 100644 --- a/examples/l4fwd/udp.h +++ b/examples/l4fwd/udp.h @@ -252,25 +252,25 @@ static inline void netfe_pkt_addr(const struct rte_mbuf *m, struct sockaddr_storage *ps, uint16_t family) { - const struct ipv4_hdr *ip4h; - const struct ipv6_hdr *ip6h; - const struct udp_hdr *udph; + const struct rte_ipv4_hdr *ip4h; + const struct rte_ipv6_hdr *ip6h; + const struct rte_udp_hdr *udph; struct sockaddr_in *in4; struct sockaddr_in6 *in6; NETFE_PKT_DUMP(m); - udph = rte_pktmbuf_mtod_offset(m, struct udp_hdr *, -m->l4_len); + udph = rte_pktmbuf_mtod_offset(m, struct rte_udp_hdr *, -m->l4_len); if (family == AF_INET) { in4 = (struct sockaddr_in *)ps; - ip4h = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, + ip4h = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, -(m->l4_len + m->l3_len)); in4->sin_port = udph->src_port; in4->sin_addr.s_addr = ip4h->src_addr; } else { in6 = (struct sockaddr_in6 *)ps; - ip6h = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *, + ip6h = rte_pktmbuf_mtod_offset(m, struct rte_ipv6_hdr *, -(m->l4_len + m->l3_len)); in6->sin6_port = udph->src_port; rte_memcpy(&in6->sin6_addr, ip6h->src_addr, diff --git a/lib/libtle_l4p/Makefile b/lib/libtle_l4p/Makefile index e1357d1..5c8407e 100644 --- a/lib/libtle_l4p/Makefile +++ b/lib/libtle_l4p/Makefile @@ -48,6 +48,7 @@ SYMLINK-y-include += tle_udp.h # this lib dependencies DEPDIRS-y += lib/libtle_misc +DEPDIRS-y += lib/libtle_memtank DEPDIRS-y += lib/libtle_dring DEPDIRS-y += lib/libtle_timer diff --git a/lib/libtle_l4p/ctx.c b/lib/libtle_l4p/ctx.c index b8067f0..b810983 100644 --- a/lib/libtle_l4p/ctx.c +++ b/lib/libtle_l4p/ctx.c @@ -116,8 +116,6 @@ tle_ctx_create(const struct tle_ctx_param *ctx_prm) for (i = 0; i != RTE_DIM(ctx->use); i++) tle_pbm_init(ctx->use + i, LPORT_START_BLK); - ctx->streams.nb_free = ctx->prm.max_streams; - /* Initialization of siphash state is done here to speed up the * fastpath processing. */ diff --git a/lib/libtle_l4p/misc.h b/lib/libtle_l4p/misc.h index 327296f..e1efb0d 100644 --- a/lib/libtle_l4p/misc.h +++ b/lib/libtle_l4p/misc.h @@ -207,7 +207,7 @@ __udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs, * The non-complemented checksum to set in the L4 header. */ static inline uint16_t -_ipv4x_phdr_cksum(const struct ipv4_hdr *ipv4_hdr, size_t ipv4h_len, +_ipv4x_phdr_cksum(const struct rte_ipv4_hdr *ipv4_hdr, size_t ipv4h_len, uint64_t ol_flags) { uint32_t s0, s1; @@ -243,7 +243,7 @@ _ipv4x_phdr_cksum(const struct ipv4_hdr *ipv4_hdr, size_t ipv4h_len, */ static inline int _ipv4_udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs, - const struct ipv4_hdr *ipv4_hdr) + const struct rte_ipv4_hdr *ipv4_hdr) { uint32_t cksum; @@ -267,7 +267,7 @@ _ipv4_udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs, */ static inline int _ipv6_udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs, - const struct ipv6_hdr *ipv6_hdr) + const struct rte_ipv6_hdr *ipv6_hdr) { uint32_t cksum; @@ -293,9 +293,9 @@ static inline int check_pkt_csum(const struct rte_mbuf *m, uint64_t ol_flags, uint32_t type, uint32_t proto) { - const struct ipv4_hdr *l3h4; - const struct ipv6_hdr *l3h6; - const struct udp_hdr *l4h; + const struct rte_ipv4_hdr *l3h4; + const struct rte_ipv6_hdr *l3h6; + const struct rte_udp_hdr *l4h; uint64_t fl3, fl4; uint16_t csum; int32_t ret; @@ -313,8 +313,10 @@ check_pkt_csum(const struct rte_mbuf *m, uint64_t ol_flags, uint32_t type, return 1; /* case 2: either ip or l4 or both cksum is unknown */ - l3h4 = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, m->l2_len); - l3h6 = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *, m->l2_len); + l3h4 = rte_pktmbuf_mtod_offset(m, const struct rte_ipv4_hdr *, + m->l2_len); + l3h6 = rte_pktmbuf_mtod_offset(m, const struct rte_ipv6_hdr *, + m->l2_len); ret = 0; if (fl3 == PKT_RX_IP_CKSUM_UNKNOWN && l3h4->hdr_checksum != 0) { @@ -329,7 +331,7 @@ check_pkt_csum(const struct rte_mbuf *m, uint64_t ol_flags, uint32_t type, * for IPv6 valid UDP cksum is mandatory. */ if (type == TLE_V4) { - l4h = (const struct udp_hdr *)((uintptr_t)l3h4 + + l4h = (const struct rte_udp_hdr *)((uintptr_t)l3h4 + m->l3_len); csum = (proto == IPPROTO_UDP && l4h->dgram_cksum == 0) ? UINT16_MAX : _ipv4_udptcp_mbuf_cksum(m, diff --git a/lib/libtle_l4p/stream.h b/lib/libtle_l4p/stream.h index 49a2809..ebefa6c 100644 --- a/lib/libtle_l4p/stream.h +++ b/lib/libtle_l4p/stream.h @@ -151,13 +151,13 @@ stream_get_dest(struct tle_stream *s, const void *dst_addr, dst->ol_flags = dev->tx.ol_flags[s->type]; if (s->type == TLE_V4) { - struct ipv4_hdr *l3h; - l3h = (struct ipv4_hdr *)(dst->hdr + dst->l2_len); + struct rte_ipv4_hdr *l3h; + l3h = (struct rte_ipv4_hdr *)(dst->hdr + dst->l2_len); l3h->src_addr = dev->prm.local_addr4.s_addr; l3h->dst_addr = d4->s_addr; } else { - struct ipv6_hdr *l3h; - l3h = (struct ipv6_hdr *)(dst->hdr + dst->l2_len); + struct rte_ipv6_hdr *l3h; + l3h = (struct rte_ipv6_hdr *)(dst->hdr + dst->l2_len); rte_memcpy(l3h->src_addr, &dev->prm.local_addr6, sizeof(l3h->src_addr)); rte_memcpy(l3h->dst_addr, d6, sizeof(l3h->dst_addr)); diff --git a/lib/libtle_l4p/tcp_ctl.h b/lib/libtle_l4p/tcp_ctl.h index bec1e76..7dde8ff 100644 --- a/lib/libtle_l4p/tcp_ctl.h +++ b/lib/libtle_l4p/tcp_ctl.h @@ -143,10 +143,10 @@ empty_lq(struct tle_tcp_stream *s) static inline void tcp_stream_reset(struct tle_ctx *ctx, struct tle_tcp_stream *s) { - struct stbl *st; uint16_t uop; + struct tcp_streams *ts; - st = CTX_TCP_STLB(ctx); + ts = CTX_TCP_STREAMS(ctx); /* reset TX armed */ rte_atomic32_set(&s->tx.arm, 0); @@ -167,7 +167,7 @@ tcp_stream_reset(struct tle_ctx *ctx, struct tle_tcp_stream *s) if (s->ste != NULL) { /* remove entry from RX streams table */ - stbl_del_stream(st, s->ste, s, + stbl_del_stream(&ts->st, s->ste, s, (s->flags & TLE_CTX_FLAG_ST) == 0); s->ste = NULL; empty_rq(s); @@ -181,7 +181,36 @@ tcp_stream_reset(struct tle_ctx *ctx, struct tle_tcp_stream *s) * if there still are pkts queued for TX, * then put this stream to the tail of free list. */ - put_stream(ctx, &s->s, TCP_STREAM_TX_FINISHED(s)); + if (TCP_STREAM_TX_PENDING(s)) + put_stream(ctx, &s->s, 0); + else { + s->s.type = TLE_VNUM; + tle_memtank_free(ts->mts, (void **)&s, 1, 0); + } +} + +static inline struct tle_tcp_stream * +tcp_stream_get(struct tle_ctx *ctx, uint32_t flag) +{ + struct tle_stream *s; + struct tle_tcp_stream *cs; + struct tcp_streams *ts; + + ts = CTX_TCP_STREAMS(ctx); + + /* check TX pending list */ + s = get_stream(ctx); + cs = TCP_STREAM(s); + if (s != NULL) { + if (TCP_STREAM_TX_FINISHED(cs)) + return cs; + put_stream(ctx, &cs->s, 0); + } + + if (tle_memtank_alloc(ts->mts, (void **)&cs, 1, flag) != 1) + return NULL; + + return cs; } #ifdef __cplusplus diff --git a/lib/libtle_l4p/tcp_misc.h b/lib/libtle_l4p/tcp_misc.h index 0cef8b2..01c1e67 100644 --- a/lib/libtle_l4p/tcp_misc.h +++ b/lib/libtle_l4p/tcp_misc.h @@ -33,10 +33,10 @@ extern "C" { #define TCP_WSCALE_DEFAULT 7 #define TCP_WSCALE_NONE 0 -#define TCP_TX_HDR_MAX (sizeof(struct tcp_hdr) + TCP_TX_OPT_LEN_MAX) +#define TCP_TX_HDR_MAX (sizeof(struct rte_tcp_hdr) + TCP_TX_OPT_LEN_MAX) /* max header size for normal data+ack packet */ -#define TCP_TX_HDR_DACK (sizeof(struct tcp_hdr) + TCP_TX_OPT_LEN_TMS) +#define TCP_TX_HDR_DACK (sizeof(struct rte_tcp_hdr) + TCP_TX_OPT_LEN_TMS) #define TCP4_MIN_MSS 536 @@ -44,10 +44,12 @@ extern "C" { /* default MTU, no TCP options. */ #define TCP4_NOP_MSS \ - (ETHER_MTU - sizeof(struct ipv4_hdr) - sizeof(struct tcp_hdr)) + (RTE_ETHER_MTU - sizeof(struct rte_ipv4_hdr) - \ + sizeof(struct rte_tcp_hdr)) #define TCP6_NOP_MSS \ - (ETHER_MTU - sizeof(struct ipv6_hdr) - sizeof(struct tcp_hdr)) + (RTE_ETHER_MTU - sizeof(struct rte_ipv6_hdr) - \ + sizeof(struct rte_tcp_hdr)) /* default MTU, TCP options present */ #define TCP4_OP_MSS (TCP4_NOP_MSS - TCP_TX_OPT_LEN_MAX) @@ -256,7 +258,7 @@ tcp_seq_min(uint32_t l, uint32_t r) } static inline void -get_seg_info(const struct tcp_hdr *th, union seg_info *si) +get_seg_info(const struct rte_tcp_hdr *th, union seg_info *si) { __m128i v; const __m128i bswap_mask = @@ -421,7 +423,7 @@ static inline void get_pkt_info(const struct rte_mbuf *m, union pkt_info *pi, union seg_info *si) { uint32_t len, type; - const struct tcp_hdr *tcph; + const struct rte_tcp_hdr *tcph; const union l4_ports *prt; const union ipv4_addrs *pa4; @@ -436,17 +438,17 @@ get_pkt_info(const struct rte_mbuf *m, union pkt_info *pi, union seg_info *si) if (type == TLE_V4) { pa4 = rte_pktmbuf_mtod_offset(m, const union ipv4_addrs *, - len + offsetof(struct ipv4_hdr, src_addr)); + len + offsetof(struct rte_ipv4_hdr, src_addr)); pi->addr4.raw = pa4->raw; } else if (type == TLE_V6) { pi->addr6 = rte_pktmbuf_mtod_offset(m, const union ipv6_addrs *, - len + offsetof(struct ipv6_hdr, src_addr)); + len + offsetof(struct rte_ipv6_hdr, src_addr)); } len += m->l3_len; - tcph = rte_pktmbuf_mtod_offset(m, const struct tcp_hdr *, len); + tcph = rte_pktmbuf_mtod_offset(m, const struct rte_tcp_hdr *, len); prt = (const union l4_ports *) - ((uintptr_t)tcph + offsetof(struct tcp_hdr, src_port)); + ((uintptr_t)tcph + offsetof(struct rte_tcp_hdr, src_port)); pi->tf.flags = tcph->tcp_flags; pi->tf.type = type; pi->csf = m->ol_flags & (PKT_RX_IP_CKSUM_MASK | PKT_RX_L4_CKSUM_MASK); diff --git a/lib/libtle_l4p/tcp_rxtx.c b/lib/libtle_l4p/tcp_rxtx.c index a519645..b1aad60 100644 --- a/lib/libtle_l4p/tcp_rxtx.c +++ b/lib/libtle_l4p/tcp_rxtx.c @@ -183,7 +183,7 @@ get_ip_pid(struct tle_dev *dev, uint32_t num, uint32_t type, uint32_t st) } static inline void -fill_tcph(struct tcp_hdr *l4h, const struct tcb *tcb, union l4_ports port, +fill_tcph(struct rte_tcp_hdr *l4h, const struct tcb *tcb, union l4_ports port, uint32_t seq, uint8_t hlen, uint8_t flags) { uint16_t wnd; @@ -217,7 +217,7 @@ tcp_fill_mbuf(struct rte_mbuf *m, const struct tle_tcp_stream *s, uint32_t pid, uint32_t swcsm) { uint32_t l4, len, plen; - struct tcp_hdr *l4h; + struct rte_tcp_hdr *l4h; char *l2h; len = dst->l2_len + dst->l3_len; @@ -239,7 +239,7 @@ tcp_fill_mbuf(struct rte_mbuf *m, const struct tle_tcp_stream *s, rte_memcpy(l2h, dst->hdr, len); /* setup TCP header & options */ - l4h = (struct tcp_hdr *)(l2h + len); + l4h = (struct rte_tcp_hdr *)(l2h + len); fill_tcph(l4h, &s->tcb, port, seq, l4, flags); /* setup mbuf TX offload related fields. */ @@ -249,8 +249,8 @@ tcp_fill_mbuf(struct rte_mbuf *m, const struct tle_tcp_stream *s, /* update proto specific fields. */ if (s->s.type == TLE_V4) { - struct ipv4_hdr *l3h; - l3h = (struct ipv4_hdr *)(l2h + dst->l2_len); + struct rte_ipv4_hdr *l3h; + l3h = (struct rte_ipv4_hdr *)(l2h + dst->l2_len); l3h->packet_id = rte_cpu_to_be_16(pid); l3h->total_length = rte_cpu_to_be_16(plen + dst->l3_len + l4); @@ -263,8 +263,8 @@ tcp_fill_mbuf(struct rte_mbuf *m, const struct tle_tcp_stream *s, if ((ol_flags & PKT_TX_IP_CKSUM) == 0 && swcsm != 0) l3h->hdr_checksum = _ipv4x_cksum(l3h, m->l3_len); } else { - struct ipv6_hdr *l3h; - l3h = (struct ipv6_hdr *)(l2h + dst->l2_len); + struct rte_ipv6_hdr *l3h; + l3h = (struct rte_ipv6_hdr *)(l2h + dst->l2_len); l3h->payload_len = rte_cpu_to_be_16(plen + l4); if ((ol_flags & PKT_TX_TCP_CKSUM) != 0) l4h->cksum = rte_ipv6_phdr_cksum(l3h, ol_flags); @@ -285,11 +285,11 @@ static inline void tcp_update_mbuf(struct rte_mbuf *m, uint32_t type, const struct tcb *tcb, uint32_t seq, uint32_t pid) { - struct tcp_hdr *l4h; + struct rte_tcp_hdr *l4h; uint32_t len; len = m->l2_len + m->l3_len; - l4h = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, len); + l4h = rte_pktmbuf_mtod_offset(m, struct rte_tcp_hdr *, len); l4h->sent_seq = rte_cpu_to_be_32(seq); l4h->recv_ack = rte_cpu_to_be_32(tcb->rcv.nxt); @@ -298,8 +298,9 @@ tcp_update_mbuf(struct rte_mbuf *m, uint32_t type, const struct tcb *tcb, fill_tms_opts(l4h + 1, tcb->snd.ts, tcb->rcv.ts); if (type == TLE_V4) { - struct ipv4_hdr *l3h; - l3h = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, m->l2_len); + struct rte_ipv4_hdr *l3h; + l3h = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, + m->l2_len); l3h->hdr_checksum = 0; l3h->packet_id = rte_cpu_to_be_16(pid); if ((m->ol_flags & PKT_TX_IP_CKSUM) == 0) @@ -312,14 +313,14 @@ tcp_update_mbuf(struct rte_mbuf *m, uint32_t type, const struct tcb *tcb, l4h->cksum = 0; if (type == TLE_V4) { - struct ipv4_hdr *l3h; - l3h = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, + struct rte_ipv4_hdr *l3h; + l3h = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, m->l2_len); l4h->cksum = _ipv4_udptcp_mbuf_cksum(m, len, l3h); } else { - struct ipv6_hdr *l3h; - l3h = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *, + struct rte_ipv6_hdr *l3h; + l3h = rte_pktmbuf_mtod_offset(m, struct rte_ipv6_hdr *, m->l2_len); l4h->cksum = _ipv6_udptcp_mbuf_cksum(m, len, l3h); } @@ -635,7 +636,7 @@ sync_ack(struct tle_tcp_stream *s, const union pkt_info *pi, struct tle_dev *dev; const void *da; struct tle_dest dst; - const struct tcp_hdr *th; + const struct rte_tcp_hdr *th; type = s->s.type; @@ -649,10 +650,14 @@ sync_ack(struct tle_tcp_stream *s, const union pkt_info *pi, if (rc < 0) return rc; - th = rte_pktmbuf_mtod_offset(m, const struct tcp_hdr *, + th = rte_pktmbuf_mtod_offset(m, const struct rte_tcp_hdr *, m->l2_len + m->l3_len); get_syn_opts(&s->tcb.so, (uintptr_t)(th + 1), m->l4_len - sizeof(*th)); + /* reset wscale option if timestamp is not present */ + if (s->tcb.so.ts.val == 0) + s->tcb.so.wscale = 0; + s->tcb.rcv.nxt = si->seq + 1; seq = sync_gen_seq(pi, s->tcb.rcv.nxt, ts, s->tcb.so.mss, s->s.ctx->prm.hash_alg, @@ -710,7 +715,7 @@ rx_tms_opt(const struct tcb *tcb, const struct rte_mbuf *mb) { union tsopt ts; uintptr_t opt; - const struct tcp_hdr *th; + const struct rte_tcp_hdr *th; if (tcb->so.ts.val != 0) { opt = rte_pktmbuf_mtod_offset(mb, uintptr_t, @@ -782,7 +787,7 @@ restore_syn_opt(union seg_info *si, union tsopt *to, { int32_t rc; uint32_t len; - const struct tcp_hdr *th; + const struct rte_tcp_hdr *th; /* check that ACK, etc fields are what we expected. */ rc = sync_check_ack(pi, si->seq, si->ack - 1, ts, @@ -793,7 +798,7 @@ restore_syn_opt(union seg_info *si, union tsopt *to, si->mss = rc; - th = rte_pktmbuf_mtod_offset(mb, const struct tcp_hdr *, + th = rte_pktmbuf_mtod_offset(mb, const struct rte_tcp_hdr *, mb->l2_len + mb->l3_len); len = mb->l4_len - sizeof(*th); to[0] = get_tms_opts((uintptr_t)(th + 1), len); @@ -947,15 +952,15 @@ rx_ack_listen(struct tle_tcp_stream *s, struct stbl *st, return rc; /* allocate new stream */ - ts = get_stream(ctx); - cs = TCP_STREAM(ts); - if (ts == NULL) + cs = tcp_stream_get(ctx, 0); + if (cs == NULL) return ENFILE; /* prepare stream to handle new connection */ if (accept_prep_stream(s, st, cs, &to, tms, pi, si) == 0) { /* put new stream in the accept queue */ + ts = &cs->s; if (_rte_ring_enqueue_burst(s->rx.q, (void * const *)&ts, 1) == 1) { *csp = cs; @@ -1549,7 +1554,7 @@ rx_synack(struct tle_tcp_stream *s, uint32_t ts, uint32_t state, struct resp_info *rsp) { struct syn_opts so; - struct tcp_hdr *th; + struct rte_tcp_hdr *th; if (state != TCP_ST_SYN_SENT) return -EINVAL; @@ -1566,7 +1571,7 @@ rx_synack(struct tle_tcp_stream *s, uint32_t ts, uint32_t state, return 0; } - th = rte_pktmbuf_mtod_offset(mb, struct tcp_hdr *, + th = rte_pktmbuf_mtod_offset(mb, struct rte_tcp_hdr *, mb->l2_len + mb->l3_len); get_syn_opts(&so, (uintptr_t)(th + 1), mb->l4_len - sizeof(*th)); @@ -1951,12 +1956,15 @@ tle_tcp_stream_accept(struct tle_stream *ts, struct tle_stream *rs[], { uint32_t n; struct tle_tcp_stream *s; + struct tle_memtank *mts; s = TCP_STREAM(ts); n = _rte_ring_dequeue_burst(s->rx.q, (void **)rs, num); if (n == 0) return 0; + mts = CTX_TCP_MTS(ts->ctx); + /* * if we still have packets to read, * then rearm stream RX event. @@ -1967,6 +1975,7 @@ tle_tcp_stream_accept(struct tle_stream *ts, struct tle_stream *rs[], tcp_stream_release(s); } + tle_memtank_grow(mts); return n; } diff --git a/lib/libtle_l4p/tcp_stream.c b/lib/libtle_l4p/tcp_stream.c index a212405..fce3b9a 100644 --- a/lib/libtle_l4p/tcp_stream.c +++ b/lib/libtle_l4p/tcp_stream.c @@ -28,6 +28,8 @@ #include "tcp_ofo.h" #include "tcp_txq.h" +#define MAX_STREAM_BURST 0x40 + static void unuse_stream(struct tle_tcp_stream *s) { @@ -42,11 +44,13 @@ tcp_fini_streams(struct tle_ctx *ctx) ts = CTX_TCP_STREAMS(ctx); if (ts != NULL) { - stbl_fini(&ts->st); - /* free the timer wheel */ + stbl_fini(&ts->st); tle_timer_free(ts->tmr); rte_free(ts->tsq); + tle_memtank_dump(stdout, ts->mts, TLE_MTANK_DUMP_STAT); + tle_memtank_sanity_check(ts->mts, 0); + tle_memtank_destroy(ts->mts); STAILQ_INIT(&ts->dr.fe); STAILQ_INIT(&ts->dr.be); @@ -122,7 +126,7 @@ calc_stream_szofs(struct tle_ctx *ctx, struct stream_szofs *szofs) szofs->size = sz; } -static int +static void init_stream(struct tle_ctx *ctx, struct tle_tcp_stream *s, const struct stream_szofs *szofs) { @@ -163,9 +167,6 @@ init_stream(struct tle_ctx *ctx, struct tle_tcp_stream *s, s->s.ctx = ctx; unuse_stream(s); - STAILQ_INSERT_TAIL(&ctx->streams.free, &s->s, link); - - return 0; } static void @@ -178,38 +179,107 @@ tcp_free_drbs(struct tle_stream *s, struct tle_drb *drb[], uint32_t nb_drb) } static struct tle_timer_wheel * -alloc_timers(uint32_t num, uint32_t mshift, int32_t socket) +alloc_timers(const struct tle_ctx *ctx) { + struct tle_timer_wheel *twl; struct tle_timer_wheel_args twprm; twprm.tick_size = TCP_RTO_GRANULARITY; - twprm.max_timer = num; - twprm.socket_id = socket; - return tle_timer_create(&twprm, tcp_get_tms(mshift)); + twprm.max_timer = ctx->prm.max_streams; + twprm.socket_id = ctx->prm.socket_id; + + twl = tle_timer_create(&twprm, tcp_get_tms(ctx->cycles_ms_shift)); + if (twl == NULL) + TCP_LOG(ERR, "alloc_timers(ctx=%p) failed with error=%d\n", + ctx, rte_errno); + return twl; +} + +static void * +mts_alloc(size_t sz, void *udata) +{ + struct tle_ctx *ctx; + + ctx = udata; + return rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE, + ctx->prm.socket_id); +} + +static void +mts_free(void *p, void *udata) +{ + RTE_SET_USED(udata); + rte_free(p); +} + +static void +mts_init(void *pa[], uint32_t num, void *udata) +{ + uint32_t i; + struct tle_ctx *ctx; + struct tcp_streams *ts; + + ctx = udata; + ts = CTX_TCP_STREAMS(ctx); + + for (i = 0; i != num; i++) + init_stream(ctx, pa[i], &ts->szofs); +} + +static struct tle_memtank * +alloc_mts(struct tle_ctx *ctx, uint32_t stream_size) +{ + struct tle_memtank *mts; + struct tle_memtank_prm prm; + + static const struct tle_memtank_prm cprm = { + .obj_align = RTE_CACHE_LINE_SIZE, + .flags = TLE_MTANK_OBJ_DBG, + .alloc = mts_alloc, + .free = mts_free, + .init = mts_init, + }; + + prm = cprm; + prm.udata = ctx; + + prm.obj_size = stream_size; + + prm.min_free = (ctx->prm.free_streams.min != 0) ? + ctx->prm.free_streams.min : ctx->prm.max_streams; + prm.max_free = (ctx->prm.free_streams.max > prm.min_free) ? + ctx->prm.free_streams.max : prm.min_free; + + prm.nb_obj_chunk = MAX_STREAM_BURST; + prm.max_obj = ctx->prm.max_streams; + + mts = tle_memtank_create(&prm); + if (mts == NULL) + TCP_LOG(ERR, "%s(ctx=%p) failed with error=%d\n", + __func__, ctx, rte_errno); + else + tle_memtank_grow(mts); + + return mts; } static int tcp_init_streams(struct tle_ctx *ctx) { - size_t sz; - uint32_t f, i; + uint32_t f; int32_t rc; struct tcp_streams *ts; - struct tle_tcp_stream *ps; struct stream_szofs szofs; f = ((ctx->prm.flags & TLE_CTX_FLAG_ST) == 0) ? 0 : (RING_F_SP_ENQ | RING_F_SC_DEQ); calc_stream_szofs(ctx, &szofs); + TCP_LOG(NOTICE, "ctx:%p, caluclated stream size: %u\n", + ctx, szofs.size); - sz = sizeof(*ts) + szofs.size * ctx->prm.max_streams; - ts = rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE, + ts = rte_zmalloc_socket(NULL, sizeof(*ts), RTE_CACHE_LINE_SIZE, ctx->prm.socket_id); - - TCP_LOG(NOTICE, "allocation of %zu bytes on socket %d " - "for %u tcp_streams returns %p\n", - sz, ctx->prm.socket_id, ctx->prm.max_streams, ts); if (ts == NULL) return -ENOMEM; @@ -221,29 +291,22 @@ tcp_init_streams(struct tle_ctx *ctx) ctx->streams.buf = ts; STAILQ_INIT(&ctx->streams.free); - ts->tmr = alloc_timers(ctx->prm.max_streams, ctx->cycles_ms_shift, - ctx->prm.socket_id); - if (ts->tmr == NULL) { - TCP_LOG(ERR, "alloc_timers(ctx=%p) failed with error=%d\n", - ctx, rte_errno); - rc = -ENOMEM; - } else { - ts->tsq = alloc_ring(ctx->prm.max_streams, - f | RING_F_SC_DEQ, ctx->prm.socket_id); - if (ts->tsq == NULL) + rc = stbl_init(&ts->st, ctx->prm.max_streams, ctx->prm.socket_id); + + if (rc == 0) { + ts->tsq = alloc_ring(ctx->prm.max_streams, f | RING_F_SC_DEQ, + ctx->prm.socket_id); + ts->tmr = alloc_timers(ctx); + ts->mts = alloc_mts(ctx, szofs.size); + + if (ts->tsq == NULL || ts->tmr == NULL || ts->mts == NULL) rc = -ENOMEM; - else - rc = stbl_init(&ts->st, ctx->prm.max_streams, - ctx->prm.socket_id); - } - for (i = 0; rc == 0 && i != ctx->prm.max_streams; i++) { - ps = (void *)((uintptr_t)ts->s + i * ts->szofs.size); - rc = init_stream(ctx, ps, &ts->szofs); + tle_memtank_dump(stdout, ts->mts, TLE_MTANK_DUMP_STAT); } if (rc != 0) { - TCP_LOG(ERR, "initalisation of %u-th stream failed", i); + TCP_LOG(ERR, "initalisation of tcp streams failed"); tcp_fini_streams(ctx); } @@ -302,6 +365,7 @@ struct tle_stream * tle_tcp_stream_open(struct tle_ctx *ctx, const struct tle_tcp_stream_param *prm) { + struct tcp_streams *ts; struct tle_tcp_stream *s; int32_t rc; @@ -310,15 +374,11 @@ tle_tcp_stream_open(struct tle_ctx *ctx, return NULL; } - s = (struct tle_tcp_stream *)get_stream(ctx); - if (s == NULL) { - rte_errno = ENFILE; - return NULL; + ts = CTX_TCP_STREAMS(ctx); - /* some TX still pending for that stream. */ - } else if (TCP_STREAM_TX_PENDING(s)) { - put_stream(ctx, &s->s, 0); - rte_errno = EAGAIN; + s = tcp_stream_get(ctx, TLE_MTANK_ALLOC_CHUNK | TLE_MTANK_ALLOC_GROW); + if (s == NULL) { + rte_errno = ENFILE; return NULL; } @@ -328,7 +388,8 @@ tle_tcp_stream_open(struct tle_ctx *ctx, (const struct sockaddr *)&prm->addr.remote); if (rc != 0) { - put_stream(ctx, &s->s, 1); + tle_memtank_free(ts->mts, (void **)&s, 1, + TLE_MTANK_FREE_SHRINK); rte_errno = rc; return NULL; } @@ -426,18 +487,17 @@ tle_tcp_stream_close_bulk(struct tle_stream *ts[], uint32_t num) rc = 0; - for (i = 0; i != num; i++) { + for (i = 0; i != num && rc == 0; i++) { s = TCP_STREAM(ts[i]); - if (ts[i] == NULL || s->s.type >= TLE_VNUM) { + if (ts[i] == NULL || s->s.type >= TLE_VNUM) rc = EINVAL; - break; - } - ctx = s->s.ctx; - rc = stream_close(ctx, s); - if (rc != 0) - break; + else { + ctx = s->s.ctx; + rc = stream_close(ctx, s); + tle_memtank_shrink(CTX_TCP_MTS(ctx)); + } } if (rc != 0) @@ -448,6 +508,7 @@ tle_tcp_stream_close_bulk(struct tle_stream *ts[], uint32_t num) int tle_tcp_stream_close(struct tle_stream *ts) { + int32_t rc; struct tle_ctx *ctx; struct tle_tcp_stream *s; @@ -456,7 +517,9 @@ tle_tcp_stream_close(struct tle_stream *ts) return -EINVAL; ctx = s->s.ctx; - return stream_close(ctx, s); + rc = stream_close(ctx, s); + tle_memtank_shrink(CTX_TCP_MTS(ctx)); + return rc; } int diff --git a/lib/libtle_l4p/tcp_stream.h b/lib/libtle_l4p/tcp_stream.h index 1bb2a42..a3d00dc 100644 --- a/lib/libtle_l4p/tcp_stream.h +++ b/lib/libtle_l4p/tcp_stream.h @@ -18,6 +18,7 @@ #include <rte_vect.h> #include <tle_dring.h> +#include <tle_memtank.h> #include <tle_tcp.h> #include <tle_event.h> @@ -176,9 +177,9 @@ struct tcp_streams { struct stbl st; struct tle_timer_wheel *tmr; /* timer wheel */ struct rte_ring *tsq; /* to-send streams queue */ + struct tle_memtank *mts; /* memtank to allocate streams from */ struct sdr dr; /* death row for zombie streams */ struct stream_szofs szofs; /* size and offsets for stream data */ - struct tle_tcp_stream s[]; /* array of allocated streams. */ }; #define CTX_TCP_STREAMS(ctx) ((struct tcp_streams *)(ctx)->streams.buf) @@ -186,6 +187,7 @@ struct tcp_streams { #define CTX_TCP_TMWHL(ctx) (CTX_TCP_STREAMS(ctx)->tmr) #define CTX_TCP_TSQ(ctx) (CTX_TCP_STREAMS(ctx)->tsq) #define CTX_TCP_SDR(ctx) (&CTX_TCP_STREAMS(ctx)->dr) +#define CTX_TCP_MTS(ctx) (CTX_TCP_STREAMS(ctx)->mts) #ifdef __cplusplus } diff --git a/lib/libtle_l4p/tle_ctx.h b/lib/libtle_l4p/tle_ctx.h index de78a6b..391cfe3 100644 --- a/lib/libtle_l4p/tle_ctx.h +++ b/lib/libtle_l4p/tle_ctx.h @@ -112,6 +112,12 @@ struct tle_ctx_param { int32_t socket_id; /**< socket ID to allocate memory for. */ uint32_t proto; /**< L4 proto to handle. */ uint32_t max_streams; /**< max number of streams in context. */ + struct { + uint32_t min; + /**< min number of free streams (grow threshold). */ + uint32_t max; + /**< max number of free streams (shrink threshold). */ + } free_streams; uint32_t max_stream_rbufs; /**< max recv mbufs per stream. */ uint32_t max_stream_sbufs; /**< max send mbufs per stream. */ uint32_t send_bulk_size; /**< expected # of packets per send call. */ diff --git a/lib/libtle_l4p/udp_rxtx.c b/lib/libtle_l4p/udp_rxtx.c index 84a13ea..8963df5 100644 --- a/lib/libtle_l4p/udp_rxtx.c +++ b/lib/libtle_l4p/udp_rxtx.c @@ -69,16 +69,16 @@ pkt_info(struct rte_mbuf *m, union l4_ports *ports, union ipv4_addrs *addr4, len = m->l2_len; if (ret.src == TLE_V4) { pa4 = rte_pktmbuf_mtod_offset(m, union ipv4_addrs *, - len + offsetof(struct ipv4_hdr, src_addr)); + len + offsetof(struct rte_ipv4_hdr, src_addr)); addr4->raw = pa4->raw; } else if (ret.src == TLE_V6) { *addr6 = rte_pktmbuf_mtod_offset(m, union ipv6_addrs *, - len + offsetof(struct ipv6_hdr, src_addr)); + len + offsetof(struct rte_ipv6_hdr, src_addr)); } len += m->l3_len; up = rte_pktmbuf_mtod_offset(m, union l4_ports *, - len + offsetof(struct udp_hdr, src_port)); + len + offsetof(struct rte_udp_hdr, src_port)); ports->raw = up->raw; ret.dst = ports->dst; return ret; @@ -355,8 +355,8 @@ udp_fill_mbuf(struct rte_mbuf *m, /* update proto specific fields. */ if (type == TLE_V4) { - struct ipv4_hdr *l3h; - l3h = (struct ipv4_hdr *)(l2h + dst->l2_len); + struct rte_ipv4_hdr *l3h; + l3h = (struct rte_ipv4_hdr *)(l2h + dst->l2_len); l3h->packet_id = rte_cpu_to_be_16(pid); l3h->total_length = rte_cpu_to_be_16(plen + dst->l3_len + sizeof(*l4h)); @@ -370,8 +370,8 @@ udp_fill_mbuf(struct rte_mbuf *m, if ((ol_flags & PKT_TX_IP_CKSUM) == 0) l3h->hdr_checksum = _ipv4x_cksum(l3h, m->l3_len); } else { - struct ipv6_hdr *l3h; - l3h = (struct ipv6_hdr *)(l2h + dst->l2_len); + struct rte_ipv6_hdr *l3h; + l3h = (struct rte_ipv6_hdr *)(l2h + dst->l2_len); l3h->payload_len = rte_cpu_to_be_16(plen + sizeof(*l4h)); if ((ol_flags & PKT_TX_UDP_CKSUM) != 0) l4h->cksum = rte_ipv6_phdr_cksum(l3h, ol_flags); @@ -389,13 +389,13 @@ udp_fill_mbuf(struct rte_mbuf *m, static inline void frag_fixup(const struct rte_mbuf *ms, struct rte_mbuf *mf, uint32_t type) { - struct ipv4_hdr *l3h; + struct rte_ipv4_hdr *l3h; mf->ol_flags = ms->ol_flags; mf->tx_offload = ms->tx_offload; if (type == TLE_V4 && (ms->ol_flags & PKT_TX_IP_CKSUM) == 0) { - l3h = rte_pktmbuf_mtod(mf, struct ipv4_hdr *); + l3h = rte_pktmbuf_mtod(mf, struct rte_ipv4_hdr *); l3h->hdr_checksum = _ipv4x_cksum(l3h, mf->l3_len); } } diff --git a/lib/libtle_l4p/udp_stream.c b/lib/libtle_l4p/udp_stream.c index 29f5a40..a50c420 100644 --- a/lib/libtle_l4p/udp_stream.c +++ b/lib/libtle_l4p/udp_stream.c @@ -170,6 +170,7 @@ udp_init_streams(struct tle_ctx *ctx) } } + ctx->streams.nb_free = ctx->prm.max_streams; return 0; } diff --git a/test/gtest/Makefile b/test/gtest/Makefile index e980c23..3858306 100644 --- a/test/gtest/Makefile +++ b/test/gtest/Makefile @@ -125,7 +125,7 @@ LDLIBS += -lstdc++ LDLIBS += -L$(GMOCK_DIR) -L$(GMOCK_DIR)/../lib -lgmock LDLIBS += -L$(GMOCK_DIR)/gtest -L$(GMOCK_DIR)/../lib -lgtest LDLIBS += -L$(RTE_OUTPUT)/lib -LDLIBS += -whole-archive -ltle_l4p -ltle_dring -ltle_timer +LDLIBS += -whole-archive -ltle_l4p -ltle_dring -ltle_memtank -ltle_timer include $(TLDK_ROOT)/mk/tle.app.mk endif diff --git a/test/gtest/test_common.cpp b/test/gtest/test_common.cpp index 65e3a51..a91c8ba 100644 --- a/test/gtest/test_common.cpp +++ b/test/gtest/test_common.cpp @@ -27,7 +27,7 @@ port_init(dpdk_port_t port, struct rte_mempool *mbuf_pool) socket_id = rte_eth_dev_socket_id(port); memset(&port_conf, 0, sizeof(struct rte_eth_conf)); - port_conf.rxmode.max_rx_pkt_len = ETHER_MAX_LEN; + port_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MAX_LEN; /* Configure the Ethernet device. */ retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); @@ -77,9 +77,9 @@ fill_pkt_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t l3, uint32_t l4) } int -is_ipv4_frag(const struct ipv4_hdr *iph) +is_ipv4_frag(const struct rte_ipv4_hdr *iph) { - const uint16_t mask = rte_cpu_to_be_16(~IPV4_HDR_DF_FLAG); + const uint16_t mask = rte_cpu_to_be_16(~RTE_IPV4_HDR_DF_FLAG); return ((mask & iph->fragment_offset) != 0); } @@ -88,14 +88,14 @@ void fill_ipv4_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t proto, uint32_t frag) { - const struct ipv4_hdr *iph; + const struct rte_ipv4_hdr *iph; int32_t dlen, len; dlen = rte_pktmbuf_data_len(m); - dlen -= l2 + sizeof(struct udp_hdr); + dlen -= l2 + sizeof(struct rte_udp_hdr); - iph = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, l2); - len = (iph->version_ihl & IPV4_HDR_IHL_MASK) * IPV4_IHL_MULTIPLIER; + iph = rte_pktmbuf_mtod_offset(m, const struct rte_ipv4_hdr *, l2); + len = (iph->version_ihl & RTE_IPV4_HDR_IHL_MASK) * RTE_IPV4_IHL_MULTIPLIER; if (frag != 0 && is_ipv4_frag(iph)) { m->packet_type &= ~RTE_PTYPE_L4_MASK; @@ -105,7 +105,7 @@ fill_ipv4_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t proto, if (len > dlen || (proto <= IPPROTO_MAX && iph->next_proto_id != proto)) m->packet_type = RTE_PTYPE_UNKNOWN; else - fill_pkt_hdr_len(m, l2, len, sizeof(struct udp_hdr)); + fill_pkt_hdr_len(m, l2, len, sizeof(struct rte_udp_hdr)); } int @@ -135,10 +135,10 @@ fill_ipv6x_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t nproto, const struct ip6_ext *ipx; int32_t dlen, len, ofs; - len = sizeof(struct ipv6_hdr); + len = sizeof(struct rte_ipv6_hdr); dlen = rte_pktmbuf_data_len(m); - dlen -= l2 + sizeof(struct udp_hdr); + dlen -= l2 + sizeof(struct rte_udp_hdr); ofs = l2 + len; ipx = rte_pktmbuf_mtod_offset(m, const struct ip6_ext *, ofs); @@ -179,20 +179,20 @@ fill_ipv6x_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t nproto, if ((ofs == 0 && nproto != fproto) || len > dlen) m->packet_type = RTE_PTYPE_UNKNOWN; else - fill_pkt_hdr_len(m, l2, len, sizeof(struct udp_hdr)); + fill_pkt_hdr_len(m, l2, len, sizeof(struct rte_udp_hdr)); } void fill_ipv6_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t fproto) { - const struct ipv6_hdr *iph; + const struct rte_ipv6_hdr *iph; - iph = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *, - sizeof(struct ether_hdr)); + iph = rte_pktmbuf_mtod_offset(m, const struct rte_ipv6_hdr *, + sizeof(struct rte_ether_hdr)); if (iph->proto == fproto) - fill_pkt_hdr_len(m, l2, sizeof(struct ipv6_hdr), - sizeof(struct udp_hdr)); + fill_pkt_hdr_len(m, l2, sizeof(struct rte_ipv6_hdr), + sizeof(struct rte_udp_hdr)); else if (ipv6x_hdr(iph->proto) != 0) fill_ipv6x_hdr_len(m, l2, iph->proto, fproto); } @@ -202,32 +202,32 @@ fill_eth_hdr_len(struct rte_mbuf *m) { uint32_t dlen, l2; uint16_t etp; - const struct ether_hdr *eth; + const struct rte_ether_hdr *eth; dlen = rte_pktmbuf_data_len(m); /* check that first segment is at least 42B long. */ - if (dlen < sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr) + - sizeof(struct udp_hdr)) { + if (dlen < sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + + sizeof(struct rte_udp_hdr)) { m->packet_type = RTE_PTYPE_UNKNOWN; return; } l2 = sizeof(*eth); - eth = rte_pktmbuf_mtod(m, const struct ether_hdr *); + eth = rte_pktmbuf_mtod(m, const struct rte_ether_hdr *); etp = eth->ether_type; - if (etp == rte_be_to_cpu_16(ETHER_TYPE_VLAN)) - l2 += sizeof(struct vlan_hdr); + if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_VLAN)) + l2 += sizeof(struct rte_vlan_hdr); - if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv4)) { + if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_IPV4)) { m->packet_type = RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER; fill_ipv4_hdr_len(m, l2, IPPROTO_UDP, 1); - } else if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv6) && - dlen >= l2 + sizeof(struct ipv6_hdr) + - sizeof(struct udp_hdr)) { + } else if (etp == rte_be_to_cpu_16(RTE_ETHER_TYPE_IPV6) && + dlen >= l2 + sizeof(struct rte_ipv6_hdr) + + sizeof(struct rte_udp_hdr)) { m->packet_type = RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER; diff --git a/test/gtest/test_common.h b/test/gtest/test_common.h index 9c521b8..5b01114 100644 --- a/test/gtest/test_common.h +++ b/test/gtest/test_common.h @@ -64,7 +64,7 @@ void fill_pkt_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t l3, uint32_t l4); int -is_ipv4_frag(const struct ipv4_hdr *iph); +is_ipv4_frag(const struct rte_ipv4_hdr *iph); void fill_ipv4_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t proto, diff --git a/test/gtest/test_tle_tcp_stream.h b/test/gtest/test_tle_tcp_stream.h index 2caf2b5..cb2946e 100644 --- a/test/gtest/test_tle_tcp_stream.h +++ b/test/gtest/test_tle_tcp_stream.h @@ -41,6 +41,10 @@ static struct tle_ctx_param ctx_prm_tmpl = { .socket_id = SOCKET_ID_ANY, .proto = TLE_PROTO_TCP, .max_streams = MAX_STREAMS, + . free_streams = { + .min = 0, + .max = 0, + }, .max_stream_rbufs = MAX_STREAM_RBUFS, .max_stream_sbufs = MAX_STREAM_SBUFS, }; diff --git a/test/gtest/test_tle_udp_stream_gen.h b/test/gtest/test_tle_udp_stream_gen.h index 1f3d210..8476375 100644 --- a/test/gtest/test_tle_udp_stream_gen.h +++ b/test/gtest/test_tle_udp_stream_gen.h @@ -65,8 +65,8 @@ static int lookup4_function(void *opaque, const struct in_addr *addr, struct tle_dest *res) { struct in_addr route; - struct ether_hdr *eth; - struct ipv4_hdr *ip4h; + struct rte_ether_hdr *eth; + struct rte_ipv4_hdr *ip4h; auto routes = static_cast<map<string, tle_dev *> *>(opaque); /* Check all routes added in map for a match with dest *addr */ @@ -81,11 +81,11 @@ lookup4_function(void *opaque, const struct in_addr *addr, struct tle_dest *res) res->l2_len = sizeof(*eth); res->l3_len = sizeof(*ip4h); res->head_mp = mbuf_pool; - eth = (struct ether_hdr *)res->hdr; - eth->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); - ip4h = (struct ipv4_hdr *)(eth + 1); + eth = (struct rte_ether_hdr *)res->hdr; + eth->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); + ip4h = (struct rte_ipv4_hdr *)(eth + 1); ip4h->version_ihl = (4 << 4) | - (sizeof(*ip4h) / IPV4_IHL_MULTIPLIER); + (sizeof(*ip4h) / RTE_IPV4_IHL_MULTIPLIER); ip4h->time_to_live = 64; ip4h->next_proto_id = IPPROTO_UDP; ip4h->fragment_offset = 0; @@ -101,8 +101,8 @@ static int lookup6_function(void *opaque, const struct in6_addr *addr, struct tle_dest *res) { - struct ether_hdr *eth; - struct ipv6_hdr *ip6h; + struct rte_ether_hdr *eth; + struct rte_ipv6_hdr *ip6h; struct in6_addr route; auto routes = static_cast<map<string, tle_dev *> *>(opaque); @@ -118,9 +118,9 @@ lookup6_function(void *opaque, const struct in6_addr *addr, res->l2_len = sizeof(*eth); res->l3_len = sizeof(*ip6h); res->head_mp = mbuf_pool; - eth = (struct ether_hdr *)res->hdr; - eth->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv6); - ip6h = (struct ipv6_hdr *)(eth + 1); + eth = (struct rte_ether_hdr *)res->hdr; + eth->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); + ip6h = (struct rte_ipv6_hdr *)(eth + 1); ip6h->vtc_flow = 6 << 4; ip6h->proto = IPPROTO_UDP; ip6h->hop_limits = 64; diff --git a/test/memtank/test_memtank.c b/test/memtank/test_memtank.c index 51e86be..6312339 100644 --- a/test/memtank/test_memtank.c +++ b/test/memtank/test_memtank.c @@ -57,21 +57,29 @@ struct memtank_stat { uint64_t nb_req; uint64_t nb_alloc; uint64_t nb_cycle; + uint64_t max_cycle; + uint64_t min_cycle; } alloc; struct { uint64_t nb_call; uint64_t nb_free; uint64_t nb_cycle; + uint64_t max_cycle; + uint64_t min_cycle; } free; struct { uint64_t nb_call; uint64_t nb_chunk; uint64_t nb_cycle; + uint64_t max_cycle; + uint64_t min_cycle; } grow; struct { uint64_t nb_call; uint64_t nb_chunk; uint64_t nb_cycle; + uint64_t max_cycle; + uint64_t min_cycle; } shrink; }; @@ -86,6 +94,7 @@ struct worker_args { uint32_t obj_size; uint32_t alloc_flags; uint32_t free_flags; + struct rte_ring *rng; }; struct memtank_arg { @@ -95,16 +104,16 @@ struct memtank_arg { struct worker_args worker; }; struct memtank_stat stats; -}; +} __rte_cache_aligned; #define BULK_NUM 32 -#define MAX_OBJ 0x100000 #define OBJ_SZ_MIN 1 #define OBJ_SZ_MAX 0x100000 #define OBJ_SZ_DEF (4 * RTE_CACHE_LINE_SIZE + 1) #define TEST_TIME 10 +#define CLEANUP_TIME 3 #define FREE_THRSH_MIN 0 #define FREE_THRSH_MAX 100 @@ -129,7 +138,6 @@ static uint32_t wrk_cmd __rte_cache_aligned; static struct tle_memtank_prm mtnk_prm = { .min_free = 4 * BULK_NUM, .max_free = 32 * BULK_NUM, - .max_obj = MAX_OBJ, .obj_size = OBJ_SZ_DEF, .obj_align = RTE_CACHE_LINE_SIZE, .nb_obj_chunk = BULK_NUM, @@ -247,11 +255,73 @@ memstat_dump(FILE *f, struct memstat *ms) } static void +memtank_stat_reset(struct memtank_stat *ms) +{ + static const struct memtank_stat init_stat = { + .alloc.min_cycle = UINT64_MAX, + .free.min_cycle = UINT64_MAX, + .grow.min_cycle = UINT64_MAX, + .shrink.min_cycle = UINT64_MAX, + }; + + *ms = init_stat; +} + +static void +memtank_stat_aggr(struct memtank_stat *as, const struct memtank_stat *ms) +{ + if (ms->alloc.nb_call != 0) { + as->alloc.nb_call += ms->alloc.nb_call; + as->alloc.nb_req += ms->alloc.nb_req; + as->alloc.nb_alloc += ms->alloc.nb_alloc; + as->alloc.nb_cycle += ms->alloc.nb_cycle; + as->alloc.max_cycle = RTE_MAX(as->alloc.max_cycle, + ms->alloc.max_cycle); + as->alloc.min_cycle = RTE_MIN(as->alloc.min_cycle, + ms->alloc.min_cycle); + } + if (ms->free.nb_call != 0) { + as->free.nb_call += ms->free.nb_call; + as->free.nb_free += ms->free.nb_free; + as->free.nb_cycle += ms->free.nb_cycle; + as->free.max_cycle = RTE_MAX(as->free.max_cycle, + ms->free.max_cycle); + as->free.min_cycle = RTE_MIN(as->free.min_cycle, + ms->free.min_cycle); + } + if (ms->grow.nb_call != 0) { + as->grow.nb_call += ms->grow.nb_call; + as->grow.nb_chunk += ms->grow.nb_chunk; + as->grow.nb_cycle += ms->grow.nb_cycle; + as->grow.max_cycle = RTE_MAX(as->grow.max_cycle, + ms->grow.max_cycle); + as->grow.min_cycle = RTE_MIN(as->grow.min_cycle, + ms->grow.min_cycle); + } + if (ms->shrink.nb_call != 0) { + as->shrink.nb_call += ms->shrink.nb_call; + as->shrink.nb_chunk += ms->shrink.nb_chunk; + as->shrink.nb_cycle += ms->shrink.nb_cycle; + as->shrink.max_cycle = RTE_MAX(as->shrink.max_cycle, + ms->shrink.max_cycle); + as->shrink.min_cycle = RTE_MIN(as->shrink.min_cycle, + ms->shrink.min_cycle); + } +} + +static void memtank_stat_dump(FILE *f, uint32_t lc, const struct memtank_stat *ms) { uint64_t t; + long double st; + + st = (long double)rte_get_timer_hz() / US_PER_S; + + if (lc == UINT32_MAX) + fprintf(f, "%s(AGGREGATE)={\n", __func__); + else + fprintf(f, "%s(lc=%u)={\n", __func__, lc); - fprintf(f, "%s(lc=%u)={\n", __func__, lc); fprintf(f, "\tnb_cycle=%" PRIu64 ",\n", ms->nb_cycle); if (ms->alloc.nb_call != 0) { fprintf(f, "\talloc={\n"); @@ -268,6 +338,13 @@ memtank_stat_dump(FILE *f, uint32_t lc, const struct memtank_stat *ms) fprintf(f, "\t\tobj/call(avg): %.2Lf\n", (long double)ms->alloc.nb_alloc / ms->alloc.nb_call); + fprintf(f, "\t\tmax cycles/call=%" PRIu64 "(%.2Lf usec),\n", + ms->alloc.max_cycle, + (long double)ms->alloc.max_cycle / st); + fprintf(f, "\t\tmin cycles/call=%" PRIu64 "(%.2Lf usec),\n", + ms->alloc.min_cycle, + (long double)ms->alloc.min_cycle / st); + fprintf(f, "\t},\n"); } if (ms->free.nb_call != 0) { @@ -281,6 +358,13 @@ memtank_stat_dump(FILE *f, uint32_t lc, const struct memtank_stat *ms) fprintf(f, "\t\tobj/call(avg): %.2Lf\n", (long double)ms->free.nb_free / ms->free.nb_call); + fprintf(f, "\t\tmax cycles/call=%" PRIu64 "(%.2Lf usec),\n", + ms->free.max_cycle, + (long double)ms->free.max_cycle / st); + fprintf(f, "\t\tmin cycles/call=%" PRIu64 "(%.2Lf usec),\n", + ms->free.min_cycle, + (long double)ms->free.min_cycle / st); + fprintf(f, "\t},\n"); } if (ms->grow.nb_call != 0) { @@ -294,6 +378,13 @@ memtank_stat_dump(FILE *f, uint32_t lc, const struct memtank_stat *ms) fprintf(f, "\t\tobj/call(avg): %.2Lf\n", (long double)ms->grow.nb_chunk / ms->grow.nb_call); + fprintf(f, "\t\tmax cycles/call=%" PRIu64 "(%.2Lf usec),\n", + ms->grow.max_cycle, + (long double)ms->grow.max_cycle / st); + fprintf(f, "\t\tmin cycles/call=%" PRIu64 "(%.2Lf usec),\n", + ms->grow.min_cycle, + (long double)ms->grow.min_cycle / st); + fprintf(f, "\t},\n"); } if (ms->shrink.nb_call != 0) { @@ -307,6 +398,13 @@ memtank_stat_dump(FILE *f, uint32_t lc, const struct memtank_stat *ms) fprintf(f, "\t\tobj/call(avg): %.2Lf\n", (long double)ms->shrink.nb_chunk / ms->shrink.nb_call); + fprintf(f, "\t\tmax cycles/call=%" PRIu64 "(%.2Lf usec),\n", + ms->shrink.max_cycle, + (long double)ms->shrink.max_cycle / st); + fprintf(f, "\t\tmin cycles/call=%" PRIu64 "(%.2Lf usec),\n", + ms->shrink.min_cycle, + (long double)ms->shrink.min_cycle / st); + fprintf(f, "\t},\n"); } fprintf(f, "};\n"); @@ -340,37 +438,86 @@ check_fill_objs(void *obj[], uint32_t sz, uint32_t num, } static int -test_memtank_worker(void *arg) +create_worker_ring(struct worker_args *wa, uint32_t lc) { int32_t rc; size_t sz; - uint32_t ft, lc, n, num; - uint64_t cl, tm0, tm1; - struct memtank_arg *ma; struct rte_ring *ring; - void *obj[BULK_NUM]; - - ma = arg; - lc = rte_lcore_id(); - sz = rte_ring_get_memsize(ma->worker.max_obj); - ring = alloca(sz); + sz = rte_ring_get_memsize(wa->max_obj); + ring = malloc(sz); if (ring == NULL) { printf("%s(%u): alloca(%zu) for FIFO with %u elems failed", - __func__, lc, sz, ma->worker.max_obj); + __func__, lc, sz, wa->max_obj); return -ENOMEM; } - rc = rte_ring_init(ring, "", ma->worker.max_obj, + rc = rte_ring_init(ring, "", wa->max_obj, RING_F_SP_ENQ | RING_F_SC_DEQ); if (rc != 0) { printf("%s(%u): rte_ring_init(%p, %u) failed, error: %d(%s)\n", - __func__, lc, ring, ma->worker.max_obj, + __func__, lc, ring, wa->max_obj, rc, strerror(-rc)); + free(ring); return rc; } + wa->rng = ring; + return rc; +} + +static int +test_worker_cleanup(void *arg) +{ + void *obj[BULK_NUM]; + int32_t rc; + uint32_t lc, n, num; + struct memtank_arg *ma; + struct rte_ring *ring; + + ma = arg; + ring = ma->worker.rng; + lc = rte_lcore_id(); + + rc = 0; + for (n = rte_ring_count(ring); rc == 0 && n != 0; n -= num) { + + num = rte_rand() % RTE_DIM(obj); + num = RTE_MIN(num, n); + + if (num != 0) { + /* retrieve objects to free */ + rte_ring_dequeue_bulk(ring, obj, num, NULL); + + /* check and fill contents of freeing objects */ + rc = check_fill_objs(obj, ma->worker.obj_size, num, + lc, 0); + if (rc == 0) { + tle_memtank_free(ma->mt, obj, num, + ma->worker.free_flags); + ma->stats.free.nb_free += num; + } + } + } + + return rc; +} + +static int +test_memtank_worker(void *arg) +{ + int32_t rc; + uint32_t ft, lc, n, num; + uint64_t cl, tm0, tm1; + struct memtank_arg *ma; + struct rte_ring *ring; + void *obj[BULK_NUM]; + + ma = arg; + lc = rte_lcore_id(); + /* calculate free threshold */ ft = ma->worker.max_obj * global_cfg.wrk_free_thrsh / FREE_THRSH_MAX; + ring = ma->worker.rng; while (wrk_cmd != WRK_CMD_RUN) { rte_smp_rmb(); @@ -397,11 +544,17 @@ test_memtank_worker(void *arg) if (rc != 0) break; + tm1 = tm1 - tm0; + /* collect alloc stat */ ma->stats.alloc.nb_call++; ma->stats.alloc.nb_req += num; ma->stats.alloc.nb_alloc += n; - ma->stats.alloc.nb_cycle += tm1 - tm0; + ma->stats.alloc.nb_cycle += tm1; + ma->stats.alloc.max_cycle = + RTE_MAX(ma->stats.alloc.max_cycle, tm1); + ma->stats.alloc.min_cycle = + RTE_MIN(ma->stats.alloc.min_cycle, tm1); /* store allocated objects */ rte_ring_enqueue_bulk(ring, obj, n, NULL); @@ -429,10 +582,16 @@ test_memtank_worker(void *arg) ma->worker.free_flags); tm1 = rte_rdtsc_precise(); + tm1 = tm1 - tm0; + /* collect free stat */ ma->stats.free.nb_call++; ma->stats.free.nb_free += num; - ma->stats.free.nb_cycle += tm1 - tm0; + ma->stats.free.nb_cycle += tm1; + ma->stats.free.max_cycle = + RTE_MAX(ma->stats.free.max_cycle, tm1); + ma->stats.free.min_cycle = + RTE_MIN(ma->stats.free.min_cycle, tm1); } rte_smp_mb(); @@ -463,8 +622,17 @@ test_memtank_master(void *arg) tm1 = rte_rdtsc_precise(); ma->stats.shrink.nb_call++; ma->stats.shrink.nb_chunk += n; - if (n != 0) - ma->stats.shrink.nb_cycle += tm1 - tm0; + tm1 = tm1 - tm0; + + if (n != 0) { + ma->stats.shrink.nb_cycle += tm1; + ma->stats.shrink.max_cycle = + RTE_MAX(ma->stats.shrink.max_cycle, + tm1); + ma->stats.shrink.min_cycle = + RTE_MIN(ma->stats.shrink.min_cycle, + tm1); + } } if (ma->master.flags & MASTER_FLAG_GROW) { @@ -474,8 +642,17 @@ test_memtank_master(void *arg) tm2 = rte_rdtsc_precise(); ma->stats.grow.nb_call++; ma->stats.grow.nb_chunk += n; - if (n != 0) - ma->stats.grow.nb_cycle += tm2 - tm1; + tm2 = tm2 - tm1; + + if (n != 0) { + ma->stats.grow.nb_cycle += tm2; + ma->stats.grow.max_cycle = + RTE_MAX(ma->stats.grow.max_cycle, + tm2); + ma->stats.grow.min_cycle = + RTE_MIN(ma->stats.grow.min_cycle, + tm2); + } } wrk_cmd = WRK_CMD_RUN; @@ -493,14 +670,16 @@ test_memtank_master(void *arg) return 0; } -static void +static int fill_worker_args(struct worker_args *wa, uint32_t alloc_flags, - uint32_t free_flags) + uint32_t free_flags, uint32_t lc) { wa->max_obj = global_cfg.wrk_max_obj; wa->obj_size = mtnk_prm.obj_size; wa->alloc_flags = alloc_flags; wa->free_flags = free_flags; + + return create_worker_ring(wa, lc); } static void @@ -515,91 +694,54 @@ fill_master_args(struct master_args *ma, uint32_t flags) ma->flags = flags; } -/* - * alloc/free by workers threads. - * grow/shrink by master - */ static int -test_memtank_mt1(void) +test_memtank_cleanup(struct tle_memtank *mt, struct memstat *ms, + struct memtank_arg arg[], const char *tname) { int32_t rc; uint32_t lc; - struct tle_memtank *mt; - struct tle_memtank_prm prm; - struct memstat ms; - struct memtank_arg arg[RTE_MAX_LCORE]; - - printf("%s start\n", __func__); - - memset(&prm, 0, sizeof(prm)); - memset(&ms, 0, sizeof(ms)); - - prm = mtnk_prm; - prm.alloc = test_alloc1; - prm.free = test_free1; - prm.udata = &ms; - mt = tle_memtank_create(&prm); - if (mt == NULL) { - printf("%s: memtank_create() failed\n", __func__); - return -ENOMEM; - } + printf("%s(%s)\n", __func__, tname); - memset(arg, 0, sizeof(arg)); - - /* launch on all slaves */ - RTE_LCORE_FOREACH_SLAVE(lc) { - arg[lc].mt = mt; - fill_worker_args(&arg[lc].worker, 0, 0); - rte_eal_remote_launch(test_memtank_worker, &arg[lc], lc); - } + RTE_LCORE_FOREACH_SLAVE(lc) + rte_eal_remote_launch(test_worker_cleanup, &arg[lc], lc); /* launch on master */ lc = rte_lcore_id(); - arg[lc].mt = mt; - fill_master_args(&arg[lc].master, - MASTER_FLAG_GROW | MASTER_FLAG_SHRINK); + arg[lc].master.run_cycles = CLEANUP_TIME * rte_get_timer_hz(); test_memtank_master(&arg[lc]); - /* wait for slaves and collect stats. */ - rc = 0; + ms->nb_alloc_obj = 0; RTE_LCORE_FOREACH_SLAVE(lc) { rc |= rte_eal_wait_lcore(lc); - memtank_stat_dump(stdout, lc, &arg[lc].stats); - ms.nb_alloc_obj += arg[lc].stats.alloc.nb_alloc - + ms->nb_alloc_obj += arg[lc].stats.alloc.nb_alloc - arg[lc].stats.free.nb_free; } - lc = rte_lcore_id(); - memtank_stat_dump(stdout, lc, &arg[lc].stats); tle_memtank_dump(stdout, mt, TLE_MTANK_DUMP_STAT); - memstat_dump(stdout, &ms); + memstat_dump(stdout, ms); + rc = tle_memtank_sanity_check(mt, 0); - rc |= tle_memtank_sanity_check(mt, 0); - tle_memtank_destroy(mt); return rc; } /* - * alloc/free with grow/shrink by worker threads. - * master does nothing + * alloc/free by workers threads. + * grow/shrink by master */ static int -test_memtank_mt2(void) +test_memtank_mt(const char *tname, uint32_t alloc_flags, uint32_t free_flags) { int32_t rc; uint32_t lc; struct tle_memtank *mt; struct tle_memtank_prm prm; struct memstat ms; + struct memtank_stat wrk_stats; struct memtank_arg arg[RTE_MAX_LCORE]; - const uint32_t alloc_flags = TLE_MTANK_ALLOC_CHUNK | - TLE_MTANK_ALLOC_GROW; - const uint32_t free_flags = TLE_MTANK_FREE_SHRINK; - - printf("%s start\n", __func__); + printf("%s(%s) start\n", __func__, tname); memset(&prm, 0, sizeof(prm)); memset(&ms, 0, sizeof(ms)); @@ -611,45 +753,96 @@ test_memtank_mt2(void) mt = tle_memtank_create(&prm); if (mt == NULL) { - printf("%s: memtank_create() failed\n", __func__); + printf("%s(%s): memtank_create() failed\n", __func__, tname); return -ENOMEM; } + /* dump initial memory stats */ + memstat_dump(stdout, &ms); + + rc = 0; memset(arg, 0, sizeof(arg)); - /* launch on all slaves */ + /* prepare args on all slaves */ RTE_LCORE_FOREACH_SLAVE(lc) { arg[lc].mt = mt; - fill_worker_args(&arg[lc].worker, alloc_flags, free_flags); - rte_eal_remote_launch(test_memtank_worker, &arg[lc], lc); + rc = fill_worker_args(&arg[lc].worker, alloc_flags, + free_flags, lc); + if (rc != 0) + break; + memtank_stat_reset(&arg[lc].stats); + } + + if (rc != 0) { + tle_memtank_destroy(mt); + return rc; } + /* launch on all slaves */ + RTE_LCORE_FOREACH_SLAVE(lc) + rte_eal_remote_launch(test_memtank_worker, &arg[lc], lc); + /* launch on master */ lc = rte_lcore_id(); arg[lc].mt = mt; - fill_master_args(&arg[lc].master, 0); + fill_master_args(&arg[lc].master, + MASTER_FLAG_GROW | MASTER_FLAG_SHRINK); test_memtank_master(&arg[lc]); /* wait for slaves and collect stats. */ + + memtank_stat_reset(&wrk_stats); + rc = 0; RTE_LCORE_FOREACH_SLAVE(lc) { rc |= rte_eal_wait_lcore(lc); memtank_stat_dump(stdout, lc, &arg[lc].stats); + memtank_stat_aggr(&wrk_stats, &arg[lc].stats); ms.nb_alloc_obj += arg[lc].stats.alloc.nb_alloc - arg[lc].stats.free.nb_free; } + memtank_stat_dump(stdout, UINT32_MAX, &wrk_stats); + lc = rte_lcore_id(); memtank_stat_dump(stdout, lc, &arg[lc].stats); tle_memtank_dump(stdout, mt, TLE_MTANK_DUMP_STAT); memstat_dump(stdout, &ms); - rc |= tle_memtank_sanity_check(mt, 0); + + /* run cleanup on all slave cores */ + if (rc == 0) + rc = test_memtank_cleanup(mt, &ms, arg, tname); + tle_memtank_destroy(mt); return rc; } +/* + * alloc/free by workers threads. + * grow/shrink by master + */ +static int +test_memtank_mt1(const char *tname) +{ + return test_memtank_mt(tname, 0, 0); +} + +/* + * alloc/free with grow/shrink by worker threads. + * master does nothing + */ +static int +test_memtank_mt2(const char *tname) +{ + const uint32_t alloc_flags = TLE_MTANK_ALLOC_CHUNK | + TLE_MTANK_ALLOC_GROW; + const uint32_t free_flags = TLE_MTANK_FREE_SHRINK; + + return test_memtank_mt(tname, alloc_flags, free_flags); +} + static int parse_uint_val(const char *str, uint32_t *val, uint32_t min, uint32_t max) { @@ -749,7 +942,7 @@ main(int argc, char * argv[]) const struct { const char *name; - int (*func)(void); + int (*func)(const char *); } tests[] = { { .name = "MT1-WRK_ALLOC_FREE-MST_GROW_SHRINK", @@ -774,11 +967,14 @@ main(int argc, char * argv[]) "%s: parse_op failed with error code: %d\n", __func__, rc); + /* update global values based on provided user input */ + mtnk_prm.max_obj = global_cfg.wrk_max_obj * rte_lcore_count(); + for (i = 0, k = 0; i != RTE_DIM(tests); i++) { printf("TEST %s START\n", tests[i].name); - rc = tests[i].func(); + rc = tests[i].func(tests[i].name); k += (rc == 0); if (rc != 0) |