diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/libtle_l4p/Makefile | 1 | ||||
| -rw-r--r-- | lib/libtle_l4p/ctx.c | 2 | ||||
| -rw-r--r-- | lib/libtle_l4p/misc.h | 20 | ||||
| -rw-r--r-- | lib/libtle_l4p/stream.h | 8 | ||||
| -rw-r--r-- | lib/libtle_l4p/tcp_ctl.h | 37 | ||||
| -rw-r--r-- | lib/libtle_l4p/tcp_misc.h | 22 | ||||
| -rw-r--r-- | lib/libtle_l4p/tcp_rxtx.c | 59 | ||||
| -rw-r--r-- | lib/libtle_l4p/tcp_stream.c | 173 | ||||
| -rw-r--r-- | lib/libtle_l4p/tcp_stream.h | 4 | ||||
| -rw-r--r-- | lib/libtle_l4p/tle_ctx.h | 6 | ||||
| -rw-r--r-- | lib/libtle_l4p/udp_rxtx.c | 18 | ||||
| -rw-r--r-- | lib/libtle_l4p/udp_stream.c | 1 |
12 files changed, 232 insertions, 119 deletions
diff --git a/lib/libtle_l4p/Makefile b/lib/libtle_l4p/Makefile index e1357d1..5c8407e 100644 --- a/lib/libtle_l4p/Makefile +++ b/lib/libtle_l4p/Makefile @@ -48,6 +48,7 @@ SYMLINK-y-include += tle_udp.h # this lib dependencies DEPDIRS-y += lib/libtle_misc +DEPDIRS-y += lib/libtle_memtank DEPDIRS-y += lib/libtle_dring DEPDIRS-y += lib/libtle_timer diff --git a/lib/libtle_l4p/ctx.c b/lib/libtle_l4p/ctx.c index b8067f0..b810983 100644 --- a/lib/libtle_l4p/ctx.c +++ b/lib/libtle_l4p/ctx.c @@ -116,8 +116,6 @@ tle_ctx_create(const struct tle_ctx_param *ctx_prm) for (i = 0; i != RTE_DIM(ctx->use); i++) tle_pbm_init(ctx->use + i, LPORT_START_BLK); - ctx->streams.nb_free = ctx->prm.max_streams; - /* Initialization of siphash state is done here to speed up the * fastpath processing. */ diff --git a/lib/libtle_l4p/misc.h b/lib/libtle_l4p/misc.h index 327296f..e1efb0d 100644 --- a/lib/libtle_l4p/misc.h +++ b/lib/libtle_l4p/misc.h @@ -207,7 +207,7 @@ __udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs, * The non-complemented checksum to set in the L4 header. */ static inline uint16_t -_ipv4x_phdr_cksum(const struct ipv4_hdr *ipv4_hdr, size_t ipv4h_len, +_ipv4x_phdr_cksum(const struct rte_ipv4_hdr *ipv4_hdr, size_t ipv4h_len, uint64_t ol_flags) { uint32_t s0, s1; @@ -243,7 +243,7 @@ _ipv4x_phdr_cksum(const struct ipv4_hdr *ipv4_hdr, size_t ipv4h_len, */ static inline int _ipv4_udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs, - const struct ipv4_hdr *ipv4_hdr) + const struct rte_ipv4_hdr *ipv4_hdr) { uint32_t cksum; @@ -267,7 +267,7 @@ _ipv4_udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs, */ static inline int _ipv6_udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs, - const struct ipv6_hdr *ipv6_hdr) + const struct rte_ipv6_hdr *ipv6_hdr) { uint32_t cksum; @@ -293,9 +293,9 @@ static inline int check_pkt_csum(const struct rte_mbuf *m, uint64_t ol_flags, uint32_t type, uint32_t proto) { - const struct ipv4_hdr *l3h4; - const struct ipv6_hdr *l3h6; - const struct udp_hdr *l4h; + const struct rte_ipv4_hdr *l3h4; + const struct rte_ipv6_hdr *l3h6; + const struct rte_udp_hdr *l4h; uint64_t fl3, fl4; uint16_t csum; int32_t ret; @@ -313,8 +313,10 @@ check_pkt_csum(const struct rte_mbuf *m, uint64_t ol_flags, uint32_t type, return 1; /* case 2: either ip or l4 or both cksum is unknown */ - l3h4 = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, m->l2_len); - l3h6 = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *, m->l2_len); + l3h4 = rte_pktmbuf_mtod_offset(m, const struct rte_ipv4_hdr *, + m->l2_len); + l3h6 = rte_pktmbuf_mtod_offset(m, const struct rte_ipv6_hdr *, + m->l2_len); ret = 0; if (fl3 == PKT_RX_IP_CKSUM_UNKNOWN && l3h4->hdr_checksum != 0) { @@ -329,7 +331,7 @@ check_pkt_csum(const struct rte_mbuf *m, uint64_t ol_flags, uint32_t type, * for IPv6 valid UDP cksum is mandatory. */ if (type == TLE_V4) { - l4h = (const struct udp_hdr *)((uintptr_t)l3h4 + + l4h = (const struct rte_udp_hdr *)((uintptr_t)l3h4 + m->l3_len); csum = (proto == IPPROTO_UDP && l4h->dgram_cksum == 0) ? UINT16_MAX : _ipv4_udptcp_mbuf_cksum(m, diff --git a/lib/libtle_l4p/stream.h b/lib/libtle_l4p/stream.h index 49a2809..ebefa6c 100644 --- a/lib/libtle_l4p/stream.h +++ b/lib/libtle_l4p/stream.h @@ -151,13 +151,13 @@ stream_get_dest(struct tle_stream *s, const void *dst_addr, dst->ol_flags = dev->tx.ol_flags[s->type]; if (s->type == TLE_V4) { - struct ipv4_hdr *l3h; - l3h = (struct ipv4_hdr *)(dst->hdr + dst->l2_len); + struct rte_ipv4_hdr *l3h; + l3h = (struct rte_ipv4_hdr *)(dst->hdr + dst->l2_len); l3h->src_addr = dev->prm.local_addr4.s_addr; l3h->dst_addr = d4->s_addr; } else { - struct ipv6_hdr *l3h; - l3h = (struct ipv6_hdr *)(dst->hdr + dst->l2_len); + struct rte_ipv6_hdr *l3h; + l3h = (struct rte_ipv6_hdr *)(dst->hdr + dst->l2_len); rte_memcpy(l3h->src_addr, &dev->prm.local_addr6, sizeof(l3h->src_addr)); rte_memcpy(l3h->dst_addr, d6, sizeof(l3h->dst_addr)); diff --git a/lib/libtle_l4p/tcp_ctl.h b/lib/libtle_l4p/tcp_ctl.h index bec1e76..7dde8ff 100644 --- a/lib/libtle_l4p/tcp_ctl.h +++ b/lib/libtle_l4p/tcp_ctl.h @@ -143,10 +143,10 @@ empty_lq(struct tle_tcp_stream *s) static inline void tcp_stream_reset(struct tle_ctx *ctx, struct tle_tcp_stream *s) { - struct stbl *st; uint16_t uop; + struct tcp_streams *ts; - st = CTX_TCP_STLB(ctx); + ts = CTX_TCP_STREAMS(ctx); /* reset TX armed */ rte_atomic32_set(&s->tx.arm, 0); @@ -167,7 +167,7 @@ tcp_stream_reset(struct tle_ctx *ctx, struct tle_tcp_stream *s) if (s->ste != NULL) { /* remove entry from RX streams table */ - stbl_del_stream(st, s->ste, s, + stbl_del_stream(&ts->st, s->ste, s, (s->flags & TLE_CTX_FLAG_ST) == 0); s->ste = NULL; empty_rq(s); @@ -181,7 +181,36 @@ tcp_stream_reset(struct tle_ctx *ctx, struct tle_tcp_stream *s) * if there still are pkts queued for TX, * then put this stream to the tail of free list. */ - put_stream(ctx, &s->s, TCP_STREAM_TX_FINISHED(s)); + if (TCP_STREAM_TX_PENDING(s)) + put_stream(ctx, &s->s, 0); + else { + s->s.type = TLE_VNUM; + tle_memtank_free(ts->mts, (void **)&s, 1, 0); + } +} + +static inline struct tle_tcp_stream * +tcp_stream_get(struct tle_ctx *ctx, uint32_t flag) +{ + struct tle_stream *s; + struct tle_tcp_stream *cs; + struct tcp_streams *ts; + + ts = CTX_TCP_STREAMS(ctx); + + /* check TX pending list */ + s = get_stream(ctx); + cs = TCP_STREAM(s); + if (s != NULL) { + if (TCP_STREAM_TX_FINISHED(cs)) + return cs; + put_stream(ctx, &cs->s, 0); + } + + if (tle_memtank_alloc(ts->mts, (void **)&cs, 1, flag) != 1) + return NULL; + + return cs; } #ifdef __cplusplus diff --git a/lib/libtle_l4p/tcp_misc.h b/lib/libtle_l4p/tcp_misc.h index 0cef8b2..01c1e67 100644 --- a/lib/libtle_l4p/tcp_misc.h +++ b/lib/libtle_l4p/tcp_misc.h @@ -33,10 +33,10 @@ extern "C" { #define TCP_WSCALE_DEFAULT 7 #define TCP_WSCALE_NONE 0 -#define TCP_TX_HDR_MAX (sizeof(struct tcp_hdr) + TCP_TX_OPT_LEN_MAX) +#define TCP_TX_HDR_MAX (sizeof(struct rte_tcp_hdr) + TCP_TX_OPT_LEN_MAX) /* max header size for normal data+ack packet */ -#define TCP_TX_HDR_DACK (sizeof(struct tcp_hdr) + TCP_TX_OPT_LEN_TMS) +#define TCP_TX_HDR_DACK (sizeof(struct rte_tcp_hdr) + TCP_TX_OPT_LEN_TMS) #define TCP4_MIN_MSS 536 @@ -44,10 +44,12 @@ extern "C" { /* default MTU, no TCP options. */ #define TCP4_NOP_MSS \ - (ETHER_MTU - sizeof(struct ipv4_hdr) - sizeof(struct tcp_hdr)) + (RTE_ETHER_MTU - sizeof(struct rte_ipv4_hdr) - \ + sizeof(struct rte_tcp_hdr)) #define TCP6_NOP_MSS \ - (ETHER_MTU - sizeof(struct ipv6_hdr) - sizeof(struct tcp_hdr)) + (RTE_ETHER_MTU - sizeof(struct rte_ipv6_hdr) - \ + sizeof(struct rte_tcp_hdr)) /* default MTU, TCP options present */ #define TCP4_OP_MSS (TCP4_NOP_MSS - TCP_TX_OPT_LEN_MAX) @@ -256,7 +258,7 @@ tcp_seq_min(uint32_t l, uint32_t r) } static inline void -get_seg_info(const struct tcp_hdr *th, union seg_info *si) +get_seg_info(const struct rte_tcp_hdr *th, union seg_info *si) { __m128i v; const __m128i bswap_mask = @@ -421,7 +423,7 @@ static inline void get_pkt_info(const struct rte_mbuf *m, union pkt_info *pi, union seg_info *si) { uint32_t len, type; - const struct tcp_hdr *tcph; + const struct rte_tcp_hdr *tcph; const union l4_ports *prt; const union ipv4_addrs *pa4; @@ -436,17 +438,17 @@ get_pkt_info(const struct rte_mbuf *m, union pkt_info *pi, union seg_info *si) if (type == TLE_V4) { pa4 = rte_pktmbuf_mtod_offset(m, const union ipv4_addrs *, - len + offsetof(struct ipv4_hdr, src_addr)); + len + offsetof(struct rte_ipv4_hdr, src_addr)); pi->addr4.raw = pa4->raw; } else if (type == TLE_V6) { pi->addr6 = rte_pktmbuf_mtod_offset(m, const union ipv6_addrs *, - len + offsetof(struct ipv6_hdr, src_addr)); + len + offsetof(struct rte_ipv6_hdr, src_addr)); } len += m->l3_len; - tcph = rte_pktmbuf_mtod_offset(m, const struct tcp_hdr *, len); + tcph = rte_pktmbuf_mtod_offset(m, const struct rte_tcp_hdr *, len); prt = (const union l4_ports *) - ((uintptr_t)tcph + offsetof(struct tcp_hdr, src_port)); + ((uintptr_t)tcph + offsetof(struct rte_tcp_hdr, src_port)); pi->tf.flags = tcph->tcp_flags; pi->tf.type = type; pi->csf = m->ol_flags & (PKT_RX_IP_CKSUM_MASK | PKT_RX_L4_CKSUM_MASK); diff --git a/lib/libtle_l4p/tcp_rxtx.c b/lib/libtle_l4p/tcp_rxtx.c index a519645..b1aad60 100644 --- a/lib/libtle_l4p/tcp_rxtx.c +++ b/lib/libtle_l4p/tcp_rxtx.c @@ -183,7 +183,7 @@ get_ip_pid(struct tle_dev *dev, uint32_t num, uint32_t type, uint32_t st) } static inline void -fill_tcph(struct tcp_hdr *l4h, const struct tcb *tcb, union l4_ports port, +fill_tcph(struct rte_tcp_hdr *l4h, const struct tcb *tcb, union l4_ports port, uint32_t seq, uint8_t hlen, uint8_t flags) { uint16_t wnd; @@ -217,7 +217,7 @@ tcp_fill_mbuf(struct rte_mbuf *m, const struct tle_tcp_stream *s, uint32_t pid, uint32_t swcsm) { uint32_t l4, len, plen; - struct tcp_hdr *l4h; + struct rte_tcp_hdr *l4h; char *l2h; len = dst->l2_len + dst->l3_len; @@ -239,7 +239,7 @@ tcp_fill_mbuf(struct rte_mbuf *m, const struct tle_tcp_stream *s, rte_memcpy(l2h, dst->hdr, len); /* setup TCP header & options */ - l4h = (struct tcp_hdr *)(l2h + len); + l4h = (struct rte_tcp_hdr *)(l2h + len); fill_tcph(l4h, &s->tcb, port, seq, l4, flags); /* setup mbuf TX offload related fields. */ @@ -249,8 +249,8 @@ tcp_fill_mbuf(struct rte_mbuf *m, const struct tle_tcp_stream *s, /* update proto specific fields. */ if (s->s.type == TLE_V4) { - struct ipv4_hdr *l3h; - l3h = (struct ipv4_hdr *)(l2h + dst->l2_len); + struct rte_ipv4_hdr *l3h; + l3h = (struct rte_ipv4_hdr *)(l2h + dst->l2_len); l3h->packet_id = rte_cpu_to_be_16(pid); l3h->total_length = rte_cpu_to_be_16(plen + dst->l3_len + l4); @@ -263,8 +263,8 @@ tcp_fill_mbuf(struct rte_mbuf *m, const struct tle_tcp_stream *s, if ((ol_flags & PKT_TX_IP_CKSUM) == 0 && swcsm != 0) l3h->hdr_checksum = _ipv4x_cksum(l3h, m->l3_len); } else { - struct ipv6_hdr *l3h; - l3h = (struct ipv6_hdr *)(l2h + dst->l2_len); + struct rte_ipv6_hdr *l3h; + l3h = (struct rte_ipv6_hdr *)(l2h + dst->l2_len); l3h->payload_len = rte_cpu_to_be_16(plen + l4); if ((ol_flags & PKT_TX_TCP_CKSUM) != 0) l4h->cksum = rte_ipv6_phdr_cksum(l3h, ol_flags); @@ -285,11 +285,11 @@ static inline void tcp_update_mbuf(struct rte_mbuf *m, uint32_t type, const struct tcb *tcb, uint32_t seq, uint32_t pid) { - struct tcp_hdr *l4h; + struct rte_tcp_hdr *l4h; uint32_t len; len = m->l2_len + m->l3_len; - l4h = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, len); + l4h = rte_pktmbuf_mtod_offset(m, struct rte_tcp_hdr *, len); l4h->sent_seq = rte_cpu_to_be_32(seq); l4h->recv_ack = rte_cpu_to_be_32(tcb->rcv.nxt); @@ -298,8 +298,9 @@ tcp_update_mbuf(struct rte_mbuf *m, uint32_t type, const struct tcb *tcb, fill_tms_opts(l4h + 1, tcb->snd.ts, tcb->rcv.ts); if (type == TLE_V4) { - struct ipv4_hdr *l3h; - l3h = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, m->l2_len); + struct rte_ipv4_hdr *l3h; + l3h = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, + m->l2_len); l3h->hdr_checksum = 0; l3h->packet_id = rte_cpu_to_be_16(pid); if ((m->ol_flags & PKT_TX_IP_CKSUM) == 0) @@ -312,14 +313,14 @@ tcp_update_mbuf(struct rte_mbuf *m, uint32_t type, const struct tcb *tcb, l4h->cksum = 0; if (type == TLE_V4) { - struct ipv4_hdr *l3h; - l3h = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, + struct rte_ipv4_hdr *l3h; + l3h = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, m->l2_len); l4h->cksum = _ipv4_udptcp_mbuf_cksum(m, len, l3h); } else { - struct ipv6_hdr *l3h; - l3h = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *, + struct rte_ipv6_hdr *l3h; + l3h = rte_pktmbuf_mtod_offset(m, struct rte_ipv6_hdr *, m->l2_len); l4h->cksum = _ipv6_udptcp_mbuf_cksum(m, len, l3h); } @@ -635,7 +636,7 @@ sync_ack(struct tle_tcp_stream *s, const union pkt_info *pi, struct tle_dev *dev; const void *da; struct tle_dest dst; - const struct tcp_hdr *th; + const struct rte_tcp_hdr *th; type = s->s.type; @@ -649,10 +650,14 @@ sync_ack(struct tle_tcp_stream *s, const union pkt_info *pi, if (rc < 0) return rc; - th = rte_pktmbuf_mtod_offset(m, const struct tcp_hdr *, + th = rte_pktmbuf_mtod_offset(m, const struct rte_tcp_hdr *, m->l2_len + m->l3_len); get_syn_opts(&s->tcb.so, (uintptr_t)(th + 1), m->l4_len - sizeof(*th)); + /* reset wscale option if timestamp is not present */ + if (s->tcb.so.ts.val == 0) + s->tcb.so.wscale = 0; + s->tcb.rcv.nxt = si->seq + 1; seq = sync_gen_seq(pi, s->tcb.rcv.nxt, ts, s->tcb.so.mss, s->s.ctx->prm.hash_alg, @@ -710,7 +715,7 @@ rx_tms_opt(const struct tcb *tcb, const struct rte_mbuf *mb) { union tsopt ts; uintptr_t opt; - const struct tcp_hdr *th; + const struct rte_tcp_hdr *th; if (tcb->so.ts.val != 0) { opt = rte_pktmbuf_mtod_offset(mb, uintptr_t, @@ -782,7 +787,7 @@ restore_syn_opt(union seg_info *si, union tsopt *to, { int32_t rc; uint32_t len; - const struct tcp_hdr *th; + const struct rte_tcp_hdr *th; /* check that ACK, etc fields are what we expected. */ rc = sync_check_ack(pi, si->seq, si->ack - 1, ts, @@ -793,7 +798,7 @@ restore_syn_opt(union seg_info *si, union tsopt *to, si->mss = rc; - th = rte_pktmbuf_mtod_offset(mb, const struct tcp_hdr *, + th = rte_pktmbuf_mtod_offset(mb, const struct rte_tcp_hdr *, mb->l2_len + mb->l3_len); len = mb->l4_len - sizeof(*th); to[0] = get_tms_opts((uintptr_t)(th + 1), len); @@ -947,15 +952,15 @@ rx_ack_listen(struct tle_tcp_stream *s, struct stbl *st, return rc; /* allocate new stream */ - ts = get_stream(ctx); - cs = TCP_STREAM(ts); - if (ts == NULL) + cs = tcp_stream_get(ctx, 0); + if (cs == NULL) return ENFILE; /* prepare stream to handle new connection */ if (accept_prep_stream(s, st, cs, &to, tms, pi, si) == 0) { /* put new stream in the accept queue */ + ts = &cs->s; if (_rte_ring_enqueue_burst(s->rx.q, (void * const *)&ts, 1) == 1) { *csp = cs; @@ -1549,7 +1554,7 @@ rx_synack(struct tle_tcp_stream *s, uint32_t ts, uint32_t state, struct resp_info *rsp) { struct syn_opts so; - struct tcp_hdr *th; + struct rte_tcp_hdr *th; if (state != TCP_ST_SYN_SENT) return -EINVAL; @@ -1566,7 +1571,7 @@ rx_synack(struct tle_tcp_stream *s, uint32_t ts, uint32_t state, return 0; } - th = rte_pktmbuf_mtod_offset(mb, struct tcp_hdr *, + th = rte_pktmbuf_mtod_offset(mb, struct rte_tcp_hdr *, mb->l2_len + mb->l3_len); get_syn_opts(&so, (uintptr_t)(th + 1), mb->l4_len - sizeof(*th)); @@ -1951,12 +1956,15 @@ tle_tcp_stream_accept(struct tle_stream *ts, struct tle_stream *rs[], { uint32_t n; struct tle_tcp_stream *s; + struct tle_memtank *mts; s = TCP_STREAM(ts); n = _rte_ring_dequeue_burst(s->rx.q, (void **)rs, num); if (n == 0) return 0; + mts = CTX_TCP_MTS(ts->ctx); + /* * if we still have packets to read, * then rearm stream RX event. @@ -1967,6 +1975,7 @@ tle_tcp_stream_accept(struct tle_stream *ts, struct tle_stream *rs[], tcp_stream_release(s); } + tle_memtank_grow(mts); return n; } diff --git a/lib/libtle_l4p/tcp_stream.c b/lib/libtle_l4p/tcp_stream.c index a212405..fce3b9a 100644 --- a/lib/libtle_l4p/tcp_stream.c +++ b/lib/libtle_l4p/tcp_stream.c @@ -28,6 +28,8 @@ #include "tcp_ofo.h" #include "tcp_txq.h" +#define MAX_STREAM_BURST 0x40 + static void unuse_stream(struct tle_tcp_stream *s) { @@ -42,11 +44,13 @@ tcp_fini_streams(struct tle_ctx *ctx) ts = CTX_TCP_STREAMS(ctx); if (ts != NULL) { - stbl_fini(&ts->st); - /* free the timer wheel */ + stbl_fini(&ts->st); tle_timer_free(ts->tmr); rte_free(ts->tsq); + tle_memtank_dump(stdout, ts->mts, TLE_MTANK_DUMP_STAT); + tle_memtank_sanity_check(ts->mts, 0); + tle_memtank_destroy(ts->mts); STAILQ_INIT(&ts->dr.fe); STAILQ_INIT(&ts->dr.be); @@ -122,7 +126,7 @@ calc_stream_szofs(struct tle_ctx *ctx, struct stream_szofs *szofs) szofs->size = sz; } -static int +static void init_stream(struct tle_ctx *ctx, struct tle_tcp_stream *s, const struct stream_szofs *szofs) { @@ -163,9 +167,6 @@ init_stream(struct tle_ctx *ctx, struct tle_tcp_stream *s, s->s.ctx = ctx; unuse_stream(s); - STAILQ_INSERT_TAIL(&ctx->streams.free, &s->s, link); - - return 0; } static void @@ -178,38 +179,107 @@ tcp_free_drbs(struct tle_stream *s, struct tle_drb *drb[], uint32_t nb_drb) } static struct tle_timer_wheel * -alloc_timers(uint32_t num, uint32_t mshift, int32_t socket) +alloc_timers(const struct tle_ctx *ctx) { + struct tle_timer_wheel *twl; struct tle_timer_wheel_args twprm; twprm.tick_size = TCP_RTO_GRANULARITY; - twprm.max_timer = num; - twprm.socket_id = socket; - return tle_timer_create(&twprm, tcp_get_tms(mshift)); + twprm.max_timer = ctx->prm.max_streams; + twprm.socket_id = ctx->prm.socket_id; + + twl = tle_timer_create(&twprm, tcp_get_tms(ctx->cycles_ms_shift)); + if (twl == NULL) + TCP_LOG(ERR, "alloc_timers(ctx=%p) failed with error=%d\n", + ctx, rte_errno); + return twl; +} + +static void * +mts_alloc(size_t sz, void *udata) +{ + struct tle_ctx *ctx; + + ctx = udata; + return rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE, + ctx->prm.socket_id); +} + +static void +mts_free(void *p, void *udata) +{ + RTE_SET_USED(udata); + rte_free(p); +} + +static void +mts_init(void *pa[], uint32_t num, void *udata) +{ + uint32_t i; + struct tle_ctx *ctx; + struct tcp_streams *ts; + + ctx = udata; + ts = CTX_TCP_STREAMS(ctx); + + for (i = 0; i != num; i++) + init_stream(ctx, pa[i], &ts->szofs); +} + +static struct tle_memtank * +alloc_mts(struct tle_ctx *ctx, uint32_t stream_size) +{ + struct tle_memtank *mts; + struct tle_memtank_prm prm; + + static const struct tle_memtank_prm cprm = { + .obj_align = RTE_CACHE_LINE_SIZE, + .flags = TLE_MTANK_OBJ_DBG, + .alloc = mts_alloc, + .free = mts_free, + .init = mts_init, + }; + + prm = cprm; + prm.udata = ctx; + + prm.obj_size = stream_size; + + prm.min_free = (ctx->prm.free_streams.min != 0) ? + ctx->prm.free_streams.min : ctx->prm.max_streams; + prm.max_free = (ctx->prm.free_streams.max > prm.min_free) ? + ctx->prm.free_streams.max : prm.min_free; + + prm.nb_obj_chunk = MAX_STREAM_BURST; + prm.max_obj = ctx->prm.max_streams; + + mts = tle_memtank_create(&prm); + if (mts == NULL) + TCP_LOG(ERR, "%s(ctx=%p) failed with error=%d\n", + __func__, ctx, rte_errno); + else + tle_memtank_grow(mts); + + return mts; } static int tcp_init_streams(struct tle_ctx *ctx) { - size_t sz; - uint32_t f, i; + uint32_t f; int32_t rc; struct tcp_streams *ts; - struct tle_tcp_stream *ps; struct stream_szofs szofs; f = ((ctx->prm.flags & TLE_CTX_FLAG_ST) == 0) ? 0 : (RING_F_SP_ENQ | RING_F_SC_DEQ); calc_stream_szofs(ctx, &szofs); + TCP_LOG(NOTICE, "ctx:%p, caluclated stream size: %u\n", + ctx, szofs.size); - sz = sizeof(*ts) + szofs.size * ctx->prm.max_streams; - ts = rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE, + ts = rte_zmalloc_socket(NULL, sizeof(*ts), RTE_CACHE_LINE_SIZE, ctx->prm.socket_id); - - TCP_LOG(NOTICE, "allocation of %zu bytes on socket %d " - "for %u tcp_streams returns %p\n", - sz, ctx->prm.socket_id, ctx->prm.max_streams, ts); if (ts == NULL) return -ENOMEM; @@ -221,29 +291,22 @@ tcp_init_streams(struct tle_ctx *ctx) ctx->streams.buf = ts; STAILQ_INIT(&ctx->streams.free); - ts->tmr = alloc_timers(ctx->prm.max_streams, ctx->cycles_ms_shift, - ctx->prm.socket_id); - if (ts->tmr == NULL) { - TCP_LOG(ERR, "alloc_timers(ctx=%p) failed with error=%d\n", - ctx, rte_errno); - rc = -ENOMEM; - } else { - ts->tsq = alloc_ring(ctx->prm.max_streams, - f | RING_F_SC_DEQ, ctx->prm.socket_id); - if (ts->tsq == NULL) + rc = stbl_init(&ts->st, ctx->prm.max_streams, ctx->prm.socket_id); + + if (rc == 0) { + ts->tsq = alloc_ring(ctx->prm.max_streams, f | RING_F_SC_DEQ, + ctx->prm.socket_id); + ts->tmr = alloc_timers(ctx); + ts->mts = alloc_mts(ctx, szofs.size); + + if (ts->tsq == NULL || ts->tmr == NULL || ts->mts == NULL) rc = -ENOMEM; - else - rc = stbl_init(&ts->st, ctx->prm.max_streams, - ctx->prm.socket_id); - } - for (i = 0; rc == 0 && i != ctx->prm.max_streams; i++) { - ps = (void *)((uintptr_t)ts->s + i * ts->szofs.size); - rc = init_stream(ctx, ps, &ts->szofs); + tle_memtank_dump(stdout, ts->mts, TLE_MTANK_DUMP_STAT); } if (rc != 0) { - TCP_LOG(ERR, "initalisation of %u-th stream failed", i); + TCP_LOG(ERR, "initalisation of tcp streams failed"); tcp_fini_streams(ctx); } @@ -302,6 +365,7 @@ struct tle_stream * tle_tcp_stream_open(struct tle_ctx *ctx, const struct tle_tcp_stream_param *prm) { + struct tcp_streams *ts; struct tle_tcp_stream *s; int32_t rc; @@ -310,15 +374,11 @@ tle_tcp_stream_open(struct tle_ctx *ctx, return NULL; } - s = (struct tle_tcp_stream *)get_stream(ctx); - if (s == NULL) { - rte_errno = ENFILE; - return NULL; + ts = CTX_TCP_STREAMS(ctx); - /* some TX still pending for that stream. */ - } else if (TCP_STREAM_TX_PENDING(s)) { - put_stream(ctx, &s->s, 0); - rte_errno = EAGAIN; + s = tcp_stream_get(ctx, TLE_MTANK_ALLOC_CHUNK | TLE_MTANK_ALLOC_GROW); + if (s == NULL) { + rte_errno = ENFILE; return NULL; } @@ -328,7 +388,8 @@ tle_tcp_stream_open(struct tle_ctx *ctx, (const struct sockaddr *)&prm->addr.remote); if (rc != 0) { - put_stream(ctx, &s->s, 1); + tle_memtank_free(ts->mts, (void **)&s, 1, + TLE_MTANK_FREE_SHRINK); rte_errno = rc; return NULL; } @@ -426,18 +487,17 @@ tle_tcp_stream_close_bulk(struct tle_stream *ts[], uint32_t num) rc = 0; - for (i = 0; i != num; i++) { + for (i = 0; i != num && rc == 0; i++) { s = TCP_STREAM(ts[i]); - if (ts[i] == NULL || s->s.type >= TLE_VNUM) { + if (ts[i] == NULL || s->s.type >= TLE_VNUM) rc = EINVAL; - break; - } - ctx = s->s.ctx; - rc = stream_close(ctx, s); - if (rc != 0) - break; + else { + ctx = s->s.ctx; + rc = stream_close(ctx, s); + tle_memtank_shrink(CTX_TCP_MTS(ctx)); + } } if (rc != 0) @@ -448,6 +508,7 @@ tle_tcp_stream_close_bulk(struct tle_stream *ts[], uint32_t num) int tle_tcp_stream_close(struct tle_stream *ts) { + int32_t rc; struct tle_ctx *ctx; struct tle_tcp_stream *s; @@ -456,7 +517,9 @@ tle_tcp_stream_close(struct tle_stream *ts) return -EINVAL; ctx = s->s.ctx; - return stream_close(ctx, s); + rc = stream_close(ctx, s); + tle_memtank_shrink(CTX_TCP_MTS(ctx)); + return rc; } int diff --git a/lib/libtle_l4p/tcp_stream.h b/lib/libtle_l4p/tcp_stream.h index 1bb2a42..a3d00dc 100644 --- a/lib/libtle_l4p/tcp_stream.h +++ b/lib/libtle_l4p/tcp_stream.h @@ -18,6 +18,7 @@ #include <rte_vect.h> #include <tle_dring.h> +#include <tle_memtank.h> #include <tle_tcp.h> #include <tle_event.h> @@ -176,9 +177,9 @@ struct tcp_streams { struct stbl st; struct tle_timer_wheel *tmr; /* timer wheel */ struct rte_ring *tsq; /* to-send streams queue */ + struct tle_memtank *mts; /* memtank to allocate streams from */ struct sdr dr; /* death row for zombie streams */ struct stream_szofs szofs; /* size and offsets for stream data */ - struct tle_tcp_stream s[]; /* array of allocated streams. */ }; #define CTX_TCP_STREAMS(ctx) ((struct tcp_streams *)(ctx)->streams.buf) @@ -186,6 +187,7 @@ struct tcp_streams { #define CTX_TCP_TMWHL(ctx) (CTX_TCP_STREAMS(ctx)->tmr) #define CTX_TCP_TSQ(ctx) (CTX_TCP_STREAMS(ctx)->tsq) #define CTX_TCP_SDR(ctx) (&CTX_TCP_STREAMS(ctx)->dr) +#define CTX_TCP_MTS(ctx) (CTX_TCP_STREAMS(ctx)->mts) #ifdef __cplusplus } diff --git a/lib/libtle_l4p/tle_ctx.h b/lib/libtle_l4p/tle_ctx.h index de78a6b..391cfe3 100644 --- a/lib/libtle_l4p/tle_ctx.h +++ b/lib/libtle_l4p/tle_ctx.h @@ -112,6 +112,12 @@ struct tle_ctx_param { int32_t socket_id; /**< socket ID to allocate memory for. */ uint32_t proto; /**< L4 proto to handle. */ uint32_t max_streams; /**< max number of streams in context. */ + struct { + uint32_t min; + /**< min number of free streams (grow threshold). */ + uint32_t max; + /**< max number of free streams (shrink threshold). */ + } free_streams; uint32_t max_stream_rbufs; /**< max recv mbufs per stream. */ uint32_t max_stream_sbufs; /**< max send mbufs per stream. */ uint32_t send_bulk_size; /**< expected # of packets per send call. */ diff --git a/lib/libtle_l4p/udp_rxtx.c b/lib/libtle_l4p/udp_rxtx.c index 84a13ea..8963df5 100644 --- a/lib/libtle_l4p/udp_rxtx.c +++ b/lib/libtle_l4p/udp_rxtx.c @@ -69,16 +69,16 @@ pkt_info(struct rte_mbuf *m, union l4_ports *ports, union ipv4_addrs *addr4, len = m->l2_len; if (ret.src == TLE_V4) { pa4 = rte_pktmbuf_mtod_offset(m, union ipv4_addrs *, - len + offsetof(struct ipv4_hdr, src_addr)); + len + offsetof(struct rte_ipv4_hdr, src_addr)); addr4->raw = pa4->raw; } else if (ret.src == TLE_V6) { *addr6 = rte_pktmbuf_mtod_offset(m, union ipv6_addrs *, - len + offsetof(struct ipv6_hdr, src_addr)); + len + offsetof(struct rte_ipv6_hdr, src_addr)); } len += m->l3_len; up = rte_pktmbuf_mtod_offset(m, union l4_ports *, - len + offsetof(struct udp_hdr, src_port)); + len + offsetof(struct rte_udp_hdr, src_port)); ports->raw = up->raw; ret.dst = ports->dst; return ret; @@ -355,8 +355,8 @@ udp_fill_mbuf(struct rte_mbuf *m, /* update proto specific fields. */ if (type == TLE_V4) { - struct ipv4_hdr *l3h; - l3h = (struct ipv4_hdr *)(l2h + dst->l2_len); + struct rte_ipv4_hdr *l3h; + l3h = (struct rte_ipv4_hdr *)(l2h + dst->l2_len); l3h->packet_id = rte_cpu_to_be_16(pid); l3h->total_length = rte_cpu_to_be_16(plen + dst->l3_len + sizeof(*l4h)); @@ -370,8 +370,8 @@ udp_fill_mbuf(struct rte_mbuf *m, if ((ol_flags & PKT_TX_IP_CKSUM) == 0) l3h->hdr_checksum = _ipv4x_cksum(l3h, m->l3_len); } else { - struct ipv6_hdr *l3h; - l3h = (struct ipv6_hdr *)(l2h + dst->l2_len); + struct rte_ipv6_hdr *l3h; + l3h = (struct rte_ipv6_hdr *)(l2h + dst->l2_len); l3h->payload_len = rte_cpu_to_be_16(plen + sizeof(*l4h)); if ((ol_flags & PKT_TX_UDP_CKSUM) != 0) l4h->cksum = rte_ipv6_phdr_cksum(l3h, ol_flags); @@ -389,13 +389,13 @@ udp_fill_mbuf(struct rte_mbuf *m, static inline void frag_fixup(const struct rte_mbuf *ms, struct rte_mbuf *mf, uint32_t type) { - struct ipv4_hdr *l3h; + struct rte_ipv4_hdr *l3h; mf->ol_flags = ms->ol_flags; mf->tx_offload = ms->tx_offload; if (type == TLE_V4 && (ms->ol_flags & PKT_TX_IP_CKSUM) == 0) { - l3h = rte_pktmbuf_mtod(mf, struct ipv4_hdr *); + l3h = rte_pktmbuf_mtod(mf, struct rte_ipv4_hdr *); l3h->hdr_checksum = _ipv4x_cksum(l3h, mf->l3_len); } } diff --git a/lib/libtle_l4p/udp_stream.c b/lib/libtle_l4p/udp_stream.c index 29f5a40..a50c420 100644 --- a/lib/libtle_l4p/udp_stream.c +++ b/lib/libtle_l4p/udp_stream.c @@ -170,6 +170,7 @@ udp_init_streams(struct tle_ctx *ctx) } } + ctx->streams.nb_free = ctx->prm.max_streams; return 0; } |