From 47eb00f25ab06a699dc27507814c7656940340af Mon Sep 17 00:00:00 2001 From: Konstantin Ananyev Date: Fri, 25 Oct 2019 11:56:34 +0100 Subject: v6 rework TCP stream allocation Allocate TCP stream and all necessary metadata (RX/TX queues, OFO queue, DRBs, etc.) as one big buffer, instead of separate alloc() calls for each of the sub-components. Signed-off-by: Konstantin Ananyev Change-Id: Idc9f3e9329920dfb34916f9bff28664ee5e99a42 --- lib/libtle_l4p/tcp_ofo.c | 39 +++++------- lib/libtle_l4p/tcp_ofo.h | 6 +- lib/libtle_l4p/tcp_stream.c | 141 +++++++++++++++++++++++--------------------- lib/libtle_l4p/tcp_stream.h | 23 ++++++++ 4 files changed, 115 insertions(+), 94 deletions(-) diff --git a/lib/libtle_l4p/tcp_ofo.c b/lib/libtle_l4p/tcp_ofo.c index 1565445..8724a0a 100644 --- a/lib/libtle_l4p/tcp_ofo.c +++ b/lib/libtle_l4p/tcp_ofo.c @@ -27,14 +27,20 @@ #define OFO_OBJ_MAX (OFODB_OBJ_MAX * OFO_DB_MAX) -void -tcp_ofo_free(struct ofo *ofo) +static uint32_t +calc_ofo_size(uint32_t nobj, uint32_t ndb) { - rte_free(ofo); + uint32_t dsz, osz, sz; + const struct ofo *ofo = NULL; + + osz = sizeof(*ofo) + sizeof(ofo->db[0]) * ndb; + dsz = sizeof(ofo->db[0].obj[0]) * nobj * ndb; + sz = osz + dsz; + return sz; } -static void -calc_ofo_elems(uint32_t nbufs, uint32_t *nobj, uint32_t *ndb) +void +tcp_ofo_calc_elems(uint32_t nbufs, uint32_t *nobj, uint32_t *ndb, uint32_t *sz) { uint32_t n, nd, no; @@ -50,29 +56,15 @@ calc_ofo_elems(uint32_t nbufs, uint32_t *nobj, uint32_t *ndb) *nobj = no; *ndb = nd; + *sz = calc_ofo_size(no, nd); } -struct ofo * -tcp_ofo_alloc(uint32_t nbufs, int32_t socket) +void +tcp_ofo_init(struct ofo *ofo, uint32_t nobj, uint32_t ndb) { - uint32_t i, ndb, nobj; - size_t dsz, osz, sz; - struct ofo *ofo; + uint32_t i; struct rte_mbuf **obj; - calc_ofo_elems(nbufs, &nobj, &ndb); - osz = sizeof(*ofo) + sizeof(ofo->db[0]) * ndb; - dsz = sizeof(ofo->db[0].obj[0]) * nobj * ndb; - sz = osz + dsz; - - ofo = rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE, socket); - if (ofo == NULL) { - TCP_LOG(ERR, "%s: allocation of %zu bytes on socket %d " - "failed with error code: %d\n", - __func__, sz, socket, rte_errno); - return NULL; - } - obj = (struct rte_mbuf **)&ofo->db[ndb]; for (i = 0; i != ndb; i++) { ofo->db[i].nb_max = nobj; @@ -80,6 +72,5 @@ tcp_ofo_alloc(uint32_t nbufs, int32_t socket) } ofo->nb_max = ndb; - return ofo; } diff --git a/lib/libtle_l4p/tcp_ofo.h b/lib/libtle_l4p/tcp_ofo.h index 9d88266..6cb5e3f 100644 --- a/lib/libtle_l4p/tcp_ofo.h +++ b/lib/libtle_l4p/tcp_ofo.h @@ -289,11 +289,11 @@ _ofodb_enqueue(struct rte_ring *r, const struct ofodb *db, uint32_t *seq) return num - n; } -struct ofo * -tcp_ofo_alloc(uint32_t nbufs, int32_t socket); +void +tcp_ofo_calc_elems(uint32_t nbufs, uint32_t *nobj, uint32_t *ndb, uint32_t *sz); void -tcp_ofo_free(struct ofo *ofo); +tcp_ofo_init(struct ofo *ofo, uint32_t nobj, uint32_t ndb); #ifdef __cplusplus } diff --git a/lib/libtle_l4p/tcp_stream.c b/lib/libtle_l4p/tcp_stream.c index 676521b..a212405 100644 --- a/lib/libtle_l4p/tcp_stream.c +++ b/lib/libtle_l4p/tcp_stream.c @@ -35,28 +35,14 @@ unuse_stream(struct tle_tcp_stream *s) rte_atomic32_set(&s->use, INT32_MIN); } -static void -fini_stream(struct tle_tcp_stream *s) -{ - if (s != NULL) { - rte_free(s->rx.q); - tcp_ofo_free(s->rx.ofo); - rte_free(s->tx.q); - rte_free(s->tx.drb.r); - } -} - static void tcp_fini_streams(struct tle_ctx *ctx) { - uint32_t i; struct tcp_streams *ts; ts = CTX_TCP_STREAMS(ctx); if (ts != NULL) { stbl_fini(&ts->st); - for (i = 0; i != ctx->prm.max_streams; i++) - fini_stream(&ts->s[i]); /* free the timer wheel */ tle_timer_free(ts->tmr); @@ -94,71 +80,84 @@ alloc_ring(uint32_t n, uint32_t flags, int32_t socket) return r; } -static int -init_stream(struct tle_ctx *ctx, struct tle_tcp_stream *s) +static void +calc_stream_szofs(struct tle_ctx *ctx, struct stream_szofs *szofs) { - size_t bsz, rsz, sz; - uint32_t f, i, k, n, nb; - struct tle_drb *drb; - char name[RTE_RING_NAMESIZE]; - - f = ((ctx->prm.flags & TLE_CTX_FLAG_ST) == 0) ? 0 : - (RING_F_SP_ENQ | RING_F_SC_DEQ); + uint32_t n, na, sz, tsz; - /* init RX part. */ + sz = sizeof(struct tle_tcp_stream); n = RTE_MAX(ctx->prm.max_stream_rbufs, 1U); - s->rx.q = alloc_ring(n, f | RING_F_SP_ENQ, ctx->prm.socket_id); - if (s->rx.q == NULL) - return -ENOMEM; + tcp_ofo_calc_elems(n, &szofs->ofo.nb_obj, &szofs->ofo.nb_max, &tsz); + szofs->ofo.ofs = sz; - s->rx.ofo = tcp_ofo_alloc(n, ctx->prm.socket_id); - if (s->rx.ofo == NULL) - return -ENOMEM; + sz += tsz; + sz = RTE_ALIGN_CEIL(sz, RTE_CACHE_LINE_SIZE); - /* init TX part. */ + na = rte_align32pow2(n); + szofs->rxq.ofs = sz; + szofs->rxq.nb_obj = na; + + sz += rte_ring_get_memsize(na); + sz = RTE_ALIGN_CEIL(sz, RTE_CACHE_LINE_SIZE); n = RTE_MAX(ctx->prm.max_stream_sbufs, 1U); - s->tx.q = alloc_ring(n, f | RING_F_SC_DEQ, ctx->prm.socket_id); - if (s->tx.q == NULL) - return -ENOMEM; + na = rte_align32pow2(n); + szofs->txq.ofs = sz; + szofs->txq.nb_obj = na; - nb = drb_nb_elem(ctx); - k = calc_stream_drb_num(ctx, nb); - n = rte_align32pow2(k); + sz += rte_ring_get_memsize(na); + sz = RTE_ALIGN_CEIL(sz, RTE_CACHE_LINE_SIZE); - /* size of the drbs ring */ - rsz = rte_ring_get_memsize(n); - rsz = RTE_ALIGN_CEIL(rsz, RTE_CACHE_LINE_SIZE); + szofs->drb.nb_obj = drb_nb_elem(ctx); + szofs->drb.nb_max = calc_stream_drb_num(ctx, szofs->drb.nb_obj); + szofs->drb.nb_rng = rte_align32pow2(szofs->drb.nb_max); + szofs->drb.rng_sz = rte_ring_get_memsize(szofs->drb.nb_rng); + szofs->drb.blk_sz = tle_drb_calc_size(szofs->drb.nb_obj); + szofs->drb.ofs = sz; - /* size of the drb. */ - bsz = tle_drb_calc_size(nb); + sz += szofs->drb.rng_sz + szofs->drb.blk_sz * szofs->drb.nb_max; + sz = RTE_ALIGN_CEIL(sz, RTE_CACHE_LINE_SIZE); - /* total stream drbs size. */ - sz = rsz + bsz * k; + szofs->size = sz; +} - s->tx.drb.r = rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE, - ctx->prm.socket_id); - if (s->tx.drb.r == NULL) { - TCP_LOG(ERR, "%s(%p): allocation of %zu bytes on socket %d " - "failed with error code: %d\n", - __func__, s, sz, ctx->prm.socket_id, rte_errno); - return -ENOMEM; - } +static int +init_stream(struct tle_ctx *ctx, struct tle_tcp_stream *s, + const struct stream_szofs *szofs) +{ + uint32_t f, i; + struct tle_drb *drb; + + f = ((ctx->prm.flags & TLE_CTX_FLAG_ST) == 0) ? 0 : + (RING_F_SP_ENQ | RING_F_SC_DEQ); + + /* init RX part. */ - snprintf(name, sizeof(name), "%p@%zu", s, sz); - rte_ring_init(s->tx.drb.r, name, n, f); + s->rx.ofo = (void *)((uintptr_t)s + szofs->ofo.ofs); + tcp_ofo_init(s->rx.ofo, szofs->ofo.nb_obj, szofs->ofo.nb_max); - for (i = 0; i != k; i++) { + s->rx.q = (void *)((uintptr_t)s + szofs->rxq.ofs); + rte_ring_init(s->rx.q, __func__, szofs->rxq.nb_obj, f | RING_F_SP_ENQ); + + /* init TX part. */ + + s->tx.q = (void *)((uintptr_t)s + szofs->txq.ofs); + rte_ring_init(s->tx.q, __func__, szofs->txq.nb_obj, f | RING_F_SC_DEQ); + + s->tx.drb.r = (void *)((uintptr_t)s + szofs->drb.ofs); + rte_ring_init(s->tx.drb.r, __func__, szofs->drb.nb_rng, f); + + for (i = 0; i != szofs->drb.nb_max; i++) { drb = (struct tle_drb *)((uintptr_t)s->tx.drb.r + - rsz + bsz * i); + szofs->drb.rng_sz + szofs->drb.blk_sz * i); drb->udata = s; - drb->size = nb; + drb->size = szofs->drb.nb_obj; rte_ring_enqueue(s->tx.drb.r, drb); } - s->tx.drb.nb_elem = nb; - s->tx.drb.nb_max = k; + s->tx.drb.nb_elem = szofs->drb.nb_obj; + s->tx.drb.nb_max = szofs->drb.nb_max; /* mark stream as avaialble to use. */ @@ -196,19 +195,25 @@ tcp_init_streams(struct tle_ctx *ctx) uint32_t f, i; int32_t rc; struct tcp_streams *ts; + struct tle_tcp_stream *ps; + struct stream_szofs szofs; f = ((ctx->prm.flags & TLE_CTX_FLAG_ST) == 0) ? 0 : (RING_F_SP_ENQ | RING_F_SC_DEQ); - sz = sizeof(*ts) + sizeof(ts->s[0]) * ctx->prm.max_streams; + calc_stream_szofs(ctx, &szofs); + + sz = sizeof(*ts) + szofs.size * ctx->prm.max_streams; ts = rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE, ctx->prm.socket_id); - if (ts == NULL) { - TCP_LOG(ERR, "allocation of %zu bytes on socket %d " - "for %u tcp_streams failed\n", - sz, ctx->prm.socket_id, ctx->prm.max_streams); + + TCP_LOG(NOTICE, "allocation of %zu bytes on socket %d " + "for %u tcp_streams returns %p\n", + sz, ctx->prm.socket_id, ctx->prm.max_streams, ts); + if (ts == NULL) return -ENOMEM; - } + + ts->szofs = szofs; STAILQ_INIT(&ts->dr.fe); STAILQ_INIT(&ts->dr.be); @@ -232,8 +237,10 @@ tcp_init_streams(struct tle_ctx *ctx) ctx->prm.socket_id); } - for (i = 0; rc == 0 && i != ctx->prm.max_streams; i++) - rc = init_stream(ctx, &ts->s[i]); + for (i = 0; rc == 0 && i != ctx->prm.max_streams; i++) { + ps = (void *)((uintptr_t)ts->s + i * ts->szofs.size); + rc = init_stream(ctx, ps, &ts->szofs); + } if (rc != 0) { TCP_LOG(ERR, "initalisation of %u-th stream failed", i); diff --git a/lib/libtle_l4p/tcp_stream.h b/lib/libtle_l4p/tcp_stream.h index 4629fe6..1bb2a42 100644 --- a/lib/libtle_l4p/tcp_stream.h +++ b/lib/libtle_l4p/tcp_stream.h @@ -150,11 +150,34 @@ struct sdr { STAILQ_HEAD(, tle_stream) be; }; +/* tempalte sizes/offsets/etc. for tcp stream */ +struct stream_szofs { + uint32_t size; + struct { + uint32_t ofs; + uint32_t nb_obj; + uint32_t nb_max; + } ofo; + struct { + uint32_t ofs; + uint32_t nb_obj; + } rxq, txq; + struct { + uint32_t ofs; + uint32_t blk_sz; + uint32_t rng_sz; + uint32_t nb_rng; + uint32_t nb_obj; + uint32_t nb_max; + } drb; +}; + struct tcp_streams { struct stbl st; struct tle_timer_wheel *tmr; /* timer wheel */ struct rte_ring *tsq; /* to-send streams queue */ struct sdr dr; /* death row for zombie streams */ + struct stream_szofs szofs; /* size and offsets for stream data */ struct tle_tcp_stream s[]; /* array of allocated streams. */ }; -- cgit 1.2.3-korg