From d49f3784270f142825fa4bd4ae96730401a2b871 Mon Sep 17 00:00:00 2001 From: Ido Barnea Date: Mon, 20 Jun 2016 10:58:10 +0300 Subject: better support for latency with field engine + some order with mbuf functions --- src/main_dpdk.cpp | 8 ++- src/pal/common/common_mbuf.cpp | 46 +++++++++++++++ src/pal/common/common_mbuf.h | 88 +++++++++++++++++++++++++++++ src/pal/linux/mbuf.cpp | 85 ++++++---------------------- src/pal/linux/mbuf.h | 65 ++++++++------------- src/pal/linux_dpdk/mbuf.cpp | 4 +- src/pal/linux_dpdk/mbuf.h | 57 ++----------------- src/stateless/dp/trex_stateless_dp_core.cpp | 35 ++++++------ 8 files changed, 204 insertions(+), 184 deletions(-) create mode 100644 src/pal/common/common_mbuf.cpp create mode 100644 src/pal/common/common_mbuf.h (limited to 'src') diff --git a/src/main_dpdk.cpp b/src/main_dpdk.cpp index b41d4f91..b959895d 100644 --- a/src/main_dpdk.cpp +++ b/src/main_dpdk.cpp @@ -2115,8 +2115,12 @@ int CCoreEthIFStateless::send_node(CGenNode * no) { } if (unlikely(node_sl->is_stat_needed())) { - return send_node_flow_stat(m, node_sl, lp_port, lp_stats, - (node_sl->get_cache_mbuf() || node_sl->is_cache_mbuf_array())? true:false); + if ( unlikely(node_sl->is_cache_mbuf_array()) ) { + // No support for latency + cache. If user asks for cache on latency stream, we change cache to 0. + // assert here just to make sure. + assert(1); + } + return send_node_flow_stat(m, node_sl, lp_port, lp_stats, (node_sl->get_cache_mbuf()) ? true : false); } else { send_pkt(lp_port,m,lp_stats); } diff --git a/src/pal/common/common_mbuf.cpp b/src/pal/common/common_mbuf.cpp new file mode 100644 index 00000000..eba29418 --- /dev/null +++ b/src/pal/common/common_mbuf.cpp @@ -0,0 +1,46 @@ +/* +Copyright (c) 2016-2016 Cisco Systems, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +#include +#include +#include "common_mbuf.h" + +/* Dump structure of mbuf chain, without the data */ +void +utl_rte_pktmbuf_dump(const struct rte_mbuf *m) { + while (m) { + printf("(%d %d %d)", m->pkt_len, m->data_len, +#ifdef TREX_SIM + (int)m->refcnt_reserved); +#else + (int)m->refcnt_atomic.cnt); +#endif + if (RTE_MBUF_INDIRECT(m)) { +#ifdef TREX_SIM + struct rte_mbuf *md = RTE_MBUF_FROM_BADDR(m->buf_addr); +#else + struct rte_mbuf *md = rte_mbuf_from_indirect((struct rte_mbuf *)m); +#endif + printf("(direct %d %d %d)", md->pkt_len, md->data_len, +#ifdef TREX_SIM + (int)md->refcnt_reserved); +#else + (int)md->refcnt_atomic.cnt); +#endif + } + m = m->next; + } + printf("\n"); +} diff --git a/src/pal/common/common_mbuf.h b/src/pal/common/common_mbuf.h new file mode 100644 index 00000000..c52842bd --- /dev/null +++ b/src/pal/common/common_mbuf.h @@ -0,0 +1,88 @@ +#ifndef COMMON_MBUF_H +#define COMMON_MBUF_H + +/* +Copyright (c) 2016-2016 Cisco Systems, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +static inline rte_mbuf_t * utl_rte_pktmbuf_add_after2(rte_mbuf_t *m1,rte_mbuf_t *m2){ + utl_rte_pktmbuf_check(m1); + utl_rte_pktmbuf_check(m2); + + m1->next=m2; + m1->pkt_len += m2->data_len; + m1->nb_segs = m2->nb_segs + 1; + return (m1); +} + +static inline rte_mbuf_t * utl_rte_pktmbuf_add_after(rte_mbuf_t *m1,rte_mbuf_t *m2){ + + utl_rte_pktmbuf_check(m1); + utl_rte_pktmbuf_check(m2); + + rte_mbuf_refcnt_update(m2,1); + m1->next=m2; + m1->pkt_len += m2->data_len; + m1->nb_segs = m2->nb_segs + 1; + return (m1); +} + + +static inline void utl_rte_pktmbuf_add_last(rte_mbuf_t *m,rte_mbuf_t *m_last){ + + //there could be 2 cases supported + //1. one mbuf + //2. two mbug where last is indirect + + if ( m->next == NULL ) { + utl_rte_pktmbuf_add_after2(m,m_last); + }else{ + m->next->next=m_last; + m->pkt_len += m_last->data_len; + m->nb_segs = 3; + } +} + +// Create following m_buf structure: +// base -> indirect -> last +// Read only is the direct of indirect. +static inline rte_mbuf_t * utl_rte_pktmbuf_chain_with_indirect (rte_mbuf_t *base, rte_mbuf_t *indirect + , rte_mbuf_t *read_only, rte_mbuf_t *last) { + rte_pktmbuf_attach(indirect, read_only); + base->next = indirect; + indirect->next = last; + rte_pktmbuf_refcnt_update(read_only, -1); + base->nb_segs = 3; + indirect->nb_segs = 2; + last->nb_segs = 1; + return base; +} + +rte_mempool_t * utl_rte_mempool_create(const char *name, + unsigned n, + unsigned elt_size, + unsigned cache_size, + uint32_t _id , + int socket_id + ); + +rte_mempool_t * utl_rte_mempool_create_non_pkt(const char *name, + unsigned n, + unsigned elt_size, + unsigned cache_size, + uint32_t _id , + int socket_id); + +#endif diff --git a/src/pal/linux/mbuf.cpp b/src/pal/linux/mbuf.cpp index 846c776c..9f568e80 100755 --- a/src/pal/linux/mbuf.cpp +++ b/src/pal/linux/mbuf.cpp @@ -27,31 +27,12 @@ limitations under the License. #include "mbuf.h" #include -#include #include #include #include "sanb_atomic.h" - -#define RTE_MBUF_TO_BADDR(mb) (((struct rte_mbuf *)(mb)) + 1) -#define RTE_MBUF_FROM_BADDR(ba) (((struct rte_mbuf *)(ba)) - 1) - - void rte_pktmbuf_detach(struct rte_mbuf *m); - - -void utl_rte_check(rte_mempool_t * mp){ - assert(mp->magic == MAGIC0); - assert(mp->magic2 == MAGIC2); -} - -void utl_rte_pktmbuf_check(struct rte_mbuf *m){ - utl_rte_check(m->pool); - assert(m->magic == MAGIC0); - assert(m->magic2== MAGIC2); -} - rte_mempool_t * utl_rte_mempool_create_non_pkt(const char *name, unsigned n, unsigned elt_size, @@ -95,8 +76,9 @@ void utl_rte_mempool_delete(rte_mempool_t * & pool){ uint16_t rte_mbuf_refcnt_update(rte_mbuf_t *m, int16_t value) { utl_rte_pktmbuf_check(m); - uint32_t a=sanb_atomic_add_return_32_old(&m->refcnt_reserved, value); - return (a); + m->refcnt_reserved = (uint16_t)(m->refcnt_reserved + value); + assert(m->refcnt_reserved >= 0); + return m->refcnt_reserved; } @@ -109,7 +91,7 @@ void rte_pktmbuf_reset(struct rte_mbuf *m) m->pkt_len = 0; m->nb_segs = 1; m->in_port = 0xff; - m->refcnt_reserved=1; + m->ol_flags = 0; #if RTE_PKTMBUF_HEADROOM > 0 m->data_off = (RTE_PKTMBUF_HEADROOM <= m->buf_len) ? @@ -136,7 +118,7 @@ rte_mbuf_t *rte_pktmbuf_alloc(rte_mempool_t *mp){ m->magic = MAGIC0; m->magic2 = MAGIC2; m->pool = mp; - m->refcnt_reserved =0; + m->refcnt_reserved = 1; m->buf_len = buf_len; m->buf_addr =(char *)((char *)m+sizeof(rte_mbuf_t)+RTE_PKTMBUF_HEADROOM) ; @@ -146,28 +128,26 @@ rte_mbuf_t *rte_pktmbuf_alloc(rte_mempool_t *mp){ return (m); } - -void rte_pktmbuf_free_seg(rte_mbuf_t *m){ +void rte_pktmbuf_free_seg(rte_mbuf_t *m) { utl_rte_pktmbuf_check(m); - uint32_t old=sanb_atomic_dec2zero32(&m->refcnt_reserved); - if (old == 1) { - struct rte_mbuf *md = RTE_MBUF_FROM_BADDR(m->buf_addr); - if ( md != m ) { + if (rte_mbuf_refcnt_update(m, -1) == 0) { + /* if this is an indirect mbuf, then + * - detach mbuf + * - free attached mbuf segment + */ + + if (RTE_MBUF_INDIRECT(m)) { + struct rte_mbuf *md = RTE_MBUF_FROM_BADDR(m->buf_addr); rte_pktmbuf_detach(m); - if (rte_mbuf_refcnt_update(md, -1) == 0) { + if (rte_mbuf_refcnt_update(md, -1) == 0) free(md); - } - } - free(m); } } - - void rte_pktmbuf_free(rte_mbuf_t *m){ rte_mbuf_t *m_next; @@ -331,19 +311,6 @@ rte_pktmbuf_dump(const struct rte_mbuf *m, unsigned dump_len) } } - -rte_mbuf_t * utl_rte_pktmbuf_add_after2(rte_mbuf_t *m1,rte_mbuf_t *m2){ - utl_rte_pktmbuf_check(m1); - utl_rte_pktmbuf_check(m2); - - m1->next=m2; - m1->pkt_len += m2->data_len; - m1->nb_segs = m2->nb_segs + 1; - return (m1); -} - - - void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *md) { @@ -355,6 +322,7 @@ void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *md) mi->next = NULL; mi->data_len = md->data_len; mi->pkt_len = mi->data_len; + mi->ol_flags = mi->ol_flags | IND_ATTACHED_MBUF; mi->nb_segs = 1; } @@ -376,33 +344,14 @@ void rte_pktmbuf_detach(struct rte_mbuf *m) m->data_len = 0; + m->ol_flags = 0; } - - - - -rte_mbuf_t * utl_rte_pktmbuf_add_after(rte_mbuf_t *m1,rte_mbuf_t *m2){ - - utl_rte_pktmbuf_check(m1); - utl_rte_pktmbuf_check(m2); - - rte_mbuf_refcnt_update(m2,1); - m1->next=m2; - m1->pkt_len += m2->data_len; - m1->nb_segs = m2->nb_segs + 1; - return (m1); -} - - uint64_t rte_rand(void){ return ( rand() ); } - - - #ifdef ONLY_A_TEST diff --git a/src/pal/linux/mbuf.h b/src/pal/linux/mbuf.h index 174c757d..e7819148 100755 --- a/src/pal/linux/mbuf.h +++ b/src/pal/linux/mbuf.h @@ -24,10 +24,19 @@ limitations under the License. #include #include +#include + +typedef struct rte_mbuf rte_mbuf_t; #define MAGIC0 0xAABBCCDD #define MAGIC2 0x11223344 +#define IND_ATTACHED_MBUF (1ULL << 62) /**< Indirect attached mbuf */ +#define RTE_MBUF_INDIRECT(mb) ((mb)->ol_flags & IND_ATTACHED_MBUF) +#define RTE_MBUF_TO_BADDR(mb) (((struct rte_mbuf *)(mb)) + 1) +#define RTE_MBUF_FROM_BADDR(ba) (((struct rte_mbuf *)(ba)) - 1) + + struct rte_mempool { uint32_t magic; uint32_t elt_size; @@ -36,9 +45,6 @@ struct rte_mempool { int size; }; - - - struct rte_mbuf { uint32_t magic; struct rte_mempool *pool; /**< Pool from which mbuf was allocated. */ @@ -55,33 +61,16 @@ struct rte_mbuf { uint32_t pkt_len; /**< Total pkt len: sum of all segment data_len. */ uint32_t magic2; - uint32_t refcnt_reserved; /**< Do not use this field */ + uint16_t refcnt_reserved; + uint64_t ol_flags; /**< Offload features. */ } ; - -typedef struct rte_mbuf rte_mbuf_t; - typedef struct rte_mempool rte_mempool_t; #define RTE_PKTMBUF_HEADROOM 0 void utl_rte_mempool_delete(rte_mempool_t * &pool); -rte_mempool_t * utl_rte_mempool_create(const char *name, - unsigned n, - unsigned elt_size, - unsigned cache_size, - uint32_t _id , - int socket_id - ); - -rte_mempool_t * utl_rte_mempool_create_non_pkt(const char *name, - unsigned n, - unsigned elt_size, - unsigned cache_size, - uint32_t _id , - int socket_id); - inline unsigned rte_mempool_count(rte_mempool_t *mp){ return (10); } @@ -107,9 +96,6 @@ void rte_pktmbuf_free_seg(rte_mbuf_t *m); uint16_t rte_mbuf_refcnt_update(rte_mbuf_t *m, int16_t value); -rte_mbuf_t * utl_rte_pktmbuf_add_after(rte_mbuf_t *m1,rte_mbuf_t *m2); -rte_mbuf_t * utl_rte_pktmbuf_add_after2(rte_mbuf_t *m1,rte_mbuf_t *m2); - void rte_pktmbuf_dump(const struct rte_mbuf *m, unsigned dump_len); @@ -166,22 +152,6 @@ rte_lcore_to_socket_id(unsigned lcore_id){ uint64_t rte_rand(void); - -static inline void utl_rte_pktmbuf_add_last(rte_mbuf_t *m,rte_mbuf_t *m_last){ - - //there could be 2 cases supported - //1. one mbuf - //2. two mbug where last is indirect - - if ( m->next == NULL ) { - utl_rte_pktmbuf_add_after2(m,m_last); - }else{ - m->next->next=m_last; - m->pkt_len += m_last->data_len; - m->nb_segs = 3; - } -} - static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v) { do { @@ -189,8 +159,16 @@ static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v) } while ((m = m->next) != NULL); } +static inline void utl_rte_check(rte_mempool_t * mp){ + assert(mp->magic == MAGIC0); + assert(mp->magic2 == MAGIC2); +} - +static inline void utl_rte_pktmbuf_check(struct rte_mbuf *m){ + utl_rte_check(m->pool); + assert(m->magic == MAGIC0); + assert(m->magic2== MAGIC2); +} #define __rte_cache_aligned @@ -199,4 +177,7 @@ static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v) #define RTE_CACHE_LINE_SIZE 64 #define SOCKET_ID_ANY 0 +// has to be after the definition of rte_mbuf and other utility functions +#include "common_mbuf.h" + #endif diff --git a/src/pal/linux_dpdk/mbuf.cpp b/src/pal/linux_dpdk/mbuf.cpp index dd78617f..2a405ab1 100755 --- a/src/pal/linux_dpdk/mbuf.cpp +++ b/src/pal/linux_dpdk/mbuf.cpp @@ -6,7 +6,7 @@ */ /* -Copyright (c) 2015-2015 Cisco Systems, Inc. +Copyright (c) 2015-2016 Cisco Systems, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ rte_mempool_t * utl_rte_mempool_create(const char *name, unsigned elt_size, unsigned cache_size, uint32_t _id, - uint32_t socket_id ){ + int socket_id ){ char buffer[100]; sprintf(buffer,"%s-%d",name,socket_id); diff --git a/src/pal/linux_dpdk/mbuf.h b/src/pal/linux_dpdk/mbuf.h index 339c0909..0d9ca8be 100755 --- a/src/pal/linux_dpdk/mbuf.h +++ b/src/pal/linux_dpdk/mbuf.h @@ -6,7 +6,7 @@ */ /* -Copyright (c) 2015-2015 Cisco Systems, Inc. +Copyright (c) 2015-2016 Cisco Systems, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,61 +27,12 @@ limitations under the License. #include typedef struct rte_mbuf rte_mbuf_t; - +inline void utl_rte_pktmbuf_check(struct rte_mbuf *m) {} typedef struct rte_mempool rte_mempool_t; -inline void utl_rte_mempool_delete(rte_mempool_t * & pool){ -} - - -rte_mempool_t * utl_rte_mempool_create(const char *name, - unsigned n, - unsigned elt_size, - unsigned cache_size, - uint32_t _id, - uint32_t socket_id ); +#include "common_mbuf.h" -rte_mempool_t * utl_rte_mempool_create_non_pkt(const char *name, - unsigned n, - unsigned elt_size, - unsigned cache_size, - uint32_t _id , - int socket_id); - - -static inline rte_mbuf_t * utl_rte_pktmbuf_add_after(rte_mbuf_t *m1,rte_mbuf_t *m2){ - - rte_mbuf_refcnt_update(m2,1); - m1->next=m2; - - m1->pkt_len += m2->data_len; - m1->nb_segs = m2->nb_segs + 1; - return (m1); -} - -static inline rte_mbuf_t * utl_rte_pktmbuf_add_after2(rte_mbuf_t *m1,rte_mbuf_t *m2){ - - m1->next=m2; - m1->pkt_len += m2->data_len; - m1->nb_segs = m2->nb_segs + 1; - return (m1); -} - -static inline void utl_rte_pktmbuf_add_last(rte_mbuf_t *m,rte_mbuf_t *m_last){ - - //there could be 2 cases supported - //1. one mbuf - //2. two mbug where last is indirect - - if ( m->next == NULL ) { - utl_rte_pktmbuf_add_after2(m,m_last); - }else{ - m->next->next=m_last; - m->pkt_len += m_last->data_len; - m->nb_segs = 3; - } +inline void utl_rte_mempool_delete(rte_mempool_t * & pool){ } - - #endif diff --git a/src/stateless/dp/trex_stateless_dp_core.cpp b/src/stateless/dp/trex_stateless_dp_core.cpp index fe78c5b2..58d8f21a 100644 --- a/src/stateless/dp/trex_stateless_dp_core.cpp +++ b/src/stateless/dp/trex_stateless_dp_core.cpp @@ -258,23 +258,20 @@ rte_mbuf_t * CGenNodeStateless::alloc_flow_stat_mbuf(rte_mbuf_t *m, struct flow_ fsp_head = (struct flow_stat_payload_header *)(p + rte_pktmbuf_data_len(m) - fsp_head_size); return m; } else { - // r/w --> read only. Should do something like: - // Alloc indirect,. make r/w->indirect point to read_only) -> new fsp_header - // for the mean time, just copy the entire packet. - m_ret = CGlobalInfo::pktmbuf_alloc( get_socket_id(), rte_pktmbuf_pkt_len(m) ); - assert(m_ret); - char *p_new = rte_pktmbuf_append(m_ret, rte_pktmbuf_pkt_len(m)); - rte_mbuf_t *m_free = m; - while (m != NULL) { - char *p = rte_pktmbuf_mtod(m, char*); - memcpy(p_new, p, m->data_len); - p_new += m->data_len; - m = m->next; - } - p_new = rte_pktmbuf_mtod(m_ret, char*); - fsp_head = (struct flow_stat_payload_header *)(p_new + rte_pktmbuf_data_len(m_ret) - fsp_head_size); - rte_pktmbuf_free(m_free); - return m_ret; + // We have: r/w --> read only. + // Changing to: + // (original) r/w -> (new) indirect (direct is original read_only, after trimming last bytes) -> (new) latency info + rte_mbuf_t *m_read_only = m->next, *m_indirect; + + m_indirect = CGlobalInfo::pktmbuf_alloc_small(get_socket_id()); + assert(m_indirect); + // alloc mbuf just for the latency header + m_lat = CGlobalInfo::pktmbuf_alloc( get_socket_id(), fsp_head_size); + assert(m_lat); + fsp_head = (struct flow_stat_payload_header *)rte_pktmbuf_append(m_lat, fsp_head_size); + utl_rte_pktmbuf_chain_with_indirect(m, m_indirect, m_read_only, m_lat); + m_indirect->data_len = (uint16_t)(m_indirect->data_len - fsp_head_size); + return m; } } } @@ -910,6 +907,10 @@ TrexStatelessDpCore::add_stream(TrexStatelessDpPerPort * lp_port, uint8_t hw_id = stream->m_rx_check.m_hw_id; assert (hw_id < MAX_FLOW_STATS + MAX_FLOW_STATS_PAYLOAD); node->set_stat_hw_id(hw_id); + // no support for cache with flow stat payload rules + if ((TrexPlatformApi::driver_stat_cap_e)stream->m_rx_check.m_rule_type == TrexPlatformApi::IF_STAT_PAYLOAD) { + stream->m_cache_size = 0; + } } /* set socket id */ -- cgit 1.2.3-korg