better support for latency with field engine + some order with mbuf functions

6 files changed, 180 insertions, 165 deletions

diff --git a/src/pal/common/common_mbuf.cpp b/src/pal/common/common_mbuf.cpp
new file mode 100644
index 00000000..eba29418
--- /dev/null
+++ b/src/pal/common/common_mbuf.cpp
@@ -0,0 +1,46 @@
+/*
+Copyright (c) 2016-2016 Cisco Systems, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#include <stdio.h>
+#include <mbuf.h>
+#include "common_mbuf.h"
+
+/* Dump structure of mbuf chain, without the data */
+void
+utl_rte_pktmbuf_dump(const struct rte_mbuf *m) {
+            while (m) {
+                printf("(%d %d %d)", m->pkt_len, m->data_len,
+#ifdef TREX_SIM
+                       (int)m->refcnt_reserved);
+#else
+                       (int)m->refcnt_atomic.cnt);
+#endif
+                if (RTE_MBUF_INDIRECT(m)) {
+#ifdef TREX_SIM
+                    struct rte_mbuf *md = RTE_MBUF_FROM_BADDR(m->buf_addr);
+#else
+                    struct rte_mbuf *md = rte_mbuf_from_indirect((struct rte_mbuf *)m);
+#endif
+                    printf("(direct %d %d %d)", md->pkt_len, md->data_len,
+#ifdef TREX_SIM
+                           (int)md->refcnt_reserved);
+#else
+                           (int)md->refcnt_atomic.cnt);
+#endif
+                }
+                m = m->next;
+            }
+            printf("\n");
+}
diff --git a/src/pal/common/common_mbuf.h b/src/pal/common/common_mbuf.h
new file mode 100644
index 00000000..c52842bd
--- /dev/null
+++ b/src/pal/common/common_mbuf.h
@@ -0,0 +1,88 @@
+#ifndef COMMON_MBUF_H
+#define COMMON_MBUF_H
+
+/*
+Copyright (c) 2016-2016 Cisco Systems, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+static inline rte_mbuf_t * utl_rte_pktmbuf_add_after2(rte_mbuf_t *m1,rte_mbuf_t *m2){
+    utl_rte_pktmbuf_check(m1);
+    utl_rte_pktmbuf_check(m2);
+
+    m1->next=m2;
+    m1->pkt_len += m2->data_len;
+    m1->nb_segs = m2->nb_segs + 1;
+    return (m1);
+}
+
+static inline rte_mbuf_t * utl_rte_pktmbuf_add_after(rte_mbuf_t *m1,rte_mbuf_t *m2){
+
+    utl_rte_pktmbuf_check(m1);
+    utl_rte_pktmbuf_check(m2);
+
+    rte_mbuf_refcnt_update(m2,1);
+    m1->next=m2;
+    m1->pkt_len += m2->data_len;
+    m1->nb_segs = m2->nb_segs + 1;
+    return (m1);
+}
+
+
+static inline void utl_rte_pktmbuf_add_last(rte_mbuf_t *m,rte_mbuf_t *m_last){
+
+    //there could be 2 cases supported
+    //1. one mbuf
+    //2. two mbug where last is indirect
+
+    if ( m->next == NULL ) {
+        utl_rte_pktmbuf_add_after2(m,m_last);
+    }else{
+        m->next->next=m_last;
+        m->pkt_len += m_last->data_len;
+        m->nb_segs = 3;
+    }
+}
+
+// Create following m_buf structure:
+// base -> indirect -> last
+// Read only is the direct of indirect.
+static inline rte_mbuf_t * utl_rte_pktmbuf_chain_with_indirect (rte_mbuf_t *base, rte_mbuf_t *indirect
+                                                  , rte_mbuf_t *read_only, rte_mbuf_t *last) {
+    rte_pktmbuf_attach(indirect, read_only);
+    base->next = indirect;
+    indirect->next = last;
+    rte_pktmbuf_refcnt_update(read_only, -1);
+    base->nb_segs = 3;
+    indirect->nb_segs = 2;
+    last->nb_segs = 1;
+    return base;
+}
+
+rte_mempool_t * utl_rte_mempool_create(const char  *name,
+                                      unsigned n,
+                                      unsigned elt_size,
+                                      unsigned cache_size,
+                                      uint32_t _id ,
+                                      int socket_id
+                                       );
+
+rte_mempool_t * utl_rte_mempool_create_non_pkt(const char  *name,
+                                               unsigned n,
+                                               unsigned elt_size,
+                                               unsigned cache_size,
+                                               uint32_t _id ,
+                                               int socket_id);
+
+#endif
diff --git a/src/pal/linux/mbuf.cpp b/src/pal/linux/mbuf.cpp
index 846c776c..9f568e80 100755
--- a/src/pal/linux/mbuf.cpp
+++ b/src/pal/linux/mbuf.cpp
@@ -27,31 +27,12 @@ limitations under the License.
 
 #include "mbuf.h"
 #include <stdio.h>
-#include <assert.h>
 #include <stdlib.h>
 #include <ctype.h>
 #include "sanb_atomic.h"
 
-
-#define RTE_MBUF_TO_BADDR(mb)       (((struct rte_mbuf *)(mb)) + 1)
-#define RTE_MBUF_FROM_BADDR(ba)     (((struct rte_mbuf *)(ba)) - 1)
-
-
 void rte_pktmbuf_detach(struct rte_mbuf *m);
 
-
-
-void utl_rte_check(rte_mempool_t * mp){
-    assert(mp->magic  == MAGIC0);
-    assert(mp->magic2 == MAGIC2);
-}
-
-void utl_rte_pktmbuf_check(struct rte_mbuf *m){
-    utl_rte_check(m->pool);
-    assert(m->magic == MAGIC0);
-    assert(m->magic2== MAGIC2);
-}
-
 rte_mempool_t * utl_rte_mempool_create_non_pkt(const char  *name,
                                                unsigned n,
                                                unsigned elt_size,
@@ -95,8 +76,9 @@ void utl_rte_mempool_delete(rte_mempool_t * & pool){
 uint16_t rte_mbuf_refcnt_update(rte_mbuf_t *m, int16_t value)
 {
     utl_rte_pktmbuf_check(m);
-    uint32_t a=sanb_atomic_add_return_32_old(&m->refcnt_reserved, value);
-    return (a);
+    m->refcnt_reserved = (uint16_t)(m->refcnt_reserved + value);
+    assert(m->refcnt_reserved >= 0);
+	return m->refcnt_reserved;
 }
 
 
@@ -109,7 +91,7 @@ void rte_pktmbuf_reset(struct rte_mbuf *m)
     m->pkt_len = 0;
     m->nb_segs = 1;
     m->in_port = 0xff;
-    m->refcnt_reserved=1;
+    m->ol_flags = 0;
 
     #if RTE_PKTMBUF_HEADROOM > 0
     m->data_off = (RTE_PKTMBUF_HEADROOM <= m->buf_len) ?
@@ -136,7 +118,7 @@ rte_mbuf_t *rte_pktmbuf_alloc(rte_mempool_t *mp){
     m->magic  = MAGIC0;
     m->magic2 = MAGIC2;
     m->pool   = mp;
-    m->refcnt_reserved =0;
+    m->refcnt_reserved = 1;
 
     m->buf_len    = buf_len;
     m->buf_addr   =(char *)((char *)m+sizeof(rte_mbuf_t)+RTE_PKTMBUF_HEADROOM) ;
@@ -146,28 +128,26 @@ rte_mbuf_t *rte_pktmbuf_alloc(rte_mempool_t *mp){
     return (m);
 }
 
-
-void rte_pktmbuf_free_seg(rte_mbuf_t *m){
+void rte_pktmbuf_free_seg(rte_mbuf_t *m) {
 
     utl_rte_pktmbuf_check(m);
-    uint32_t old=sanb_atomic_dec2zero32(&m->refcnt_reserved);
-    if (old == 1) {
-        struct rte_mbuf *md = RTE_MBUF_FROM_BADDR(m->buf_addr);
 
-        if ( md != m ) {
+	if (rte_mbuf_refcnt_update(m, -1) == 0) {
+        /* if this is an indirect mbuf, then
+         *  - detach mbuf
+         *  - free attached mbuf segment
+         */
+
+        if (RTE_MBUF_INDIRECT(m)) {
+            struct rte_mbuf *md = RTE_MBUF_FROM_BADDR(m->buf_addr);
             rte_pktmbuf_detach(m);
-            if (rte_mbuf_refcnt_update(md, -1) == 0) {
+            if (rte_mbuf_refcnt_update(md, -1) == 0)
                 free(md);
-            }
-
         }
-
         free(m);
     }
 }
 
-
-
 void rte_pktmbuf_free(rte_mbuf_t *m){
 
     rte_mbuf_t *m_next;
@@ -331,19 +311,6 @@ rte_pktmbuf_dump(const struct rte_mbuf *m, unsigned dump_len)
     }
 }
 
-
-rte_mbuf_t * utl_rte_pktmbuf_add_after2(rte_mbuf_t *m1,rte_mbuf_t *m2){
-    utl_rte_pktmbuf_check(m1);
-    utl_rte_pktmbuf_check(m2);
-
-    m1->next=m2;
-    m1->pkt_len += m2->data_len;
-    m1->nb_segs = m2->nb_segs + 1;
-    return (m1);
-}
-
-
-
 void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *md)
 {
 
@@ -355,6 +322,7 @@ void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *md)
     mi->next = NULL;
     mi->data_len = md->data_len;
     mi->pkt_len  = mi->data_len;
+    mi->ol_flags = mi->ol_flags | IND_ATTACHED_MBUF;
     mi->nb_segs = 1;
 }
 
@@ -376,33 +344,14 @@ void rte_pktmbuf_detach(struct rte_mbuf *m)
 
 
     m->data_len = 0;
+	m->ol_flags = 0;
 }
 
-
-
-
-
-rte_mbuf_t * utl_rte_pktmbuf_add_after(rte_mbuf_t *m1,rte_mbuf_t *m2){
-
-    utl_rte_pktmbuf_check(m1);
-    utl_rte_pktmbuf_check(m2);
-
-    rte_mbuf_refcnt_update(m2,1);
-    m1->next=m2;
-    m1->pkt_len += m2->data_len;
-    m1->nb_segs = m2->nb_segs + 1;
-    return (m1);
-}
-
-
 uint64_t rte_rand(void){
     return ( rand() );
 }
 
 
-
-
-
 #ifdef ONLY_A_TEST
 
 
diff --git a/src/pal/linux/mbuf.h b/src/pal/linux/mbuf.h
index 174c757d..e7819148 100755
--- a/src/pal/linux/mbuf.h
+++ b/src/pal/linux/mbuf.h
@@ -24,10 +24,19 @@ limitations under the License.
 
 #include <stdint.h>
 #include <string.h>
+#include <assert.h>
+
+typedef struct rte_mbuf  rte_mbuf_t;
 
 #define MAGIC0 0xAABBCCDD
 #define MAGIC2 0x11223344
 
+#define IND_ATTACHED_MBUF    (1ULL << 62) /**< Indirect attached mbuf */
+#define RTE_MBUF_INDIRECT(mb)   ((mb)->ol_flags & IND_ATTACHED_MBUF)
+#define RTE_MBUF_TO_BADDR(mb)       (((struct rte_mbuf *)(mb)) + 1)
+#define RTE_MBUF_FROM_BADDR(ba)     (((struct rte_mbuf *)(ba)) - 1)
+
+
 struct rte_mempool {
     uint32_t  magic;
     uint32_t  elt_size;
@@ -36,9 +45,6 @@ struct rte_mempool {
     int size;
 };
 
-
-
-
 struct rte_mbuf {
     uint32_t magic;
     struct rte_mempool *pool; /**< Pool from which mbuf was allocated. */
@@ -55,33 +61,16 @@ struct rte_mbuf {
     uint32_t pkt_len;       /**< Total pkt len: sum of all segment data_len. */
 
     uint32_t magic2;
-    uint32_t refcnt_reserved;     /**< Do not use this field */
+    uint16_t refcnt_reserved;
+    uint64_t ol_flags;        /**< Offload features. */
 } ;
 
-
-typedef struct rte_mbuf  rte_mbuf_t;
-
 typedef struct rte_mempool rte_mempool_t;
 
 #define RTE_PKTMBUF_HEADROOM  0
 
 void utl_rte_mempool_delete(rte_mempool_t  * &pool);
 
-rte_mempool_t * utl_rte_mempool_create(const char  *name,
-                                      unsigned n, 
-                                      unsigned elt_size,
-                                      unsigned cache_size,
-                                      uint32_t _id ,
-                                      int socket_id
-                                       );
-
-rte_mempool_t * utl_rte_mempool_create_non_pkt(const char  *name,
-                                               unsigned n, 
-                                               unsigned elt_size,
-                                               unsigned cache_size,
-                                               uint32_t _id ,
-                                               int socket_id);
-
 inline unsigned rte_mempool_count(rte_mempool_t  *mp){
     return (10);
 }
@@ -107,9 +96,6 @@ void rte_pktmbuf_free_seg(rte_mbuf_t *m);
 
 uint16_t rte_mbuf_refcnt_update(rte_mbuf_t *m, int16_t value);
 
-rte_mbuf_t * utl_rte_pktmbuf_add_after(rte_mbuf_t *m1,rte_mbuf_t *m2);
-rte_mbuf_t * utl_rte_pktmbuf_add_after2(rte_mbuf_t *m1,rte_mbuf_t *m2);
-
 void rte_pktmbuf_dump(const struct rte_mbuf *m, unsigned dump_len);
 
 
@@ -166,22 +152,6 @@ rte_lcore_to_socket_id(unsigned lcore_id){
 
 uint64_t rte_rand(void);
 
-
-static inline void utl_rte_pktmbuf_add_last(rte_mbuf_t *m,rte_mbuf_t *m_last){
-
-	//there could be 2 cases supported 
-	//1. one mbuf 
-	//2. two mbug where last is indirect 
-
-	if ( m->next == NULL ) {
-		utl_rte_pktmbuf_add_after2(m,m_last);
-	}else{
-		m->next->next=m_last;
-		m->pkt_len += m_last->data_len;
-		m->nb_segs = 3;
-	}
-}
-
 static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
 {
 	do {
@@ -189,8 +159,16 @@ static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
 	} while ((m = m->next) != NULL);
 }
 
+static inline void utl_rte_check(rte_mempool_t * mp){
+    assert(mp->magic  == MAGIC0);
+    assert(mp->magic2 == MAGIC2);
+}
 
-
+static inline void utl_rte_pktmbuf_check(struct rte_mbuf *m){
+    utl_rte_check(m->pool);
+    assert(m->magic == MAGIC0);
+    assert(m->magic2== MAGIC2);
+}
 
 #define __rte_cache_aligned 
 
@@ -199,4 +177,7 @@ static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
 #define RTE_CACHE_LINE_SIZE 64
 #define SOCKET_ID_ANY 0
 
+// has to be after the definition of rte_mbuf and other utility functions
+#include "common_mbuf.h"
+
 #endif
diff --git a/src/pal/linux_dpdk/mbuf.cpp b/src/pal/linux_dpdk/mbuf.cpp
index dd78617f..2a405ab1 100755
--- a/src/pal/linux_dpdk/mbuf.cpp
+++ b/src/pal/linux_dpdk/mbuf.cpp
@@ -6,7 +6,7 @@
 */
 
 /*
-Copyright (c) 2015-2015 Cisco Systems, Inc.
+Copyright (c) 2015-2016 Cisco Systems, Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@ rte_mempool_t * utl_rte_mempool_create(const char  *name,
                                        unsigned elt_size,
                                        unsigned cache_size,
                                        uint32_t _id,
-                                       uint32_t socket_id ){
+                                       int socket_id ){
     char buffer[100];
     sprintf(buffer,"%s-%d",name,socket_id);
 
diff --git a/src/pal/linux_dpdk/mbuf.h b/src/pal/linux_dpdk/mbuf.h
index 339c0909..0d9ca8be 100755
--- a/src/pal/linux_dpdk/mbuf.h
+++ b/src/pal/linux_dpdk/mbuf.h
@@ -6,7 +6,7 @@
 */
 
 /*
-Copyright (c) 2015-2015 Cisco Systems, Inc.
+Copyright (c) 2015-2016 Cisco Systems, Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -27,61 +27,12 @@ limitations under the License.
 #include <rte_random.h>
 
 typedef struct rte_mbuf  rte_mbuf_t;
-
+inline void utl_rte_pktmbuf_check(struct rte_mbuf *m) {}
 typedef struct rte_mempool rte_mempool_t;
 
-inline void utl_rte_mempool_delete(rte_mempool_t * & pool){
-}
-
-
-rte_mempool_t * utl_rte_mempool_create(const char  *name,
-                                      unsigned n, 
-                                      unsigned elt_size,
-                                      unsigned cache_size,
-                                      uint32_t _id,
-                                      uint32_t socket_id  );
+#include "common_mbuf.h"
 
-rte_mempool_t * utl_rte_mempool_create_non_pkt(const char  *name,
-                                               unsigned n, 
-                                               unsigned elt_size,
-                                               unsigned cache_size,
-                                               uint32_t _id ,
-                                               int socket_id);
-
-
-static inline rte_mbuf_t * utl_rte_pktmbuf_add_after(rte_mbuf_t *m1,rte_mbuf_t *m2){
-
-    rte_mbuf_refcnt_update(m2,1);
-    m1->next=m2;
-
-    m1->pkt_len += m2->data_len;
-    m1->nb_segs = m2->nb_segs + 1;
-    return (m1);
-}
-
-static inline rte_mbuf_t * utl_rte_pktmbuf_add_after2(rte_mbuf_t *m1,rte_mbuf_t *m2){
-
-    m1->next=m2;
-    m1->pkt_len += m2->data_len;
-    m1->nb_segs = m2->nb_segs + 1;
-    return (m1);
-}
-
-static inline void utl_rte_pktmbuf_add_last(rte_mbuf_t *m,rte_mbuf_t *m_last){
-
-	//there could be 2 cases supported 
-	//1. one mbuf 
-	//2. two mbug where last is indirect 
-
-	if ( m->next == NULL ) {
-		utl_rte_pktmbuf_add_after2(m,m_last);
-	}else{
-		m->next->next=m_last;
-		m->pkt_len += m_last->data_len;
-		m->nb_segs = 3;
-	}
+inline void utl_rte_mempool_delete(rte_mempool_t * & pool){
 }
 
-
-
 #endif