aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--docs/report/csit_framework_documentation/index.rst9
-rw-r--r--docs/report/detailed_test_results/index.rst16
-rw-r--r--docs/report/detailed_test_results/vpp_unittest_results/index.rst7
-rw-r--r--docs/report/detailed_test_results/vpp_unittest_results/vpp_unittest_results.rst1178
-rw-r--r--docs/report/dpdk_performance_tests/index.rst12
-rw-r--r--docs/report/honeycomb_functional_tests/index.rst10
-rw-r--r--docs/report/index.rst109
-rw-r--r--docs/report/introduction/index.rst7
-rw-r--r--docs/report/nsh_sfc_functional_tests/index.rst10
-rw-r--r--docs/report/test_configuration/index.rst12
-rw-r--r--docs/report/test_operational_data/index.rst8
-rw-r--r--docs/report/vpp_functional_tests/index.rst10
-rw-r--r--docs/report/vpp_performance_tests/csit_release_notes.rst13
-rw-r--r--docs/report/vpp_performance_tests/index.rst15
-rw-r--r--docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst2
-rw-r--r--docs/report/vpp_unit_tests/documentation.rst6
-rw-r--r--docs/report/vpp_unit_tests/index.rst8
-rw-r--r--docs/report/vpp_unit_tests/overview.rst87
-rw-r--r--resources/tools/presentation/conf.py2
-rw-r--r--resources/tools/presentation/generator_report.py68
-rw-r--r--resources/tools/presentation/pal.py6
-rwxr-xr-xresources/tools/presentation/run_report.sh1
22 files changed, 179 insertions, 1417 deletions
diff --git a/docs/report/csit_framework_documentation/index.rst b/docs/report/csit_framework_documentation/index.rst
deleted file mode 100644
index 0b3228dc51..0000000000
--- a/docs/report/csit_framework_documentation/index.rst
+++ /dev/null
@@ -1,9 +0,0 @@
-CSIT Framework Documentation
-============================
-
-.. toctree::
-
- csit_design
- csit_test_naming
- pal_lld
- csit_tag_description
diff --git a/docs/report/detailed_test_results/index.rst b/docs/report/detailed_test_results/index.rst
deleted file mode 100644
index 208f3031f1..0000000000
--- a/docs/report/detailed_test_results/index.rst
+++ /dev/null
@@ -1,16 +0,0 @@
-.. only:: html
-
-
- Detailed Test Results
- =====================
-
- .. toctree::
-
- vpp_performance_results/index
- vpp_mrr_results/index
- cot_performance_results/index
- dpdk_performance_results/index
- vpp_functional_results/index
- vpp_functional_results_centos/index
- honeycomb_functional_results/index
- nshsfc_functional_results/index
diff --git a/docs/report/detailed_test_results/vpp_unittest_results/index.rst b/docs/report/detailed_test_results/vpp_unittest_results/index.rst
deleted file mode 100644
index 5659d5e1ab..0000000000
--- a/docs/report/detailed_test_results/vpp_unittest_results/index.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-VPP Unit Test Results
-=====================
-
-.. toctree::
-
- vpp_unittest_results
-
diff --git a/docs/report/detailed_test_results/vpp_unittest_results/vpp_unittest_results.rst b/docs/report/detailed_test_results/vpp_unittest_results/vpp_unittest_results.rst
deleted file mode 100644
index ba35f2b831..0000000000
--- a/docs/report/detailed_test_results/vpp_unittest_results/vpp_unittest_results.rst
+++ /dev/null
@@ -1,1178 +0,0 @@
-ACL Security Groups
-```````````````````
-::
-
- ==============================================================================
- ACL plugin Test Case
- ==============================================================================
- ACL plugin version check; learn MACs OK
- ACL create/delete test OK
- permit ACL apply test OK
- deny ACL apply test OK
- VPP_624 permit ICMPv4 OK
- VPP_624 permit ICMPv6 OK
- VPP_624 deny ICMPv4 OK
- VPP_624 deny ICMPv6 OK
- permit TCPv4 OK
- permit TCPv6 OK
- permit UDPv4 OK
- permit UDPv6 OK
- deny TCPv4/v6 OK
- deny UDPv4/v6 OK
- verify add/dump acls OK
- permit single TCPv4 OK
- permit single UDPv4 OK
- permit single TCPv6 OK
- permit single UPPv6 OK
- deny single TCPv4/v6 OK
- deny single UDPv4/v6 OK
- deny single UDPv4/v6, permit ip any, verify non-initial fragment blocked OK
- VPP-687 zero length udp ipv4 packet OK
- VPP-687 zero length udp ipv6 packet OK
- permit TCPv4 + non-match range OK
- permit TCPv6 + non-match range OK
- permit UDPv4 + non-match range OK
- permit UDPv6 + non-match range OK
- deny TCPv4/v6 + non-match range OK
- deny UDPv4/v6 + non-match range OK
-
- ==============================================================================
- IRB Test Case
- ==============================================================================
- ACL plugin prepare OK
- ACL IPv6 routed -> bridged, L2 ACL deny OK
- ACL IPv6 routed -> bridged, L3 ACL deny OK
- ACL IPv4 routed -> bridged, L2 ACL deny OK
- ACL IPv4 routed -> bridged, L3 ACL deny OK
- ACL IPv6 bridged -> routed, L2 ACL deny OK
- ACL IPv6 bridged -> routed, L3 ACL deny OK
- ACL IPv4 bridged -> routed, L2 ACL deny OK
- ACL IPv4 bridged -> routed, L3 ACL deny OK
- ACL IPv6 routed -> bridged, L2 ACL permit+reflect OK
- ACL IPv6 bridged -> routed, L2 ACL permit+reflect OK
- ACL IPv4 routed -> bridged, L2 ACL permit+reflect OK
- ACL IPv4 bridged -> routed, L2 ACL permit+reflect OK
- ACL IPv6 routed -> bridged, L3 ACL permit+reflect OK
- ACL IPv6 bridged -> routed, L3 ACL permit+reflect OK
- ACL IPv4 routed -> bridged, L3 ACL permit+reflect OK
- ACL IPv4 bridged -> routed, L3 ACL permit+reflect OK
- ACL IPv6+EH routed -> bridged, L2 ACL deny OK
- ACL IPv6+EH routed -> bridged, L3 ACL deny OK
- ACL IPv6+EH bridged -> routed, L2 ACL deny OK
- ACL IPv6+EH bridged -> routed, L3 ACL deny OK
- ACL IPv6+EH routed -> bridged, L2 ACL permit+reflect OK
- ACL IPv6+EH bridged -> routed, L2 ACL permit+reflect OK
- ACL IPv6+EH routed -> bridged, L3 ACL permit+reflect OK
- ACL IPv6+EH bridged -> routed, L3 ACL permit+reflect OK
- ACL IPv4+MF routed -> bridged, L2 ACL deny OK
- ACL IPv4+MF routed -> bridged, L3 ACL deny OK
- ACL IPv4+MF bridged -> routed, L2 ACL deny OK
- ACL IPv4+MF bridged -> routed, L3 ACL deny OK
- ACL IPv4+MF routed -> bridged, L2 ACL permit+reflect OK
- ACL IPv4+MF bridged -> routed, L2 ACL permit+reflect OK
- ACL IPv4+MF routed -> bridged, L3 ACL permit+reflect OK
- ACL IPv4+MF bridged -> routed, L3 ACL permit+reflect OK
-
- ==============================================================================
- ACL plugin connection-oriented extended testcases
- ==============================================================================
- Prepare the settings SKIP
- IPv4: Basic conn timeout test reflect on ingress SKIP
- IPv4: Basic conn timeout test reflect on egress SKIP
- IPv4: reflect egress, clear conn SKIP
- IPv4: reflect ingress, clear conn SKIP
- IPv4: Idle conn behind active conn, reflect on ingress SKIP
- IPv4: Idle conn behind active conn, reflect on egress SKIP
- IPv6: Basic conn timeout test reflect on ingress SKIP
- IPv6: Basic conn timeout test reflect on egress SKIP
- IPv6: reflect egress, clear conn SKIP
- IPv6: reflect ingress, clear conn SKIP
- IPv6: Idle conn behind active conn, reflect on ingress SKIP
- IPv6: Idle conn behind active conn, reflect on egress SKIP
- Prepare for TCP session tests SKIP
- IPv4: transient TCP session (incomplete 3WHS), ref. on ingress SKIP
- IPv4: transient TCP session (incomplete 3WHS), ref. on egress SKIP
- IPv4: established TCP session (complete 3WHS), ref. on ingress SKIP
- IPv4: established TCP session (complete 3WHS), ref. on egress SKIP
- IPv4: transient TCP session (3WHS,ACK,FINACK), ref. on ingress SKIP
- IPv4: transient TCP session (3WHS,ACK,FINACK), ref. on egress SKIP
- IPv6: transient TCP session (incomplete 3WHS), ref. on ingress SKIP
- IPv6: transient TCP session (incomplete 3WHS), ref. on egress SKIP
- IPv6: established TCP session (complete 3WHS), ref. on ingress SKIP
- IPv6: established TCP session (complete 3WHS), ref. on egress SKIP
- IPv6: transient TCP session (3WHS,ACK,FINACK), ref. on ingress SKIP
- IPv6: transient TCP session (3WHS,ACK,FINACK), ref. on egress SKIP
-
- ==============================================================================
- ACL on dot1q bridged subinterfaces Tests
- ==============================================================================
- IP4 ACL SubIf Dot1Q bridged traffic OK
- IP6 ACL SubIf Dot1Q bridged traffic OK
-
- ==============================================================================
- ACL on dot1ad bridged subinterfaces Tests
- ==============================================================================
- IP4 ACL SubIf Dot1AD bridged traffic OK
- IP6 ACL SubIf Dot1AD bridged traffic OK
-
- ==============================================================================
- ACL on dot1ad routed subinterfaces Tests
- ==============================================================================
- IP4 ACL SubIf Dot1AD routed traffic OK
- IP4 ACL SubIf wrong tags Dot1AD routed traffic OK
- IP6 ACL SubIf Dot1AD routed traffic OK
- IP6 ACL SubIf wrong tags Dot1AD routed traffic OK
-
- ==============================================================================
- ACL on dot1q routed subinterfaces Tests
- ==============================================================================
- IP4 ACL SubIf Dot1Q routed traffic OK
- IP4 ACL SubIf wrong tags Dot1Q routed traffic OK
- IP6 ACL SubIf Dot1Q routed traffic OK
- IP6 ACL SubIf wrong tags Dot1Q routed traffic OK
-
-APIs
-````
-::
-
- ==============================================================================
- VAPI test
- ==============================================================================
- run C VAPI tests SKIP
- run C++ VAPI tests SKIP
-
- ==============================================================================
- VPP Object Model Test
- ==============================================================================
- run C++ VOM tests SKIP
-
- ==============================================================================
- PAPI Test Case
- ==============================================================================
- show version OK
- show version - invalid parameters OK
- u8 array OK
-
- ==============================================================================
- PAPI Message parsing Test Case
- ==============================================================================
- New compound type with array OK
- Add new types OK
- Add new types 2 OK
- Add new message object OK
- New message with array OK
- Argument name OK
- VLA with aribtrary length field placement OK
- Message to byte encoding OK
- Nested array type OK
- Old style VLA array OK
- Old VLA compound type OK
- Old VLA array arbitrary placement OK
- Old VLA u32 OK
- Simple array OK
-
- ==============================================================================
- JVPP Core Test Case
- ==============================================================================
- JVPP Acl Callback Api Test Case OK
- JVPP Acl Future Api Test Case OK
- JVPP Core Callback Api Test Case OK
- JVPP Core Future Api Test Case OK
- JVPP Ioamexport Callback Api Test Case OK
- JVPP Ioamexport Future Api Test Case OK
- JVPP Ioampot Callback Api Test Case OK
- JVPP Ioampot Future Api Test Case OK
- JVPP Ioamtrace Callback Api Test Case OK
- JVPP Ioamtrace Future Api Test Case OK
- JVPP Snat Callback Api Test Case OK
- JVPP Snat Future Api Test Case OK
-
-ARP
-```
-::
-
- ==============================================================================
- ARP Test Case
- ==============================================================================
- ARP OK
- ARP Duplicates OK
- ARP Static OK
- ARP reply with VRRP virtual src hw addr OK
- MPLS OK
- Proxy ARP OK
- Interface Mirror Proxy ARP OK
-
- ==============================================================================
- L2BD arp termination Test Case
- ==============================================================================
- L2BD arp term - add 5 hosts, verify arp responses OK
- L2BD arp term - delete 3 hosts, verify arp responses OK
- L2BD arp term - recreate BD1, readd 3 hosts, verify arp responses OK
- L2BD arp term - 2 IP4 addrs per host OK
- L2BD arp term - create and update 10 IP4-mac pairs OK
- L2BD arp/ND term - hosts with both ip4/ip6 OK
- L2BD ND term - Add and Del hosts, verify ND replies OK
- L2BD ND term - Add and update IP+mac, verify ND replies OK
- L2BD arp term - send garps, verify arp event reports OK
- L2BD arp term - send duplicate garps, verify suppression OK
- L2BD arp term - disable ip4 arp events,send garps, verify no events OK
- L2BD ND term - send NS packets verify reports OK
- L2BD ND term - send duplicate ns, verify suppression OK
- L2BD ND term - disable ip4 arp events,send ns, verify no events OK
-
-BFD API
-````````
-::
-
- ==============================================================================
- Bidirectional Forwarding Detection (BFD) - API
- ==============================================================================
- activate SHA1 authentication SKIP
- create BFD session using non-existent SHA1 (negative case) SKIP
- create a BFD session SKIP
- create IPv6 BFD session SKIP
- create a BFD session (SHA1) SKIP
- add SHA1 keys SKIP
- change SHA1 key SKIP
- deactivate SHA1 authentication SKIP
- create the same BFD session twice (negative case) SKIP
- create the same BFD session twice (negative case) (SHA1) SKIP
- modify BFD session parameters SKIP
- share single SHA1 key between multiple BFD sessions SKIP
-
-BFD Authentication
-``````````````````
-::
-
- ==============================================================================
- Bidirectional Forwarding Detection (BFD) (SHA1 auth)
- ==============================================================================
- hold BFD session up SKIP
- hold BFD session up - meticulous auth SKIP
- session is not brought down by unauthenticated msg SKIP
- session is not brought down by msg with non-existent key-id SKIP
- session is not brought down by msg with wrong auth type SKIP
- simulate remote peer restart and resynchronization SKIP
- session is not kept alive by msgs with bad sequence numbers SKIP
- bring BFD session up SKIP
-
-BFD Authentication Change
-`````````````````````````
-::
-
- ==============================================================================
- Bidirectional Forwarding Detection (BFD) (changing auth)
- ==============================================================================
- change auth key without disturbing session state (delayed) SKIP
- change auth key without disturbing session state (immediate) SKIP
- turn auth off without disturbing session state (delayed) SKIP
- turn auth off without disturbing session state (immediate) SKIP
- turn auth on without disturbing session state (delayed) SKIP
- turn auth on without disturbing session state (immediate) SKIP
-
-BFD CLI
-````````
-::
-
- ==============================================================================
- Bidirectional Forwarding Detection (BFD) (CLI)
- ==============================================================================
- create/modify/delete IPv4 BFD UDP session SKIP
- create/modify/delete IPv6 BFD UDP session SKIP
- create/modify/delete IPv6 BFD UDP session (authenticated) SKIP
- create/modify/delete IPv4 BFD UDP session (authenticated) SKIP
- put session admin-up and admin-down SKIP
- turn authentication on and off SKIP
- turn authentication on and off (delayed) SKIP
- set/delete meticulous SHA1 auth key SKIP
- set/delete SHA1 auth key SKIP
- set/del udp echo source SKIP
- show commands SKIP
-
-BFD IPv4
-````````
-::
-
- ==============================================================================
- Bidirectional Forwarding Detection (BFD)
- ==============================================================================
- put session admin-up and admin-down SKIP
- configuration change while peer in demand mode SKIP
- verify session goes down after inactivity SKIP
- echo function SKIP
- session goes down if echo function fails SKIP
- echo packets looped back SKIP
- echo function stops if echo source is removed SKIP
- echo function stops if peer sets required min echo rx zero SKIP
- hold BFD session up SKIP
- immediately honor remote required min rx reduction SKIP
- interface with bfd session deleted SKIP
- echo packets with invalid checksum don't keep a session up SKIP
- large remote required min rx interval SKIP
- modify detect multiplier SKIP
- modify session - double required min rx SKIP
- modify session - halve required min rx SKIP
- no periodic frames outside poll sequence if remote demand set SKIP
- test correct response to control frame with poll bit set SKIP
- test poll sequence queueing SKIP
- bring BFD session down SKIP
- bring BFD session up SKIP
- bring BFD session up - first frame looked up by address pair SKIP
- verify slow periodic control frames while session down SKIP
- stale echo packets don't keep a session up SKIP
- no packets when zero remote required min rx interval SKIP
-
-BFD IPv6
-````````
-::
-
- ==============================================================================
- Bidirectional Forwarding Detection (BFD) (IPv6)
- ==============================================================================
- echo function used SKIP
- echo packets looped back SKIP
- hold BFD session up SKIP
- interface with bfd session deleted SKIP
- bring BFD session up SKIP
- bring BFD session up - first frame looked up by address pair SKIP
-
- ==============================================================================
- BFD-FIB interactions (IPv6)
- ==============================================================================
- BFD-FIB interactions SKIP
-
-BIER - Bit Indexed Explicit Replication
-```````````````````````````````````````
-::
-
- ==============================================================================
- BIER Test Case
- ==============================================================================
- BIER end-to-end OK
- BIER head OK
- BIER head over UDP OK
- BIER midpoint OK
- BIER Tail OK
- BIER Tail over UDP OK
-
- ==============================================================================
- BIER FIB Test Case
- ==============================================================================
- BFIB Unit Tests OK
-
-Classifier
-``````````
-::
-
- ==============================================================================
- Classifier Test Case
- ==============================================================================
- IP ACL test OK
- MAC ACL test OK
- IP PBR test OK
-
-Container Integration
-`````````````````````
-::
-
- ==============================================================================
- Container integration extended testcases
- ==============================================================================
- IPv4 basic connectivity test SKIP
- IPv6 basic connectivity test SKIP
- Create loopbacks overlapping with remote addresses SKIP
- IPv4 local-spoof connectivity test SKIP
- IPv6 local-spoof connectivity test SKIP
- Configure container commands SKIP
- IPv4 test after configuring container SKIP
- IPv6 test after configuring container SKIP
- Unconfigure container commands SKIP
- IPv4 local-spoof after unconfig test SKIP
- IPv6 local-spoof after unconfig test SKIP
-
-CRUD Loopback
-`````````````
-::
-
- ==============================================================================
- CRUD Loopback
- ==============================================================================
- test_crud (test_interface_crud.TestLoopbackInterfaceCRUD) OK
- test_down (test_interface_crud.TestLoopbackInterfaceCRUD) OK
-
-DHCP
-````
-::
-
- ==============================================================================
- DHCP Test Case
- ==============================================================================
- DHCPv6 Proxy OK
- DHCP Client OK
- DHCPv4 Proxy OK
-
-Distributed Virtual Router
-``````````````````````````
-::
-
- ==============================================================================
- Distributed Virtual Router
- ==============================================================================
- Distributed Virtual Router OK
- L2 Emulation OK
-
-DS-Lite Softwire
-````````````````
-::
-
- ==============================================================================
- DS-Lite Test Cases
- ==============================================================================
- Test DS-Lite OK
-
-FIB
-```
-::
-
- ==============================================================================
- FIB Test Case
- ==============================================================================
- FIB Unit Tests OK
-
-Flowprobe
-`````````
-::
-
- ==============================================================================
- Re-enable Flowprobe feature
- ==============================================================================
- disable flowprobe feature after first packets and re-enable SKIP
-
- ==============================================================================
- collect information on Ethernet, IP4 and IP6 datapath (no timers)
- ==============================================================================
- no timers, one CFLOW packet, 9 Flows inside OK
- no timers, two CFLOW packets (mtu=256), 3 Flows in each OK
- L2 data on IP4 datapath OK
- L2 data on IP6 datapath OK
- L2 data on L2 datapath OK
- L3 data on IP4 datapath OK
- L3 data on IP6 datapath OK
- L3 data on L2 datapath OK
- L4 data on IP4 datapath OK
- L4 data on IP6 datapath OK
- L4 data on L2 datapath OK
- verify templates on IP6 datapath OK
- verify templates on IP4 datapath OK
- verify template on L2 datapath OK
-
- ==============================================================================
- Disable Flowprobe feature
- ==============================================================================
- disable flowprobe feature after first packets SKIP
-
- ==============================================================================
- Re-enable IPFIX
- ==============================================================================
- disable IPFIX after first packets and re-enable after few packets SKIP
-
- ==============================================================================
- Disable IPFIX
- ==============================================================================
- disable IPFIX after first packets SKIP
-
-Geneve Tunnels
-``````````````
-::
-
- ==============================================================================
- GENEVE Test Case
- ==============================================================================
- Decapsulation test OK
- Encapsulation test OK
- Multicast flood test OK
- Multicast receive test OK
- Unicast flood test OK
-
-GRE Tunnels
-```````````
-::
-
- ==============================================================================
- GRE Test Case
- ==============================================================================
- GRE IPv4 tunnel Tests OK
- GRE IPv6 tunnel Tests OK
- GRE tunnel L2 Tests OK
- GRE tunnel VRF Tests OK
-
-GTPU Tunnels
-````````````
-::
-
- ==============================================================================
- GTPU Test Case
- ==============================================================================
- Decapsulation test OK
- Encapsulation test OK
- Multicast flood test OK
- Multicast receive test OK
- Unicast flood test OK
-
-IP Multicast Routing
-````````````````````
-::
-
- ==============================================================================
- IP Multicast Test Case
- ==============================================================================
- IP Multicast Bi-directional OK
- IPv6 Multicast Replication OK
- IPv6 Multicast Replication in non-default table OK
- IP Multicast Replication OK
- IP Multicast Connected Source check OK
- IP Multicast Signal OK
- IP Multicast Replication in non-default table OK
-
-IPSec
-`````
-::
-
- ==============================================================================
- Basic test for IPSEC using AH transport and Tunnel mode
- ==============================================================================
- ipsec ah v4 transport basic test OK
- ipsec ah v4 transport burst test OK
- ipsec ah 4o4 tunnel basic test OK
- ipsec ah 4o4 tunnel burst test OK
-
- ==============================================================================
- Basic test for ipsec esp sanity - tunnel and transport modes.
- ==============================================================================
- ipsec esp v4 transport basic test OK
- ipsec esp v4 transport burst test OK
- ipsec esp 4o4 tunnel basic test OK
- ipsec esp 4o4 tunnel burst test OK
-
-IPv4 FIB CRUD
-`````````````
-::
-
- ==============================================================================
- FIB - add/update/delete - ip4 routes
- ==============================================================================
- Add 1k routes OK
- Delete 100 routes OK
- Add 1k routes OK
- Delete 1.5k routes OK
-
-IPv4 Routing
-````````````
-::
-
- ==============================================================================
- IPv4 Test Case
- ==============================================================================
- IPv4 FIB test OK
-
- ==============================================================================
- IPv4 routes via NULL
- ==============================================================================
- IP NULL route OK
-
- ==============================================================================
- IPv4 disabled
- ==============================================================================
- IP Disabled OK
-
- ==============================================================================
- IPv4 Subnets
- ==============================================================================
- IP Sub Nets OK
-
- ==============================================================================
- IPv4 VLAN-0
- ==============================================================================
- IP VLAN-0 OK
-
- ==============================================================================
- IPv4 Load-Balancing
- ==============================================================================
- IP Load-Balancing OK
-
- ==============================================================================
- IPv4 Deaggregate Routes
- ==============================================================================
- IP Deag Routes OK
-
- ==============================================================================
- IPv4 Input Exceptions
- ==============================================================================
- IP Input Exceptions OK
-
- ==============================================================================
- IPv4 Punt Police/Redirect
- ==============================================================================
- IP punt police and redirect OK
-
-IPv4 VRF Multi-instance
-```````````````````````
-::
-
- ==============================================================================
- IP4 VRF Multi-instance Test Case
- ==============================================================================
- IP4 VRF Multi-instance test 1 - create 5 BDs OK
- IP4 VRF Multi-instance test 2 - delete 2 VRFs OK
- IP4 VRF Multi-instance 3 - add 2 VRFs OK
- IP4 VRF Multi-instance test 4 - delete 4 VRFs OK
-
-IPv6 Routing
-````````````
-::
-
- ==============================================================================
- IPv6 Test Case
- ==============================================================================
- IPv6 FIB test OK
- IPv6 Neighbour Solicitation Exceptions OK
- ND Duplicates OK
- IPv6 Router Solicitation Exceptions OK
-
- ==============================================================================
- IPv6 Punt Police/Redirect
- ==============================================================================
- IP6 punt police and redirect OK
-
- ==============================================================================
- IPv6 disabled
- ==============================================================================
- IP Disabled OK
-
- ==============================================================================
- IPv6 ND ProxyTest Case
- ==============================================================================
- IPv6 Proxy ND OK
-
- ==============================================================================
- IPv6 Load-Balancing
- ==============================================================================
- IPv6 Load-Balancing OK
-
- ==============================================================================
- IPv6 routes via NULL
- ==============================================================================
- IP NULL route OK
-
- ==============================================================================
- IPv6 Input Exceptions
- ==============================================================================
- IP6 Input Exceptions OK
-
-IPv6 VRF Multi-instance
-```````````````````````
-::
-
- ==============================================================================
- IP6 VRF Multi-instance Test Case
- ==============================================================================
- IP6 VRF Multi-instance test 1 - create 4 VRFs OK
- IP6 VRF Multi-instance test 2 - reset 2 VRFs OK
- IP6 VRF Multi-instance 3 - add 2 VRFs OK
- IP6 VRF Multi-instance test 4 - reset 4 VRFs OK
-
-IRB Integrated Routing-Bridging
-```````````````````````````````
-::
-
- ==============================================================================
- IRB Test Case
- ==============================================================================
- IPv4 IRB test 1 OK
- IPv4 IRB test 2 OK
-
-Kube-proxy
-``````````
-::
-
- ==============================================================================
- Kube-proxy Test Case
- ==============================================================================
- Kube-proxy NAT44 OK
- Kube-proxy NAT46 SKIP
- Kube-proxy NAT64 SKIP
- Kube-proxy NAT66 SKIP
-
-L2 FIB CRUD
-```````````
-::
-
- ==============================================================================
- L2 FIB Test Case
- ==============================================================================
- L2 FIB - program 100 + 100 MACs OK
- L2 FIB - program 100 + delete 12 MACs OK
- L2 FIB - flush all OK
- L2 FIB - flush BD OK
- L2 FIB - flush interface OK
- L2 FIB - mac learning events OK
- L2 FIB - mac learning max macs in event OK
- L2 FIB - program 100 MACs OK
- L2 FIB - Program 10 MACs, learn 10 OK
-
-L2BD Multi-instance
-```````````````````
-::
-
- ==============================================================================
- L2BD Multi-instance Test Case
- ==============================================================================
- L2BD Multi-instance test 1 - create 5 BDs OK
- L2BD Multi-instance test 2 - update data of 5 BDs OK
- L2BD Multi-instance test 3 - delete 2 BDs OK
- L2BD Multi-instance test 4 - add 2 BDs OK
- L2BD Multi-instance test 5 - delete 5 BDs SKIP
-
-L2BD Switching
-``````````````
-::
-
- ==============================================================================
- L2BD Test Case
- ==============================================================================
- L2BD MAC learning dual-loop test OK
- L2BD MAC learning single-loop test OK
-
-L2XC Multi-instance
-```````````````````
-::
-
- ==============================================================================
- L2XC Multi-instance Test Case
- ==============================================================================
- L2XC Multi-instance test 1 - create 10 cross-connects OK
- L2XC Multi-instance test 2 - delete 4 cross-connects OK
- L2BD Multi-instance 3 - add new 4 cross-connects OK
- L2XC Multi-instance test 4 - delete 10 cross-connects OK
-
-L2XC Switching
-``````````````
-::
-
- ==============================================================================
- L2XC Test Case
- ==============================================================================
- L2XC dual-loop test OK
- L2XC single-loop test OK
-
-LISP Tunnels
-````````````
-::
-
- ==============================================================================
- Basic LISP test
- ==============================================================================
- Test case for basic encapsulation OK
-
-Load Balancer
-`````````````
-::
-
- ==============================================================================
- Load Balancer Test Case
- ==============================================================================
- Load Balancer IP4 GRE4 OK
- Load Balancer IP4 GRE6 OK
- Load Balancer IP6 GRE4 OK
- Load Balancer IP6 GRE6 OK
-
-MACIP Access Control
-````````````````````
-::
-
- ==============================================================================
- MACIP Tests
- ==============================================================================
- MACIP 10 ACLs each with 100+ entries OK
- MACIP 10 ACLs each with 100+ entries with IP4 traffic OK
- MACIP 10 ACLs each with 100+ entries with IP6 traffic OK
- MACIP ACL with 10 entries OK
- MACIP ACL with 100 entries OK
- MACIP ACL with 2 entries OK
- MACIP ACL with 20 entries OK
- MACIP ACL with 5 entries OK
- MACIP ACL with 50 entries OK
- MACIP 2 ACLs each with 100+ entries OK
- MACIP replace ACL OK
- MACIP ACL delete intf with acl OK
-
- ==============================================================================
- MACIP with IP6 traffic
- ==============================================================================
- IP6 MACIP exactMAC|exactIP ACL bridged traffic OK
- IP6 MACIP exactMAC|subnetIP ACL bridged traffic OK
- IP6 MACIP exactMAC|wildIP ACL bridged traffic OK
- IP6 MACIP oui_MAC|exactIP ACL bridged traffic OK
- IP6 MACIP ouiMAC|subnetIP ACL bridged traffic OK
- IP6 MACIP ouiMAC|wildIP ACL bridged traffic OK
- IP6 MACIP wildcardMAC|exactIP ACL bridged traffic OK
- IP6 MACIP wildcardMAC|subnetIP ACL bridged traffic OK
- IP6 MACIP wildcardMAC|wildIP ACL bridged traffic OK
- MACIP replace ACL with IP6 traffic OK
- IP6 MACIP exactMAC|exactIP ACL routed traffic OK
- IP6 MACIP exactMAC|subnetIP ACL routed traffic OK
- IP6 MACIP exactMAC|wildIP ACL routed traffic OK
- IP6 MACIP ouiMAC|exactIP ACL routed traffic OK
- IP6 MACIP ouiMAC|subnetIP ACL routed traffic OK
- IP6 MACIP ouiMAC|wildIP ACL routed traffic OK
- IP6 MACIP wildcardMAC|exactIP ACL routed traffic OK
- IP6 MACIP wildcardMAC|subnetIP ACL routed traffic OK
- IP6 MACIP wildcardMAC|wildIP ACL OK
-
- ==============================================================================
- MACIP with IP4 traffic
- ==============================================================================
- IP4 MACIP wildcardMAC|exactIP ACL bridged traffic OK
- IP4 MACIP exactMAC|exactIP ACL bridged traffic OK
- IP4 MACIP exactMAC|subnetIP ACL bridged traffic OK
- IP4 MACIP exactMAC|wildIP ACL bridged traffic OK
- IP4 MACIP ouiMAC|exactIP ACL bridged traffic OK
- IP4 MACIP ouiMAC|subnetIP ACL bridged traffic OK
- IP4 MACIP ouiMAC|wildIP ACL bridged traffic OK
- IP4 MACIP wildcardMAC|subnetIP ACL bridged traffic OK
- IP4 MACIP wildcardMAC|wildIP ACL bridged traffic OK
- MACIP replace ACL with IP4 traffic OK
- IP4 MACIP exactMAC|exactIP ACL routed traffic OK
- IP4 MACIP exactMAC|subnetIP ACL routed traffic OK
- IP4 MACIP exactMAC|wildIP ACL routed traffic OK
- IP4 MACIP ouiMAC|exactIP ACL routed traffic OK
- IP4 MACIP ouiMAC|subnetIP ACL routed traffic OK
- IP4 MACIP ouiMAC|wildIP ACL routed traffic OK
- IP4 MACIP wildcardMAC|exactIP ACL routed traffic OK
- IP4 MACIP wildcardMAC|subnetIP ACL routed traffic OK
- IP4 MACIP wildcardMAC|wildIP ACL OK
-
-MAP Softwires
-`````````````
-::
-
- ==============================================================================
- MAP Test Case
- ==============================================================================
- MAP-E OK
-
-MFIB Multicast FIB
-``````````````````
-::
-
- ==============================================================================
- MFIB Test Case
- ==============================================================================
- MFIB Unit Tests OK
-
-MPLS Switching
-``````````````
-::
-
- ==============================================================================
- MPLS-L2
- ==============================================================================
- Virtual Private LAN Service OK
- Virtual Private Wire Service OK
-
- ==============================================================================
- MPLS Test Case
- ==============================================================================
- MPLS Local Label Binding test OK
- MPLS Deagg OK
- MPLS label imposition test OK
- MPLS Interface Receive OK
- MPLS Multicast Head-end OK
- MPLS IPv4 Multicast Tail OK
- MPLS IPv6 Multicast Tail OK
- MPLS Multicast Mid Point OK
- MPLS label swap tests OK
- MPLS Tunnel Tests OK
- MPLS V4 Explicit NULL test OK
- MPLS V6 Explicit NULL test OK
-
- ==============================================================================
- MPLS PIC edge convergence
- ==============================================================================
- MPLS eBGP PIC edge convergence OK
- MPLS iBGP PIC edge convergence OK
- MPLSv6 eBGP PIC edge convergence OK
-
- ==============================================================================
- MPLS disabled
- ==============================================================================
- MPLS Disabled OK
-
-NAT44
-`````
-::
-
- ==============================================================================
- NAT44 Test Cases
- ==============================================================================
- Delete NAT44 session OK
- NAT44 dynamic translation test OK
- NAT44 handling of client packets with TTL=1 OK
- NAT44 handling of error responses to client packets with TTL=2 OK
- NAT44 handling of server packets with TTL=1 OK
- NAT44 handling of error responses to server packets with TTL=2 OK
- NAT44 interfaces without configured IP address OK
- NAT44 forwarding test OK
- NAT44 translate fragments arriving in order OK
- NAT44 translate fragments arriving out of order OK
- NAT44 hairpinning - 1:1 NAPT OK
- NAT44 hairpinning - 1:1 NAT OK
- 1:1 NAT translate packet with unknown protocol - hairpinning OK
- NAT44 translate packet with unknown protocol - hairpinning OK
- Identity NAT OK
- NAT44 multiple inside interfaces with overlapping address space OK
- Acquire NAT44 addresses from interface OK
- Identity NAT with addresses from interface OK
- Static mapping with addresses from interface OK
- IPFIX logging NAT addresses exhausted OK
- IPFIX logging NAT44 session created/delted OK
- MAX translations per user - recycle the least recently used OK
- NAT44 multiple non-overlapping address space inside interfaces OK
- One armed NAT44 OK
- NAT44 interface output feature (in2out postrouting) OK
- NAT44 interface output feature hairpinning (in2out postrouting) OK
- NAT44 interface output feature VRF aware (in2out postrouting) OK
- Ping internal host from outside network OK
- Ping NAT44 out interface from outside network OK
- NAT44 add pool addresses to FIB OK
- Port restricted NAT44 (MAP-E CE) OK
- NAT44 fragments hairpinning OK
- NAT44 set/get virtual fragmentation reassembly OK
- 1:1 NAT initialized from inside network OK
- NAT44 interfaces without configured IP address - 1:1 NAT OK
- NAT44 local service load balancing OK
- 1:1 NAT initialized from outside network OK
- 1:1 NAT translate packet with unknown protocol OK
- 1:1 NAT VRF awareness OK
- 1:1 NAPT initialized from inside network OK
- NAT44 interfaces without configured IP address - 1:1 NAPT OK
- 1:1 NAPT initialized from outside network OK
- Twice NAT44 OK
- Acquire twice NAT44 addresses from interface OK
- Twice NAT44 local service load balancing OK
- NAT44 translate packet with unknown protocol OK
- NAT44 tenant VRF independent address pool mode OK
- NAT44 tenant VRF aware address pool mode OK
-
- ==============================================================================
- Deterministic NAT Test Cases
- ==============================================================================
- Deterministic NAT translation test (TCP, UDP, ICMP) OK
- NAT plugin run deterministic mode OK
- Deterministic NAT multiple users OK
- Deterministic NAT maximum sessions per user limit SKIP
- Deterministic NAT session timeouts SKIP
- Set deterministic NAT timeouts OK
- Deterministic NAT TCP session close from inside network OK
- Deterministic NAT TCP session close from outside network OK
-
-NAT64
-`````
-::
-
- ==============================================================================
- NAT64 Test Cases
- ==============================================================================
- NAT64 dynamic translation test OK
- NAT64 translate fragments arriving in order OK
- NAT64 translate fragments arriving out of order OK
- NAT64 hairpinning OK
- NAT64 translate packet with unknown protocol - hairpinning OK
- NAT64 ICMP Error message translation OK
- Enable/disable NAT64 feature on the interface OK
- Acquire NAT64 pool addresses from interface OK
- One armed NAT64 OK
- Add/delete address to NAT64 pool OK
- NAT64 Network-Specific Prefix OK
- NAT64 fragments hairpinning OK
- NAT64 session timeout SKIP
- Set NAT64 timeouts OK
- NAT64 static translation test OK
- Add/delete static BIB entry OK
- NAT64 translate packet with unknown protocol OK
-
-P2P Ethernet Subinterface
-`````````````````````````
-::
-
- ==============================================================================
- P2P Ethernet tests
- ==============================================================================
- delete/create p2p subif OK
- create 1k of p2p subifs OK
-
- ==============================================================================
- P2P Ethernet IPv4 tests
- ==============================================================================
- receive ipv4 packet via p2p subinterface OK
- route rx packet not matching p2p subinterface OK
- send ip4 packet via p2p subinterface OK
- drop tx ip4 packet not matching p2p subinterface OK
-
- ==============================================================================
- P2P Ethernet IPv6 tests
- ==============================================================================
- receive ipv6 packet via p2p subinterface OK
- drop rx packet not matching p2p subinterface OK
- route rx ip6 packet not matching p2p subinterface OK
- send packet via p2p subinterface OK
- drop tx ip6 packet not matching p2p subinterface OK
- standard routing without p2p subinterfaces OK
-
-PPPoE Encapsulation
-```````````````````
-::
-
- ==============================================================================
- PPPoE Test Case
- ==============================================================================
- PPPoE Add Same Session Twice Test OK
- PPPoE Decap Test OK
- PPPoE Decap Multiple Sessions Test OK
- PPPoE Delete Same Session Twice Test OK
- PPPoE Encap Test OK
- PPPoE Encap Multiple Sessions Test OK
-
-SPAN Switch Port Analyzer
-`````````````````````````
-::
-
- ==============================================================================
- SPAN Test Case
- ==============================================================================
- SPAN device rx mirror OK
- SPAN l2 broadcast mirror OK
- SPAN l2 rx tx mirror OK
- SPAN l2 tx mirror OK
- SPAN l2 rx mirror OK
- SPAN l2 rx mirror into 1ad subif+vtr OK
- SPAN l2 rx mirror into 1q subif+vtr OK
- SPAN l2 rx mirror into gre-subif+vtr OK
- SPAN l2 rx mirror into vxlan OK
-
-SRv6 Routing
-````````````
-::
-
- ==============================================================================
- SRv6 Test Case
- ==============================================================================
- Test SRv6 End (without PSP) behavior. OK
- Test SRv6 End.DT4 behavior. OK
- Test SRv6 End.DT6 behavior. OK
- Test SRv6 End.DX2 behavior. OK
- Test SRv6 End.DX4 behavior. OK
- Test SRv6 End.DX6 behavior. OK
- Test SRv6 End.X (without PSP) behavior. OK
- Test SRv6 End.X with PSP behavior. OK
- Test SRv6 End with PSP behavior. OK
- Test SRv6 Transit.Encaps behavior for IPv6. OK
- Test SRv6 Transit.Encaps behavior for IPv4. OK
- Test SRv6 Transit.Encaps behavior for L2. SKIP
- Test SRv6 Transit.Insert behavior (IPv6 only). OK
- Test SRv6 Transit.Insert behavior (IPv6 only). OK
-
-TCP/IP Stack
-````````````
-::
-
- ==============================================================================
- TCP Test Case
- ==============================================================================
- TCP builtin client/server transfer OK
- TCP Unit Tests OK
-
-UDP Stack
-`````````
-::
-
- ==============================================================================
- UDP Encap Test Case
- ==============================================================================
- UDP Encap test OK
-
-VTR VLAN Tag Rewrites
-`````````````````````
-::
-
- ==============================================================================
- VTR Test Case
- ==============================================================================
- 1AD VTR pop 1 test OK
- 1AD VTR pop 2 test OK
- 1AD VTR push 1 1AD test OK
- 1AD VTR push 1 1Q test OK
- 1AD VTR push 2 1AD test OK
- 1AD VTR push 2 1Q test OK
- 1AD VTR translate 1 -> 1 1AD test OK
- 1AD VTR translate 1 -> 1 1Q test OK
- 1AD VTR translate 1 -> 2 1AD test OK
- 1AD VTR translate 1 -> 2 1Q test OK
- 1AD VTR translate 2 -> 1 1AD test OK
- 1AD VTR translate 2 -> 1 1Q test OK
- 1AD VTR translate 2 -> 2 1AD test OK
- 1AD VTR translate 2 -> 2 1Q test OK
- 1Q VTR pop 1 test OK
- 1Q VTR push 1 test OK
- 1Q VTR push 2 test OK
- 1Q VTR translate 1 -> 1 test OK
- 1Q VTR translate 1 -> 2 test OK
-
-VXLAN Tunnels
-`````````````
-::
-
- ==============================================================================
- VXLAN Test Case
- ==============================================================================
- Decapsulation test OK
- Encapsulation test OK
- Multicast flood test OK
- Multicast receive test OK
- Unicast flood test OK
-
-VXLAN-GPE Tunnels
-`````````````````
-::
-
- ==============================================================================
- VXLAN-GPE Test Case
- ==============================================================================
- Decapsulation test SKIP
- Encapsulation test SKIP
- Multicast flood test SKIP
- Multicast receive test SKIP
- Unicast flood test SKIP
-
-Other Tests
-```````````
-::
-
- ==============================================================================
- Ping Test Case
- ==============================================================================
- basic ping test OK
- burst ping test OK
-
- ==============================================================================
- Session Test Case
- ==============================================================================
- Session Unit Tests OK
-
- ==============================================================================
- Template verification, timer tests
- ==============================================================================
- timer less than template timeout OK
- timer greater than template timeout OK
- verify cflow packet fields OK
-
diff --git a/docs/report/dpdk_performance_tests/index.rst b/docs/report/dpdk_performance_tests/index.rst
deleted file mode 100644
index 44d34d3d4d..0000000000
--- a/docs/report/dpdk_performance_tests/index.rst
+++ /dev/null
@@ -1,12 +0,0 @@
-DPDK Performance Tests
-======================
-
-.. toctree::
-
- overview
- csit_release_notes
- packet_throughput_graphs/index
- packet_latency_graphs/index
- test_environment
- documentation
-
diff --git a/docs/report/honeycomb_functional_tests/index.rst b/docs/report/honeycomb_functional_tests/index.rst
deleted file mode 100644
index 424ad3640f..0000000000
--- a/docs/report/honeycomb_functional_tests/index.rst
+++ /dev/null
@@ -1,10 +0,0 @@
-Honeycomb Functional Tests
-==========================
-
-.. toctree::
-
- overview
- csit_release_notes
- test_environment
- documentation
-
diff --git a/docs/report/index.rst b/docs/report/index.rst
index 46b5ecec19..410a245dcb 100644
--- a/docs/report/index.rst
+++ b/docs/report/index.rst
@@ -2,15 +2,100 @@ CSIT 18.07
==========
.. toctree::
- :numbered:
-
- introduction/index
- vpp_performance_tests/index
- dpdk_performance_tests/index
- vpp_functional_tests/index
- honeycomb_functional_tests/index
- nsh_sfc_functional_tests/index
- detailed_test_results/index
- test_configuration/index
- test_operational_data/index
- csit_framework_documentation/index
+ :maxdepth: 2
+ :caption: Introduction
+
+ introduction/overview
+ introduction/general_notes
+
+.. toctree::
+ :maxdepth: 2
+ :caption: VPP Performance Tests
+
+ vpp_performance_tests/overview
+ vpp_performance_tests/csit_release_notes
+ vpp_performance_tests/methodology
+ vpp_performance_tests/packet_throughput_graphs/index
+ vpp_performance_tests/throughput_speedup_multi_core/index
+ vpp_performance_tests/packet_latency_graphs/index
+ vpp_performance_tests/http_server_performance/index
+ vpp_performance_tests/mdr_search
+ vpp_performance_tests/test_environment
+ vpp_performance_tests/documentation/index
+
+.. toctree::
+ :maxdepth: 2
+ :caption: DPDK Performance Tests
+
+ dpdk_performance_tests/overview
+ dpdk_performance_tests/csit_release_notes
+ dpdk_performance_tests/packet_throughput_graphs/index
+ dpdk_performance_tests/packet_latency_graphs/index
+ dpdk_performance_tests/test_environment
+ dpdk_performance_tests/documentation
+
+.. toctree::
+ :maxdepth: 2
+ :caption: VPP Functional Tests
+
+ vpp_functional_tests/overview
+ vpp_functional_tests/csit_release_notes
+ vpp_functional_tests/test_environment
+ vpp_functional_tests/documentation
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Honeycomb Functional Tests
+
+ honeycomb_functional_tests/overview
+ honeycomb_functional_tests/csit_release_notes
+ honeycomb_functional_tests/test_environment
+ honeycomb_functional_tests/documentation
+
+.. toctree::
+ :maxdepth: 2
+ :caption: NSH_SFC Functional Tests
+
+ nsh_sfc_functional_tests/overview
+ nsh_sfc_functional_tests/csit_release_notes
+ nsh_sfc_functional_tests/test_environment
+ nsh_sfc_functional_tests/documentation
+
+.. only:: html
+
+ .. toctree::
+ :maxdepth: 2
+ :caption: Detailed Test Results
+
+ detailed_test_results/vpp_performance_results/index
+ detailed_test_results/vpp_mrr_results/index
+ detailed_test_results/cot_performance_results/index
+ detailed_test_results/dpdk_performance_results/index
+ detailed_test_results/vpp_functional_results/index
+ detailed_test_results/vpp_functional_results_centos/index
+ detailed_test_results/honeycomb_functional_results/index
+ detailed_test_results/nshsfc_functional_results/index
+
+ .. toctree::
+ :maxdepth: 2
+ :caption: Test Configuration
+
+ test_configuration/vpp_performance_configuration/index
+ test_configuration/vpp_mrr_configuration/index
+ test_configuration/vpp_functional_configuration/index
+ test_configuration/vpp_functional_configuration_centos/index
+
+ .. toctree::
+ :maxdepth: 2
+ :caption: Test Operational Data
+
+ test_operational_data/vpp_performance_operational_data/index
+
+.. toctree::
+ :maxdepth: 2
+ :caption: CSIT Framework Documentation
+
+ csit_framework_documentation/csit_design
+ csit_framework_documentation/csit_test_naming
+ csit_framework_documentation/pal_lld
+ csit_framework_documentation/csit_tag_description
diff --git a/docs/report/introduction/index.rst b/docs/report/introduction/index.rst
deleted file mode 100644
index a5c7f5acff..0000000000
--- a/docs/report/introduction/index.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-Introduction
-============
-
-.. toctree::
-
- overview
- general_notes
diff --git a/docs/report/nsh_sfc_functional_tests/index.rst b/docs/report/nsh_sfc_functional_tests/index.rst
deleted file mode 100644
index c7678d26a0..0000000000
--- a/docs/report/nsh_sfc_functional_tests/index.rst
+++ /dev/null
@@ -1,10 +0,0 @@
-NSH_SFC Functional Tests
-========================
-
-.. toctree::
-
- overview
- csit_release_notes
- test_environment
- documentation
-
diff --git a/docs/report/test_configuration/index.rst b/docs/report/test_configuration/index.rst
deleted file mode 100644
index 8c231a85cd..0000000000
--- a/docs/report/test_configuration/index.rst
+++ /dev/null
@@ -1,12 +0,0 @@
-.. only:: html
-
-
- Test Configuration
- ==================
-
- .. toctree::
-
- vpp_performance_configuration/index
- vpp_mrr_configuration/index
- vpp_functional_configuration/index
- vpp_functional_configuration_centos/index
diff --git a/docs/report/test_operational_data/index.rst b/docs/report/test_operational_data/index.rst
deleted file mode 100644
index a85e8eaea2..0000000000
--- a/docs/report/test_operational_data/index.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-.. only:: html
-
- Test Operational Data
- =====================
-
- .. toctree::
-
- vpp_performance_operational_data/index
diff --git a/docs/report/vpp_functional_tests/index.rst b/docs/report/vpp_functional_tests/index.rst
deleted file mode 100644
index 5111296c78..0000000000
--- a/docs/report/vpp_functional_tests/index.rst
+++ /dev/null
@@ -1,10 +0,0 @@
-VPP Functional Tests
-====================
-
-.. toctree::
-
- overview
- csit_release_notes
- test_environment
- documentation
-
diff --git a/docs/report/vpp_performance_tests/csit_release_notes.rst b/docs/report/vpp_performance_tests/csit_release_notes.rst
index 0309885c0d..afb5b36bce 100644
--- a/docs/report/vpp_performance_tests/csit_release_notes.rst
+++ b/docs/report/vpp_performance_tests/csit_release_notes.rst
@@ -103,6 +103,19 @@ CSV and pretty ASCII formats:
with Robot Framework result files csit-vpp-perf-mrr-daily-master__*__output.xml.gz
`archived here <https://docs.fd.io/csit/master/trending/_static/archive/>`_.
+Comparison Across Testbeds
+--------------------------
+
+.. warning::
+
+ TODO: Add:
+
+ Table 1.
+ Test Case 3-Node Hsw 3-Node Skx Skx vs. Hsw Delta [%]
+
+ Table 2.
+ Test Case 3-Node Skx 2-Node Skx 2-Node vs. 3-Node Delta [%]
+
Throughput Trending
-------------------
diff --git a/docs/report/vpp_performance_tests/index.rst b/docs/report/vpp_performance_tests/index.rst
deleted file mode 100644
index aa4539dcfa..0000000000
--- a/docs/report/vpp_performance_tests/index.rst
+++ /dev/null
@@ -1,15 +0,0 @@
-VPP Performance Tests
-=====================
-
-.. toctree::
-
- overview
- csit_release_notes
- methodology
- packet_throughput_graphs/index
- throughput_speedup_multi_core/index
- packet_latency_graphs/index
- http_server_performance/index
- mdr_search
- test_environment
- documentation/index
diff --git a/docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst b/docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst
index bc9ceeae31..dca77c5b7b 100644
--- a/docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst
+++ b/docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst
@@ -1,3 +1,5 @@
+.. _throughput_speedup_multi_core:
+
Throughput Speedup Multi-Core
=============================
diff --git a/docs/report/vpp_unit_tests/documentation.rst b/docs/report/vpp_unit_tests/documentation.rst
deleted file mode 100644
index b6bc0272a8..0000000000
--- a/docs/report/vpp_unit_tests/documentation.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-Documentation
-=============
-
-For complete description of the VPP Test Framework including anatomy of a test
-case and detailed documentation of existing VPP unit test cases please refer
-to the `VPP test framework documentation`_.
diff --git a/docs/report/vpp_unit_tests/index.rst b/docs/report/vpp_unit_tests/index.rst
deleted file mode 100644
index 525eb1230d..0000000000
--- a/docs/report/vpp_unit_tests/index.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-VPP Unit Tests
-==============
-
-.. toctree::
-
- overview
- documentation
-
diff --git a/docs/report/vpp_unit_tests/overview.rst b/docs/report/vpp_unit_tests/overview.rst
deleted file mode 100644
index 6165c71c50..0000000000
--- a/docs/report/vpp_unit_tests/overview.rst
+++ /dev/null
@@ -1,87 +0,0 @@
-Overview
-========
-
-.. note::
-
- This section includes an abbreviated version of the VPP Test Framework
- overview maintained within the VPP project. Complete overview can be found
- in `VPP test framework documentation`_.
-
-VPP Unit Test Framework
------------------------
-
-VPP Test Framework is used to ease writing, running and debugging unit tests
-for the VPP. It is based on python as a high level language to allow rapid
-test development. scapy\_ is used as a tool for creating and dissecting
-packets.
-
-VPP Test Framework does not send any packets to VPP directly. Traffic is
-instead injected using VPP packet-generator interfaces. Packets are written
-into a temporary .pcap file, which is then read by the VPP code with packets
-getting injected into the VPP processing nodes.
-
-Similarly, VPP does not send any packets to VPP Test Framework directly.
-Instead, VPP packet capture feature is used to capture and write packets to a
-temporary .pcap file, which is then read and analyzed by the VPP Test
-Framework.
-
-For complete description of the VPP Test Framework including anatomy of a test
-case and detailed documentation of existing VPP unit test cases please refer
-to the `VPP test framework documentation`_
-
-Unit Tests Coverage
--------------------
-
-Following VPP functional test areas are covered in VPP unit test code included
-in VPP rls1710 with results listed in this report:
-
-- ACL Security - stateful and stateless security-groups access-control-lists.
-- APIs - VAPI, VOM, PAPI, JVPP.
-- ARP - ARP, proxy ARP, static arp.
-- BFD - API, Authentication, Authentication Change, CLI.
-- BFD IPv4 - sessions operation.
-- BFD IPv6 - sessions operation.
-- BIER - Bit Indexed Explicit Replication.
-- Classifier - classification with IP ACL, MAC ACL, IP PBR.
-- Container Integration - IPv4, IPv6 local-spoof connectivity tests.
-- CRUD Loopback - create, read, update, delete Loopback interfaces.
-- DHCP - DHCPv4/v6 Client and Proxy.
-- Distributed Virtual Router.
-- DS-Lite Softwire - softwire termination.
-- FIB - baseline and scale tests.
-- Flowprobe.
-- Geneve Tunnels.
-- GRE Tunnels - GRE IPv4/IPv6 tunnel, L2, VRF tests.
-- GTPU Tunnels - baseline GTPU tests.
-- IP Multicast Routing - IPv4/IPv6 multicast replication, connected source check.
-- IPSec - baseline IPSec sanity tests.
-- IPv4 FIB CRUD - add/update/delete IPv4 routes.
-- IPv4 Routing.
-- IP4 VRF Multi-instance - create, read, update, delete and verify IPv4 VRFs.
-- IPv6 Routing - baseline FIB operations, NS/RS exception handling.
-- IP6 VRF Multi-instance - create, read, update, delete and verify IPv6 VRFs.
-- IRB Integrated Routing-Bridging.
-- Kube-proxy - data plane NAT tests.
-- L2 FIB CRUD - add/update/delete L2 MAC entries.
-- L2BD Multi-instance.
-- L2BD Switching - L2 Bridge-Domain baseline tests incl. single- and dual-loop.
-- L2XC Multi-instance - L2 cross-connect multi-instance tests.
-- L2XC Switching - L2 cross-connect baseline tests incl. single- and dual-loop.
-- LISP Tunnels - basic LISP tests.
-- Load Balancer - IP4 GRE4, IP4 GRE6, IP6 GRE4, IP6 GRE6.
-- MACIP Access Control - ingress access control for IPv4, IPv6 with L2BDP and IP routing.
-- MAP Softwires - softwire termination.
-- MFIB Multicast FIB.
-- MPLS Switching - MPLS baseline, prefix independent convergence for MPLS PE.
-- NAT44 - NAT44 tests, IPFIX logging, VRF awareness, deterministic CGNAT.
-- NAT64 - NAT64 static and dynamic translation tests.
-- P2P Ethernet Subinterface.
-- PPPoE Encapsulation.
-- SPAN Switch Port Analyzer - packet mirroring.
-- SRv6 Routing - Segment Routing IPv6 tests.
-- TCP/IP Stack - unit tests, builtin client/server transfers.
-- UDP Stack - unit tests.
-- VTR VLAN Tag Rewrites - VLAN tag rewrite tests.
-- VXLAN Tunnels - baseline VXLAN tests including multicast.
-- VXLAN-GPE Tunnels - baseline VXLAN-GPE tunneling including multicast.
-- Other Tests - ping, session, template verification, timer tests.
diff --git a/resources/tools/presentation/conf.py b/resources/tools/presentation/conf.py
index 78856302f3..32a6908d5c 100644
--- a/resources/tools/presentation/conf.py
+++ b/resources/tools/presentation/conf.py
@@ -45,7 +45,7 @@ source_suffix = ['.rst', '.md']
master_doc = 'index'
# General information about the project.
-project = u'FD.io CSIT'
+project = u'FD.io CSIT rls1807 Report'
copyright = u'2018, FD.io'
author = u'FD.io CSIT'
diff --git a/resources/tools/presentation/generator_report.py b/resources/tools/presentation/generator_report.py
index 07103dbb1f..8e77ba7502 100644
--- a/resources/tools/presentation/generator_report.py
+++ b/resources/tools/presentation/generator_report.py
@@ -41,6 +41,49 @@ THEME_OVERRIDES = """/* override table width restrictions */
line-height: 18px;
margin-bottom: 0px;
}
+.wy-menu-vertical a {
+ display: inline-block;
+ line-height: 18px;
+ padding: 0 2em;
+ display: block;
+ position: relative;
+ font-size: 90%;
+ color: #d9d9d9
+}
+.wy-menu-vertical li.current a {
+ color: gray;
+ border-right: solid 1px #c9c9c9;
+ padding: 0 3em;
+}
+.wy-menu-vertical li.toctree-l2.current > a {
+ background: #c9c9c9;
+ padding: 0 3em;
+}
+.wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
+ display: block;
+ background: #c9c9c9;
+ padding: 0 4em;
+}
+.wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
+ display: block;
+ background: #bdbdbd;
+ padding: 0 5em;
+}
+.wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
+ color: #404040;
+ padding: 0 2em;
+ font-weight: bold;
+ position: relative;
+ background: #fcfcfc;
+ border: none;
+ border-top-width: medium;
+ border-bottom-width: medium;
+ border-top-style: none;
+ border-bottom-style: none;
+ border-top-color: currentcolor;
+ border-bottom-color: currentcolor;
+ padding-left: 2em -4px;
+}
"""
# Command to build the html format of the report
@@ -48,7 +91,7 @@ HTML_BUILDER = 'sphinx-build -v -c . -a ' \
'-b html -E ' \
'-t html ' \
'-D release={release} ' \
- '-D version="{release} report - {date}" ' \
+ '-D version="Report v{report_version} - {date}" ' \
'{working_dir} ' \
'{build_dir}/'
@@ -57,18 +100,20 @@ PDF_BUILDER = 'sphinx-build -v -c . -a ' \
'-b latex -E ' \
'-t latex ' \
'-D release={release} ' \
- '-D version="{release} report - {date}" ' \
+ '-D version="Report v{report_version} - {date}" ' \
'{working_dir} ' \
'{build_dir}'
-def generate_report(release, spec):
+def generate_report(release, spec, report_version):
"""Generate all formats and versions of the report.
:param release: Release string of the product.
:param spec: Specification read from the specification file.
+ :param report_version: Version of the report.
:type release: str
:type spec: Specification
+ :type report_version: str
"""
logging.info("Generating the report ...")
@@ -79,7 +124,7 @@ def generate_report(release, spec):
}
for report_format, versions in spec.output["format"].items():
- report[report_format](release, spec, versions)
+ report[report_format](release, spec, versions, report_version)
archive_input_data(spec)
archive_report(spec)
@@ -87,15 +132,17 @@ def generate_report(release, spec):
logging.info("Done.")
-def generate_html_report(release, spec, versions):
+def generate_html_report(release, spec, versions, report_version):
"""Generate html format of the report.
:param release: Release string of the product.
:param spec: Specification read from the specification file.
:param versions: List of versions to generate.
+ :param report_version: Version of the report.
:type release: str
:type spec: Specification
:type versions: list
+ :type report_version: str
"""
logging.info(" Generating the html report, give me a few minutes, please "
@@ -103,6 +150,7 @@ def generate_html_report(release, spec, versions):
cmd = HTML_BUILDER.format(
release=release,
+ report_version=report_version,
date=datetime.datetime.utcnow().strftime('%m/%d/%Y %H:%M UTC'),
working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
@@ -119,15 +167,17 @@ def generate_html_report(release, spec, versions):
logging.info(" Done.")
-def generate_pdf_report(release, spec, versions):
+def generate_pdf_report(release, spec, versions, report_version):
"""Generate html format of the report.
:param release: Release string of the product.
:param spec: Specification read from the specification file.
:param versions: List of versions to generate. Not implemented yet.
+ :param report_version: Version of the report.
:type release: str
:type spec: Specification
:type versions: list
+ :type report_version: str
"""
logging.info(" Generating the pdf report, give me a few minutes, please "
@@ -148,6 +198,7 @@ def generate_pdf_report(release, spec, versions):
build_dir = spec.environment["paths"]["DIR[BUILD,LATEX]"]
cmd = PDF_BUILDER.format(
release=release,
+ report_version=report_version,
date=datetime.datetime.utcnow().strftime('%m/%d/%Y %H:%M UTC'),
working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
build_dir=build_dir)
@@ -163,10 +214,11 @@ def generate_pdf_report(release, spec, versions):
'pdflatex -interaction nonstopmode csit.tex || true'.
format(build_dir=build_dir),
'cd {build_dir} && '
- 'cp csit.pdf ../{archive_dir}/csit_{release}.pdf'.
+ 'cp csit.pdf ../{archive_dir}/csit_{release}_{report_version}.pdf'.
format(build_dir=build_dir,
archive_dir=archive_dir,
- release=release)
+ release=release,
+ report_version=report_version)
]
for cmd in cmds:
diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py
index 013c921124..73d3b757aa 100644
--- a/resources/tools/presentation/pal.py
+++ b/resources/tools/presentation/pal.py
@@ -48,6 +48,10 @@ def parse_args():
default="master",
type=str,
help="Release string of the product.")
+ parser.add_argument("-v", "--version",
+ default="0.1",
+ type=str,
+ help="Version of the product.")
parser.add_argument("-l", "--logging",
choices=["DEBUG", "INFO", "WARNING",
"ERROR", "CRITICAL"],
@@ -103,7 +107,7 @@ def main():
generate_files(spec, data)
if spec.output["output"] == "report":
- generate_report(args.release, spec)
+ generate_report(args.release, spec, args.version)
logging.info("Successfully finished.")
elif spec.output["output"] == "CPTA":
sys.stdout.write(generate_cpta(spec, data))
diff --git a/resources/tools/presentation/run_report.sh b/resources/tools/presentation/run_report.sh
index d294640fd1..8dcd17bbf9 100755
--- a/resources/tools/presentation/run_report.sh
+++ b/resources/tools/presentation/run_report.sh
@@ -38,6 +38,7 @@ export PYTHONPATH=`pwd`
python pal.py \
--specification specification.yaml \
--release ${RELEASE} \
+ --version "0.1" \
--logging INFO \
--force