From ee52db86af140da3c919476dcd1441acdc9e7600 Mon Sep 17 00:00:00 2001
From: selias <samelias@cisco.com>
Date: Wed, 9 Aug 2017 10:50:46 +0200
Subject: HC Test: cleanup test failures

 - have unnumbered interface cases use a real interface instead of local0
 - add EXPECTED_FAILING tag and Jira link to known failing test cases

Change-Id: I8bc0f4a74ea1949289a3f12dd54168acdf9edc39
Signed-off-by: selias <samelias@cisco.com>
---
 .../mgmt-cfg-int-subint-apihc-apivat-func.robot    | 41 ++++++++++++----------
 .../mgmt-cfg-intip4-intip6-apihc-apivat-func.robot | 30 ++++++++++------
 .../mgmt-cfg-routing-apihc-apivat-func.robot       |  4 +++
 .../func/honeycomb/mgmt-notif-apihcnc-func.robot   |  3 +-
 4 files changed, 47 insertions(+), 31 deletions(-)

(limited to 'tests')

diff --git a/tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot b/tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot
index a37f538d2d..b1a4304f87 100644
--- a/tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot
+++ b/tests/vpp/func/honeycomb/mgmt-cfg-int-subint-apihc-apivat-func.robot
@@ -19,9 +19,7 @@
 | Resource | resources/libraries/robot/honeycomb/interfaces.robot
 | Variables | resources/test_data/honeycomb/sub_interfaces.py
 | ...
-| Suite Setup | Run Keywords
-| ... | Set Up Honeycomb Functional Test Suite | ${node} | AND
-| ... | Add Interface local0 To Topology | ${node}
+| Suite Setup | Set Up Honeycomb Functional Test Suite | ${node}
 | ...
 | Suite Teardown | Tear Down Honeycomb Functional Test Suite | ${node}
 | ...
@@ -471,20 +469,22 @@
 
 | TC22: Honeycomb can configure unnumbered sub-interface
 | | [Documentation] | Check if Honeycomb can configure an unnumbered interface\
-| | ... | on a sub-interface, borrowing the IP address of 'local0'.
+| | ... | on a sub-interface, borrowing the IP address of a physical interface.
 | | ...
+# CSIT-734: Intermittent failures of all unnumbered interface cases
+| | [Tags] | EXPECTED_FAILING
 | | Given sub-interface ipv4 address from Honeycomb should be empty
 | | ... | ${node} | ${super_if} | ${sub_if_id}
 | | And sub-interface ipv4 address from VAT should be empty
 | | ... | ${node} | ${sub_if_name}
 | | And Honeycomb sets interface IPv4 address | ${node}
-| | ... | local0 | ${ipv4['address']} | ${ipv4['prefix-length']}
+| | ... | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']}
 | | When Honeycomb adds unnumbered configuration to interface
-| | ... | ${node} | ${super_if}.${sub_if_id} | local0
+| | ... | ${node} | ${super_if}.${sub_if_id} | ${super_if2}
 | | Then IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4['address']} | ${ipv4['prefix-length']}
+| | ... | ${node} | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0
+| | ... | ${node} | ${super_if2}
 | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | ${ipv4['netmask']}
 | | And sub-interface ipv4 address from Honeycomb should be
 | | ... | ${node} | ${super_if} | ${sub_if_id}
@@ -497,13 +497,15 @@
 | | [Documentation] | Check if Honeycomb can remove unnumbered configuration\
 | | ... | from a sub-interface.
 | | ...
+# CSIT-734: Intermittent failures of all unnumbered interface cases
+| | [Tags] | EXPECTED_FAILING
 | | [Teardown] | Honeycomb removes interface IPv4 addresses | ${node}
-| | ... | local0
+| | ... | ${super_if2}
 | | ...
 | | Given IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4['address']} | ${ipv4['prefix-length']}
+| | ... | ${node} | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0
+| | ... | ${node} | ${super_if2}
 | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | ${ipv4['netmask']}
 | | And sub-interface ipv4 address from Honeycomb should be
 | | ... | ${node} | ${super_if} | ${sub_if_id}
@@ -514,9 +516,9 @@
 | | When Honeycomb removes unnumbered configuration from interface
 | | ... | ${node} | ${super_if}.${sub_if_id}
 | | Then IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4['address']} | ${ipv4['prefix-length']}
+| | ... | ${node} | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0
+| | ... | ${node} | ${super_if2}
 | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | ${ipv4['netmask']}
 | | And sub-interface ipv4 address from Honeycomb should be empty
 | | ... | ${node} | ${super_if} | ${sub_if_id}
@@ -526,9 +528,10 @@
 | TC24: Honeycomb can configure unnumbered interface using a sub-interface
 | | [Documentation] | Check if Honeycomb can configure an unnumbered interface\
 | | ... | on an interface, borrowing the IP address of a sub-interface.
-| | ...
-| | Given IPv4 address from Honeycomb should be empty | ${node} | local0
-| | And ipv4 address from VAT should be empty | ${node} | local0
+| | ...# CSIT-734: Intermittent failures of all unnumbered interface cases
+| | [Tags] | EXPECTED_FAILING
+| | Given IPv4 address from Honeycomb should be empty | ${node} | ${super_if2}
+| | And ipv4 address from VAT should be empty | ${node} | ${super_if2}
 | | And sub-interface ipv4 address from Honeycomb should be empty
 | | ... | ${node} | ${super_if} | ${sub_if_id}
 | | And sub-interface ipv4 address from VAT should be empty
@@ -537,11 +540,11 @@
 | | ... | ${node} | ${super_if} | ${sub_if_id}
 | | ... | ${ipv4['address']} | ${ipv4['prefix-length']}
 | | When Honeycomb adds unnumbered configuration to interface
-| | ... | ${node} | local0 | ${super_if}.${sub_if_id}
+| | ... | ${node} | ${super_if2} | ${super_if}.${sub_if_id}
 | | Then IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4['address']} | ${ipv4['prefix-length']}
+| | ... | ${node} | ${super_if2} | ${ipv4['address']} | ${ipv4['prefix-length']}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0
+| | ... | ${node} | ${super_if2}
 | | ... | ${ipv4['address']} | ${ipv4['prefix-length']} | ${ipv4['netmask']}
 | | And sub-interface ipv4 address from Honeycomb should be
 | | ... | ${node} | ${super_if} | ${sub_if_id}
diff --git a/tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot b/tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot
index f0f6e4e6ad..2266509f61 100644
--- a/tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot
+++ b/tests/vpp/func/honeycomb/mgmt-cfg-intip4-intip6-apihc-apivat-func.robot
@@ -14,6 +14,7 @@
 *** Variables ***
 # Interface to run tests on.
 | ${interface}= | ${node['interfaces']['port1']['name']}
+| ${interface2}= | ${node['interfaces']['port3']['name']}
 
 *** Settings ***
 | Resource | resources/libraries/robot/shared/default.robot
@@ -218,16 +219,20 @@
 
 | TC11: Honeycomb can configure unnumbered interface
 | | [Documentation] | Check if Honeycomb can configure an unnumbered interface\
-| | ... | on a physical interface, borrowing the IP address of 'local0'.
+| | ... | on a physical interface, borrowing the IP address of another physical\
+| | ... | interface.
 | | ...
+# CSIT-734: Intermittent failures of all unnumbered interface cases
+| | [Tags] | EXPECTED_FAILING
 | | Given Honeycomb sets interface IPv4 address | ${node}
-| | ... | local0 | ${ipv4_address} | ${ipv4_prefix}
+| | ... | ${interface2} | ${ipv4_address} | ${ipv4_prefix}
 | | When Honeycomb adds unnumbered configuration to interface
-| | ... | ${node} | ${interface} | local0
-| | Then IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4_address} | ${ipv4_prefix}
+| | ... | ${node} | ${interface} | ${interface2}
+| | Then Wait until Keyword succeeds | 10s | 2s
+| | ... | IPv4 address from Honeycomb should be
+| | ... | ${node} | ${interface2} | ${ipv4_address} | ${ipv4_prefix}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0 | ${ipv4_address}
+| | ... | ${node} | ${interface2} | ${ipv4_address}
 | | ... | ${ipv4_prefix} | ${ipv4_mask}
 | | And IPv4 address from Honeycomb should be
 | | ... | ${node} | ${interface} | ${ipv4_address} | ${ipv4_prefix}
@@ -239,10 +244,12 @@
 | | [Documentation] | Check if Honeycomb can remove unnumbered configuration\
 | | ... | from an interface.
 | | ...
+# CSIT-734: Intermittent failures of all unnumbered interface cases
+| | [Tags] | EXPECTED_FAILING
 | | Given IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4_address} | ${ipv4_prefix}
+| | ... | ${node} | ${interface2} | ${ipv4_address} | ${ipv4_prefix}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0 | ${ipv4_address}
+| | ... | ${node} | ${interface2} | ${ipv4_address}
 | | ... | ${ipv4_prefix} | ${ipv4_mask}
 | | And IPv4 address from Honeycomb should be
 | | ... | ${node} | ${interface} | ${ipv4_address} | ${ipv4_prefix}
@@ -251,10 +258,11 @@
 | | ... | ${ipv4_prefix} | ${ipv4_mask}
 | | When Honeycomb removes unnumbered configuration from interface
 | | ... | ${node} | ${interface}
-| | Then IPv4 address from Honeycomb should be
-| | ... | ${node} | local0 | ${ipv4_address} | ${ipv4_prefix}
+| | Then Wait until Keyword succeeds | 10s | 2s
+| | ... | IPv4 address from Honeycomb should be
+| | ... | ${node} | ${interface2} | ${ipv4_address} | ${ipv4_prefix}
 | | And IPv4 address from VAT should be
-| | ... | ${node} | local0 | ${ipv4_address}
+| | ... | ${node} | ${interface2} | ${ipv4_address}
 | | ... | ${ipv4_prefix} | ${ipv4_mask}
 | | And IPv4 address from Honeycomb should be empty | ${node} | ${interface}
 | | And ipv4 address from VAT should be empty | ${node} | ${interface}
diff --git a/tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot b/tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot
index 9f2dbc0ae6..404ba888a3 100644
--- a/tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot
+++ b/tests/vpp/func/honeycomb/mgmt-cfg-routing-apihc-apivat-func.robot
@@ -113,6 +113,8 @@
 | | ... | [Ver] Send ICMP packet from first TG interface to configured route
 | | ... | destination. Receive packet on the second TG interface.
 | | ...
+# VPP-930: ip6_fib_dump shows incorrect outgoing interface index
+| | [Tags] | EXPECTED_FAILING
 | | ${table}= | Set Variable | table4
 | | Given Setup interfaces and neighbors for IPv6 routing test
 | | When Honeycomb configures routing table
@@ -135,6 +137,8 @@
 | | ... | verify that each destination MAC was used by exactly 50 packets.
 | | ... | Receive packet on the second TG interface.
 | | ...
+# VPP-930: ip6_fib_dump shows incorrect outgoing interface index
+| | [Tags] | EXPECTED_FAILING
 | | ${table}= | Set Variable | table5
 | | Given Setup interfaces and neighbors for IPv6 routing test
 | | And Honeycomb adds interface IPv6 neighbor | ${dut_node} | ${dut_to_tg_if2}
diff --git a/tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot b/tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot
index 5320a7fcb7..22a65b2e39 100644
--- a/tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot
+++ b/tests/vpp/func/honeycomb/mgmt-notif-apihcnc-func.robot
@@ -31,7 +31,8 @@
 | ...
 | Suite Teardown | Tear Down Honeycomb Functional Test Suite | ${node}
 | ...
-| Force Tags | HC_FUNC
+# HC2VPP-203 Notifications are not being sent since VPP 17.10
+| Force Tags | HC_FUNC | EXPECTED_FAILING
 
 *** Test Cases ***
 | TC01: Honeycomb sends notification on interface state change
-- 
cgit 1.2.3-korg