From 0d782a5690c7e28a329952fb93afde84ddd97078 Mon Sep 17 00:00:00 2001
From: selias <samelias@cisco.com>
Date: Thu, 24 Nov 2016 14:32:18 +0100
Subject: CSIT-457: HC Test: Document expected test failures

 - add EXPECTED_FAILING tag to all known test failures
 - document every use of EXPECTED_FAILING tag ,include Jira issue ID
 - minor fixes to interface, sub-interface and ACL test suites

Change-Id: If6a535bdacc297c03e8cfade8896cdec23498bc2
Signed-off-by: selias <samelias@cisco.com>
---
 .../func/honeycomb/010_interface_management.robot  | 29 +++++-----
 tests/func/honeycomb/020_bridge_domain.robot       |  2 +
 tests/func/honeycomb/060_sub_interface.robot       |  2 +-
 .../func/honeycomb/080_access_control_lists.robot  | 66 +++++++++++++---------
 tests/func/honeycomb/900_persistence.robot         |  6 ++
 5 files changed, 63 insertions(+), 42 deletions(-)

(limited to 'tests/func/honeycomb')

diff --git a/tests/func/honeycomb/010_interface_management.robot b/tests/func/honeycomb/010_interface_management.robot
index b68b082772..5773b9bdfa 100644
--- a/tests/func/honeycomb/010_interface_management.robot
+++ b/tests/func/honeycomb/010_interface_management.robot
@@ -96,7 +96,7 @@
 
 | Honeycomb modifies IPv4 neighbor table
 | | [Documentation] | Check if Honeycomb API can add and remove ARP entries.
-# Feature not implemented
+# Operational data and VAT dump not available (HONEYCOMB-111)
 | | [Tags] | EXPECTED_FAILING
 | | [Teardown] | Honeycomb clears all interface ipv4 neighbors
 | | ... | ${node} | ${interface}
@@ -104,11 +104,10 @@
 | | ... | ${node} | ${interface} | @{ipv4_neighbor}
 | | Then IPv4 neighbor from Honeycomb should be
 | | ... | ${node} | ${interface} | @{ipv4_neighbor}
-# VAT dump not available
 
 | Honeycomb modifies interface configuration - IPv6
 | | [Documentation] | Check if Honeycomb API can configure interfaces for ipv6.
-# Feature not implemented
+# Configuring IPv6 not implemented (HONEYCOMB-102)
 | | [Tags] | EXPECTED_FAILING
 | | When Honeycomb sets interface ipv6 address
 | | ... | ${node} | ${interface} | @{ipv6_address}
@@ -117,14 +116,18 @@
 | | And IPv6 address from VAT should be
 | | ... | ${node} | ${interface} | @{ipv6_address}
 
-| Honeycomb modifies interface configuration - ethernet,routing
-| | [Documentation] | Check if Honeycomb API can configure interface ethernet\
-| | ... | and routing settings.
-# Feature not implemented
+# TODO: Honeycomb modifies IPv6 neighbor table
+
+| Honeycomb modifies interface configuration - MTU
+| | [Documentation] | Check if Honeycomb API can configure interface\
+| | ... | MTU value.
+# Configuring MTU not implemented (HONEYCOMB-126)
 | | [Tags] | EXPECTED_FAILING
-| | When Honeycomb sets interface ethernet and routing configuration
-| | ... | ${node} | ${interface} | ${ethernet} | ${routing}
-| | Then Interface ethernet and routing configuration from Honeycomb should be
-| | ... | ${node} | ${interface} | ${ethernet} | ${routing}
-| | And Interface ethernet and routing configuration from VAT should be
-| | ... | ${node} | ${interface} | ${ethernet['mtu']} | ${routing['vrf-id']}
+| | When Honeycomb sets interface ethernet configuration
+| | ... | ${node} | ${interface} | ${ethernet}
+| | Then Interface ethernet configuration from Honeycomb should be
+| | ... | ${node} | ${interface} | ${ethernet}
+| | And Interface ethernet configuration from VAT should be
+| | ... | ${node} | ${interface} | ${ethernet['mtu']}
+
+# TODO: Honeycomb configures routing on interface
diff --git a/tests/func/honeycomb/020_bridge_domain.robot b/tests/func/honeycomb/020_bridge_domain.robot
index 850b81e39e..0854051033 100644
--- a/tests/func/honeycomb/020_bridge_domain.robot
+++ b/tests/func/honeycomb/020_bridge_domain.robot
@@ -87,6 +87,8 @@
 | Honeycomb removes bridge domain with an interface assigned
 | | [Documentation] | Check if Honeycomb can remove a bridge domain that has an\
 | | ... | interface assigned to it.
+# Bridge domain references not cleaned up on delete (HONEYCOMB-267)
+| | [Tags] | EXPECTED_FAILING
 | | Given Honeycomb should show interfaces assigned to bridge domain
 | | ... | ${node} | @{interfaces} | ${bd1_name} | ${if_settings}
 | | And VAT should show interfaces assigned to bridge domain
diff --git a/tests/func/honeycomb/060_sub_interface.robot b/tests/func/honeycomb/060_sub_interface.robot
index 5d0d7a37a5..9e58f2766d 100644
--- a/tests/func/honeycomb/060_sub_interface.robot
+++ b/tests/func/honeycomb/060_sub_interface.robot
@@ -38,7 +38,7 @@
 | Honycomb creates sub-interface
 | | [Documentation] | Check if Honeycomb creates a sub-interface.
 | | ...
-| | Given Interface State Is | ${node} | ${super_if} | down
+| | Given Honeycomb sets interface state | ${node} | ${super_if} | down
 | | And sub-interface configuration from Honeycomb should be empty
 | | ... | ${node} | ${super_if} | ${sub_if_id}
 | | And interface configuration from VAT should be empty
diff --git a/tests/func/honeycomb/080_access_control_lists.robot b/tests/func/honeycomb/080_access_control_lists.robot
index e7a864e473..566ec44f14 100644
--- a/tests/func/honeycomb/080_access_control_lists.robot
+++ b/tests/func/honeycomb/080_access_control_lists.robot
@@ -40,11 +40,24 @@
 | | And ACL table from VAT should be
 | | ... | ${node} | ${table_index} | ${vat_acl_table}
 
-| Honeycomb manages more than one ACL table
-| | [Documentation] | Check if Honeycomb API can create another ACL table.
+| Honeycomb can remove ACL table
+| | [Documentation] | Check if Honeycomb API can delete an ACL table.
 | | Given ACL table from Honeycomb should be | ${node} | ${hc_acl_table}
 | | And ACL table from VAT should be
 | | ... | ${node} | ${table_index} | ${vat_acl_table}
+| | When Honeycomb removes ACL table | ${node} | ${hc_acl_table['name']}
+| | Then ACL table from Honeycomb should not exist
+| | ... | ${node} | ${hc_acl_table['name']}
+| | And ACL table from VAT should not exist
+| | ... | ${node} | ${table_index}
+
+| Honeycomb manages more than one ACL table
+| | [Documentation] | Check if Honeycomb API can create another ACL table.
+| | Given ACL table from Honeycomb should not exist
+| | ... | ${node} | ${hc_acl_table['name']}
+| | And ACL table from VAT should not exist
+| | ... | ${node} | ${table_index}
+| | When Honeycomb creates ACL table | ${node} | ${hc_acl_table}
 | | And Honeycomb creates ACL table | ${node} | ${hc_acl_table2}
 | | Then ACL table from Honeycomb should be | ${node} | ${hc_acl_table}
 | | And ACL table from VAT should be
@@ -65,14 +78,29 @@
 | | And ACL session from VAT should be
 | | ... | ${node} | ${table_index} | ${session_index} | ${vat_acl_session}
 
-| Honeycomb manages more than one ACL session on one table
-| | [Documentation] | Check if Honeycomb API can add another ACL session\
-| | ... | to a table.
+| Honeycomb can remove ACL session
+| | [Documentation] | Check if Honeycomb API can remove an ACL session.
 | | Given ACL session from Honeycomb should be
 | | ... | ${node} | ${hc_acl_table['name']} | ${hc_acl_session}
 | | And ACL session from VAT should be
 | | ... | ${node} | ${table_index} | ${session_index} | ${vat_acl_session}
+| | When Honeycomb removes ACL session
+| | ... | ${node} | ${hc_acl_table['name']} | ${hc_acl_session['match']}
+| | Then ACL session from Honeycomb should not exist
+| | ... | ${node} | ${hc_acl_table['name']} | ${hc_acl_session['match']}
+| | And ACL session from VAT should not exist
+| | ... | ${node} | ${table_index} | ${session_index}
+
+| Honeycomb manages more than one ACL session on one table
+| | [Documentation] | Check if Honeycomb API can add another ACL session\
+| | ... | to a table.
+| | Given ACL session from Honeycomb should not exist
+| | ... | ${node} | ${hc_acl_table['name']} | ${hc_acl_session['match']}
+| | And ACL session from VAT should not exist
+| | ... | ${node} | ${table_index} | ${session_index}
 | | When Honeycomb adds ACL session
+| | ... | ${node} | ${hc_acl_table['name']} | ${hc_acl_session}
+| | And Honeycomb adds ACL session
 | | ... | ${node} | ${hc_acl_table['name']} | ${hc_acl_session2}
 | | Then ACL session from Honeycomb should be
 | | ... | ${node} | ${hc_acl_table['name']} | ${hc_acl_session}
@@ -111,29 +139,11 @@
 | | And Interface ACL settings from VAT should be empty
 | | ... | ${node} | ${interface}
 
-| Honeycomb can remove ACL session
-| | [Documentation] | Check if Honeycomb API can remove an ACL session.
-| | Given ACL session from Honeycomb should be
-| | ... | ${node} | ${hc_acl_table['name']} | ${hc_acl_session}
-| | And ACL session from VAT should be
-| | ... | ${node} | ${table_index} | ${session_index} | ${vat_acl_session}
-| | And ACL session from Honeycomb should be
-| | ... | ${node} | ${hc_acl_table['name']} | ${hc_acl_session2}
-| | And ACL session from VAT should be
-| | ... | ${node} | ${table_index} | ${session_index2} | ${vat_acl_session2}
-| | When Honeycomb removes ACL session
-| | ... | ${node} | ${hc_acl_table['name']} | ${hc_acl_session2['match']}
-| | Then ACL session from Honeycomb should be
-| | ... | ${node} | ${hc_acl_table['name']} | ${hc_acl_session}
-| | And ACL session from VAT should be
-| | ... | ${node} | ${table_index} | ${session_index} | ${vat_acl_session}
-| | And ACL session from Honeycomb should not exist
-| | ... | ${node} | ${hc_acl_table['name']} | ${hc_acl_session2['match']}
-| | And ACL session from VAT should not exist
-| | ... | ${node} | ${table_index} | ${session_index2}
-
-| Honeycomb can remove ACL table
-| | [Documentation] | Check if Honeycomb API can delete an ACL table.
+| Honeycomb can remove one out of multiple ACL tables
+| | [Documentation] | Check if Honeycomb API can delete an ACL table if more\
+| | ... | than one table exists.
+# Attempting to remove one ACL table removes all of them (VPP-206)
+| | [Tags] | EXPECTED_FAILING
 | | Given ACL table from Honeycomb should be | ${node} | ${hc_acl_table}
 | | And ACL table from VAT should be
 | | ... | ${node} | ${table_index} | ${vat_acl_table}
diff --git a/tests/func/honeycomb/900_persistence.robot b/tests/func/honeycomb/900_persistence.robot
index ffbc94e635..e0c8ea1b46 100644
--- a/tests/func/honeycomb/900_persistence.robot
+++ b/tests/func/honeycomb/900_persistence.robot
@@ -27,6 +27,8 @@
 | Honeycomb persists configuration through restart of both Honeycomb and VPP
 | | [Documentation] | Checks if Honeycomb maintains configuration after both\
 | | ... | Honeycomb and VPP are restarted.
+# Vxlan tunnel name is not properly restored (HONEYCOMB-301)
+| | [Tags] | EXPECTED_FAILING
 | | Given Honeycomb configures every setting | ${node} | ${interface}
 | | And Honeycomb and VPP should verify every setting | ${node} | ${interface}
 | | When Honeycomb and VPP are restarted | ${node}
@@ -36,6 +38,8 @@
 | Honeycomb persists configuration through restart of Honeycomb
 | | [Documentation] | Checks if Honeycomb maintains configuration after it\
 | | ... | is restarted.
+# Vxlan tunnel name is not properly restored (HONEYCOMB-301)
+| | [Tags] | EXPECTED_FAILING
 | | Given Honeycomb and VPP should verify every setting | ${node} | ${interface}
 | | When Honeycomb is restarted | ${node}
 | | Then Honeycomb and VPP should verify every setting | ${node} | ${interface}
@@ -44,6 +48,8 @@
 | Honeycomb persists configuration through restart of VPP
 | | [Documentation] | Checks if Honeycomb updates VPP settings after VPP is\
 | | ... | restarted.
+# Vxlan tunnel name is not properly restored (HONEYCOMB-301)
+| | [Tags] | EXPECTED_FAILING
 | | Given Honeycomb and VPP should verify every setting | ${node} | ${interface}
 | | When VPP is restarted | ${node}
 | | Then Honeycomb and VPP should verify every setting | ${node} | ${interface}
-- 
cgit