aboutsummaryrefslogtreecommitdiffstats
path: root/resources
diff options
context:
space:
mode:
authorVratko Polak <vrpolak@cisco.com>2019-10-21 14:58:16 +0200
committerVratko Polak <vrpolak@cisco.com>2019-10-21 14:58:16 +0200
commit23b5bfe679371df276db4713f6089655b0e9bc56 (patch)
tree4e58a018d6cc422796153e773cbb47b5c2e32e4d /resources
parent431599c0dba2c7fb31630db9a5024a083ccf9013 (diff)
CRCs: Update comments and improve CSIT checks.
+ Restore alphabetic order in yaml. + Mark the lisp messages verified to be covered by dev instead of virl. + Mark the GBP messages verified by devicetest now. + Mark acl_add_replace as dev and create_subif as perf. + Mark ip_probe_neighbor as unused L1. + Improve loadbalancer comments. - Nat4 testcase is described, even if not merged yet. + Forget reported CRC after initial dir report, and report them again (once) in test cases (if hit). With this, single run (with fail on crc off) gives full info. + Small improvements to CRC library docstrings and messages. Change-Id: Ic1cfad45e10d437be827af2a89eac49e25267b59 Signed-off-by: Vratko Polak <vrpolak@cisco.com>
Diffstat (limited to 'resources')
-rw-r--r--resources/api/vpp/supported_crcs.yaml131
-rw-r--r--resources/libraries/python/VppApiCrc.py47
2 files changed, 101 insertions, 77 deletions
diff --git a/resources/api/vpp/supported_crcs.yaml b/resources/api/vpp/supported_crcs.yaml
index c397373081..0370500476 100644
--- a/resources/api/vpp/supported_crcs.yaml
+++ b/resources/api/vpp/supported_crcs.yaml
@@ -30,17 +30,17 @@
# Use bash command "env LC_COLLATE=C sort -u" if not clear.
# https://logs.fd.io/production/vex-yul-rot-jenkins-1
-# /vpp-beta-merge-master-ubuntu1804/3788/archives/build-root/
-20.01-rc0~375:
- acl_add_replace: '0x13bc8539' # perf
- acl_add_replace_reply: '0xac407b0c' # perf
+# /vpp-beta-merge-master-ubuntu1804/3866/archives/build-root/
+20.01-rc0~453:
+ acl_add_replace: '0x13bc8539' # dev
+ acl_add_replace_reply: '0xac407b0c' # dev
acl_details: '0xf89d7a88' # perf teardown
acl_dump: '0xef34fea4' # perf teardown
acl_interface_list_details: '0xd5e80809' # perf teardown
acl_interface_list_dump: '0x529cb13f' # perf teardown
acl_interface_set_acl_list: '0x8baece38' # perf
acl_interface_set_acl_list_reply: '0xe8d4e804' # perf
- # 8x^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr
+ # 6x^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr
# tc01-64B-1c-ethip4udp-ip4base-iacl1sl-10kflows-mrr
# ^^ ip4fwdANDiaclANDacl10AND100_flows
avf_create: '0xdaab8ae2' # dev
@@ -50,7 +50,7 @@
bond_enslave: '0x076ecfa7' # perf
bond_enslave_reply: '0xe8d4e804' # perf
# 4x^ tc01-64B-1c-1lbvpplacp-dot1q-l2xcbase-eth-2vhostvr1024-1vm-mrr
- # ^ l2xcfwdANDlbond_1l
+ # ^ l2xcfwdANDlbond_1lANDnf_testpmd
bridge_domain_add_del: '0xc6360720' # dev
bridge_domain_add_del_reply: '0xe8d4e804' # dev
# bridge_domain_dump / details # honeycomb
@@ -69,31 +69,31 @@
cop_whitelist_enable_disable_reply: '0xe8d4e804' # dev
create_loopback: '0x42bb5d22' # dev
create_loopback_reply: '0x5383d31f' # dev
- create_subif: '0xcb371063' # virl
- create_subif_reply: '0x5383d31f' # virl
+ create_subif: '0xcb371063' # perf
+ create_subif_reply: '0x5383d31f' # perf
+ # ^^ tc01-64B-1c-dot1ad-l2xcbase-mrr
+ # ^ 3n: dot1adANDl2xcfwd
create_vhost_user_if: '0xa3438cd4' # dev
create_vhost_user_if_reply: '0xfda5941f' # dev
create_vlan_subif: '0xaf34ac8b' # dev
create_vlan_subif_reply: '0x5383d31f' # dev
# dhcp_proxy_dump / details # honeycomb
- gbp_bridge_domain_add: '0x70f1069c' # perf
- gbp_bridge_domain_add_reply: '0xe8d4e804' # perf
- gbp_contract_add_del: '0x5b63d90a' # perf
- gbp_contract_add_del_reply: '0x1992deab' # perf
- gbp_endpoint_add: '0xf0efa120' # perf
- gbp_endpoint_add_reply: '0x1dd3ff3e' # perf
- gbp_endpoint_group_add: '0x1031b376' # perf
- gbp_endpoint_group_add_reply: '0xe8d4e804' # perf
- gbp_ext_itf_add_del: '0x6995e85f' # perf
- gbp_ext_itf_add_del_reply: '0xe8d4e804' # perf
- gbp_route_domain_add: '0x1560adc7' # perf
- gbp_route_domain_add_reply: '0xe8d4e804' # perf
- gbp_subnet_add_del: '0xf42b9430' # perf
- gbp_subnet_add_del_reply: '0xe8d4e804' # perf
- # 14x^ tc01-64B-1c-avf-dot1q-l2bdbasemaclrn-gbp-mrr
- # ^ dot1qANDdrv_avfANDgbp
- gpe_enable_disable: '0xeb0e943b' # virl
- gpe_enable_disable_reply: '0xe8d4e804' # virl
+ gbp_bridge_domain_add: '0x70f1069c' # dev
+ gbp_bridge_domain_add_reply: '0xe8d4e804' # dev
+ gbp_contract_add_del: '0x5b63d90a' # dev
+ gbp_contract_add_del_reply: '0x1992deab' # dev
+ gbp_endpoint_add: '0xf0efa120' # dev
+ gbp_endpoint_add_reply: '0x1dd3ff3e' # dev
+ gbp_endpoint_group_add: '0x1031b376' # dev
+ gbp_endpoint_group_add_reply: '0xe8d4e804' # dev
+ gbp_ext_itf_add_del: '0x6995e85f' # dev
+ gbp_ext_itf_add_del_reply: '0xe8d4e804' # dev
+ gbp_route_domain_add: '0x1560adc7' # dev
+ gbp_route_domain_add_reply: '0xe8d4e804' # dev
+ gbp_subnet_add_del: '0xf42b9430' # dev
+ gbp_subnet_add_del_reply: '0xe8d4e804' # dev
+ gpe_enable_disable: '0xeb0e943b' # dev
+ gpe_enable_disable_reply: '0xe8d4e804' # dev
# gre_tunnel_add_del / reply # unused L1 keyword: create_gre_tunnel_interface
hw_interface_set_mtu: '0xe6746899' # dev
hw_interface_set_mtu_reply: '0xe8d4e804' # dev
@@ -103,8 +103,7 @@
ip_address_dump: '0x6b7bcd0a' # dev
ip_neighbor_add_del: '0x029dad44' # dev
ip_neighbor_add_del_reply: '0x1992deab' # dev
- ip_probe_neighbor: '0x37bc128d' # virl
- ip_probe_neighbor_reply: '0xe8d4e804' # virl
+ # ip_probe_neighbor / reply # unused L1 keyword vpp_ip_probe
ip_route_add_del: '0x5ceee41c' # dev
ip_route_add_del_reply: '0x1992deab' # dev
# ip_source_check_interface_add_del / reply # unused L1 keyword vpp_ip_source_check_setup
@@ -116,8 +115,8 @@
ipsec_sad_entry_add_del_reply: '0x9ffac24b' # dev
ipsec_select_backend: '0x4fd24836' # perf
ipsec_select_backend_reply: '0xe8d4e804' # perf
- # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-policy-aes256gcm-mrr
- # ^ nic_intel-xl710ANDipsechwANDtnl_1ANDaes_256_gcm
+ # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-int-aes256gcm-mrr
+ # ^ nic_intel-xl710ANDipsechwANDipsecintANDtnl_1ANDaes_256_gcm
ipsec_spd_add_del: '0x9ffdf5da' # dev
ipsec_spd_add_del_reply: '0xe8d4e804' # dev
ipsec_spd_entry_add_del: '0xdb217840' # dev
@@ -125,33 +124,45 @@
ipsec_tunnel_if_add_del: '0xd5a98274' # perf
ipsec_tunnel_if_add_del_reply: '0xfda5941f' # perf
# ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-int-aes256gcm-mrr
- # ^ See select_backend, the tag expression selects both -int- and -policy-.
+ # ^ See select_backend.
# l2_fib_table_dump / details # honeycomb
l2_interface_vlan_tag_rewrite: '0xb90be6b4' # dev
l2_interface_vlan_tag_rewrite_reply: '0xe8d4e804' # dev
l2_patch_add_del: '0x62506e63' # perf
l2_patch_add_del_reply: '0xe8d4e804' # perf
- # ^^ tc01-64B-1c-avf-eth-l2patch-mrr
+ # ^^ tc01-64B-1c-avf-eth-l2patch-mrr # currently failing after calls
# ^ l2patchANDdrv_avf
# l2fib_add_del / reply # unused L1 keyword: vpp_add_l2fib_entry
- lisp_add_del_adjacency: '0xf047390d' # virl
- lisp_add_del_adjacency_reply: '0xe8d4e804' # virl
- lisp_add_del_local_eid: '0xe6d00717' # virl
- lisp_add_del_local_eid_reply: '0xe8d4e804' # virl
- lisp_add_del_locator: '0x006a4240' # virl
- lisp_add_del_locator_reply: '0xe8d4e804' # virl
- lisp_add_del_locator_set: '0x06968e38' # virl
- lisp_add_del_locator_set_reply: '0xb6666db4' # virl
+ lb_conf: '0x22ddb739' # perf
+ lb_conf_reply: '0xe8d4e804' # perf
+ lb_add_del_vip: '0xd15b7ddc' # perf
+ lb_add_del_vip_reply: '0xe8d4e804' # perf
+ lb_add_del_as: '0x78628987' # perf
+ lb_add_del_as_reply: '0xe8d4e804' # perf
+ # 6x^ tc01-64B-1c-ethip4-loadbalancer-maglev
+ # ^ 2n: loadbalancer
+ lb_add_del_intf_nat4: '0x47d6e753' # perf
+ lb_add_del_intf_nat4_reply: '0xe8d4e804' # perf
+ # ^^ tc01-64B-1c-ethip4-loadbalancer-nat4
+ # ^^ loadbalancer-nat4
+ lisp_add_del_adjacency: '0xf047390d' # dev
+ lisp_add_del_adjacency_reply: '0xe8d4e804' # dev
+ lisp_add_del_local_eid: '0xe6d00717' # dev
+ lisp_add_del_local_eid_reply: '0xe8d4e804' # dev
+ lisp_add_del_locator: '0x006a4240' # dev
+ lisp_add_del_locator_reply: '0xe8d4e804' # dev
+ lisp_add_del_locator_set: '0x06968e38' # dev
+ lisp_add_del_locator_set_reply: '0xb6666db4' # dev
# lisp_add_del_map_resolver / reply # unused L2 keyword: Configure LISP map resolver address
- lisp_add_del_remote_mapping: '0xb879c3a9' # virl
- lisp_add_del_remote_mapping_reply: '0xe8d4e804' # virl
+ lisp_add_del_remote_mapping: '0xb879c3a9' # dev
+ lisp_add_del_remote_mapping_reply: '0xe8d4e804' # dev
lisp_eid_table_add_del_map: '0x59e9975e' # perf
lisp_eid_table_add_del_map_reply: '0xe8d4e804' # perf
- # ^^ tc01-64B-1c-ethip4ipsectptlispgpe-ip4base-aes128cbc-hmac256sha-ndrpdr
+ # ^^ tc01-64B-1c-ethip4ipsectptlispgpe-ip4base-aes128cbc-hmac256sha-ndrpdr # currently failing before calls
# ^ ipsecANDlispgpe
# lisp_eid_table_dump / details # unused L2 keyword: LISP eid address should be set correctly to eid table
- lisp_enable_disable: '0xeb0e943b' # virl
- lisp_enable_disable_reply: '0xe8d4e804' # virl
+ lisp_enable_disable: '0xeb0e943b' # dev
+ lisp_enable_disable_reply: '0xe8d4e804' # dev
# lisp_locator_set_dump / details # unused L2 keyword: LISP locator_set shpuld be configured correctly
# lisp_map_server_dump / details # honeycomb
# lisp_map_resolver_dump / details # unused L2 keyword: LISP map resolver address should be configured correctly
@@ -179,6 +190,11 @@
nat_show_config_reply: '0x006a0786' # perf teardown
# 6x^ tc01-64B-1c-ethip4udp-ip4base-nat44-mrr
# ^ nat44NOTscaleNOTsrc_user_1
+ policer_add_del: '0xdfea2be8' # dev
+ policer_add_del_reply: '0xa177cef2' # dev
+ policer_classify_set_interface: '0xe09537b0' # dev
+ policer_classify_set_interface_reply: '0xe8d4e804' # dev
+ # 4x^ tc01-64B-ethip4-ip4base-ipolicemarkbase-dev
# show_lisp_map_register_state / reply # honeycomb
# show_lisp_map_request_mode / reply # honeycomb
# show_lisp_pitr / reply # honeycomb
@@ -203,7 +219,7 @@
sr_steering_add_del_reply: '0xe8d4e804' # perf
sr_steering_pol_details: '0x5627d483' # perf teardown
sr_steering_pol_dump: '0x51077d14' # perf teardown
- # x^ tc01-78B-1c-ethip6ip6-ip6base-srv6enc1sid-mrr
+ # 14x^ tc01-78B-1c-ethip6ip6-ip6base-srv6enc1sid-mrr
# ^ srv6_1sid
sw_interface_add_del_address: '0x5803d5c4' # dev
sw_interface_add_del_address_reply: '0xe8d4e804' # dev
@@ -249,20 +265,8 @@
vxlan_add_del_tunnel_reply: '0xfda5941f' # dev
# vxlan_gpe_tunnel_dump / details # honeycomb
# vxlan_tunnel_dump /details # unused L2 keyword: Get VXLAN dump
- policer_add_del: '0xdfea2be8' # dev
- policer_add_del_reply: '0xa177cef2' # dev
- policer_classify_set_interface: '0xe09537b0' # dev
- policer_classify_set_interface_reply: '0xe8d4e804' # dev
- # 4x^ tc01-64B-ethip4-ip4base-ipolicemarkbase-dev
- # ^^ tc01-64B-1c-ethip4-loadbalancer-maglev/l3dsr/nat4-mrr
- lb_conf: '0x22ddb739' # perf
- lb_conf_reply: '0xe8d4e804' # perf
- lb_add_del_vip: '0xd15b7ddc' # perf
- lb_add_del_vip_reply: '0xe8d4e804' # perf
- lb_add_del_as: '0x78628987' # perf
- lb_add_del_as_reply: '0xe8d4e804' # perf
- lb_add_del_intf_nat4: '0x47d6e753' # perf
- lb_add_del_intf_nat4_reply: '0xe8d4e804' # perf
+# Please keep alphabetic order.
+# Use bash command "env LC_COLLATE=C sort -u" if not clear.
# Hint to see the currently used command messages:
@@ -278,13 +282,14 @@
# mrrAND1cAND64bANDnic_intel-x710ANDip4fwdANDiaclANDacl10AND100_flows
# mrrAND1cAND64bANDnic_intel-x710ANDnat44NOTscaleNOTsrc_user_1
# mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDl2bdmaclrnANDbaseANDmemif
-# mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDlbond_1l
+# mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDlbond_1lANDnf_testpmd
# mrrAND1cAND64bANDnic_intel-x710ANDmacipANDacl1AND100_flows
# mrrAND1cAND78bANDnic_intel-x710ANDsrv6_1sid
# mrrAND1cAND64bANDnic_intel-xl710ANDipsechwANDtnl_1ANDaes_256_gcm
-# mrrAND1cAND64bANDnic_intel-xl710ANDipsecANDlispgpe
+# mrrAND1cAND64bANDnic_intel-x710ANDdot1adANDl2xcfwd
+# mrrAND1cAND64bANDnic_intel-x710ANDipsecANDlispgpe
# csit-2n-skx-perftest
-# mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDbaseANDdrv_avfANDgbp
+# mrrAND1cAND64bANDnic_intel-x710ANDloadbalancer
# mrrAND1cAND64bANDnic_intel-x710ANDl2patchANDdrv_avf
# TODO: Once device job has complete API coverage,
diff --git a/resources/libraries/python/VppApiCrc.py b/resources/libraries/python/VppApiCrc.py
index 9a5247a351..8f2979eaec 100644
--- a/resources/libraries/python/VppApiCrc.py
+++ b/resources/libraries/python/VppApiCrc.py
@@ -36,7 +36,7 @@ class VppApiCrcChecker(object):
"""Holder of data related to tracking VPP API CRCs.
Both message names and crc hexa strings are tracked as
- ordinary Python2 (bytes) str, so _str() is used when input is
+ ordinary Python2 (bytes) string, so _str() is used when input is
possibly unicode or otherwise not safe.
Each instance of this class starts with same default state,
@@ -67,14 +67,14 @@ class VppApiCrcChecker(object):
Order of addition reflects the order colections should be queried.
If an incompatible CRC is found, affected collections are removed.
A CRC that would remove all does not, added to _reported instead,
- while causing a failure in single test."""
+ while causing a failure in single test (if fail_on_mismatch)."""
self._missing = dict()
"""Mapping from collection name to mapping from API name to CRC string.
Starts the same as _expected, but each time an encountered api,crc pair
- fits the expectation, the pair is removed from this mapping.
- Ideally, the active mappings will become empty.
+ fits the expectation, the pair is removed from all collections
+ within this mapping. Ideally, the active mappings will become empty.
If not, it is an error, VPP removed or renamed a message CSIT needs."""
self._found = dict()
@@ -114,11 +114,12 @@ class VppApiCrcChecker(object):
:param name_to_crc_mapping: Mapping from API names to CRCs.
:type collection_name: str or unicode
:type name_to_crc_mapping: dict from str/unicode to str/unicode
+ :raises RuntimeError: If the name of a collection is registered already.
"""
collection_name = _str(collection_name)
if collection_name in self._expected:
- raise RuntimeError("Collection {cl!r} already registered.".format(
- cl=collection_name))
+ raise RuntimeError("Collection {cn!r} already registered.".format(
+ cn=collection_name))
mapping = {_str(k): _str(v) for k, v in name_to_crc_mapping.items()}
self._expected[collection_name] = mapping
self._missing[collection_name] = mapping.copy()
@@ -173,6 +174,8 @@ class VppApiCrcChecker(object):
def _process_crc(self, api_name, crc):
"""Compare API to verified collections, update class state.
+ Here, API stands for (message name, CRC) pair.
+
Conflict is NOT when a collection does not recognize the API.
Such APIs are merely added to _found for later reporting.
Conflict is when a collection recognizes the API under a different CRC.
@@ -225,7 +228,7 @@ class VppApiCrcChecker(object):
"""Parse every .api.json found under directory, remember conflicts.
As several collections are supported, each conflict invalidates
- one of them, failure happens only when no collections would be left.
+ some of them, failure happens only when no collections would be left.
In that case, set of collections just before the failure is preserved,
the _reported mapping is filled with conflicting APIs.
The _found mapping is filled with discovered api names and crcs.
@@ -259,6 +262,9 @@ class VppApiCrcChecker(object):
Missing reporting is disabled by default, because some messages
come from plugins that might not be enabled at runtime.
+ After the report, clear _reported, so that test cases report them again,
+ thus tracking which message is actually used (by which test).
+
:param report_missing: Whether to raise on missing messages.
:type report_missing: bool
:raises RuntimeError: If CRC mismatch or missing messages are detected,
@@ -270,8 +276,9 @@ class VppApiCrcChecker(object):
if self._reported:
reported_indented = json.dumps(
self._reported, indent=1, sort_keys=True, separators=[",", ":"])
+ self._reported = dict()
self.log_and_raise(
- "Dir check found incompatible API CRCs:\n{ri}".format(
+ "Incompatible API CRCs found in .api.json files:\n{ri}".format(
ri=reported_indented))
if not report_missing:
return
@@ -279,15 +286,16 @@ class VppApiCrcChecker(object):
if missing:
missing_indented = json.dumps(
missing, indent=1, sort_keys=True, separators=[",", ":"])
- self.log_and_raise("Dir check found missing API CRCs:\n{mi}".format(
+ self.log_and_raise("API CRCs missing from .api.json:\n{mi}".format(
mi=missing_indented))
def check_api_name(self, api_name):
- """Fail if the api_name has no known CRC associated.
+ """Fail if the api_name has no, or different from known CRC associated.
Do not fail if this particular failure has been already reported.
- Intended use: Call everytime an API call is queued or response received.
+ Intended use: Call during test (not in initialization),
+ everytime an API call is queued or response received.
:param api_name: VPP API messagee name to check.
:type api_name: str or unicode
@@ -306,8 +314,19 @@ class VppApiCrcChecker(object):
if new_expected:
# Some collections recognized the message name.
self._expected = new_expected
- return
crc = self._found.get(api_name, None)
+ matching = False
+ if crc is not None:
+ # Regardless of how many collections are remaining,
+ # verify the known CRC is on one of them.
+ for name_to_crc_mapping in self._expected.values():
+ if api_name not in name_to_crc_mapping:
+ continue
+ if name_to_crc_mapping[api_name] == crc:
+ matching = True
+ break
+ if matching:
+ return
self._reported[api_name] = crc
- self.log_and_raise("No active collection has API {api!r}"
- " CRC found {crc!r}".format(api=api_name, crc=crc))
+ self.log_and_raise("No active collection contains API {api!r}"
+ " with CRC {crc!r}".format(api=api_name, crc=crc))