diff options
| -rw-r--r-- | resources/api/vpp/supported_crcs.yaml | 131 | ||||
| -rw-r--r-- | resources/libraries/python/VppApiCrc.py | 47 |
2 files changed, 101 insertions, 77 deletions
diff --git a/resources/api/vpp/supported_crcs.yaml b/resources/api/vpp/supported_crcs.yaml index c397373081..0370500476 100644 --- a/resources/api/vpp/supported_crcs.yaml +++ b/resources/api/vpp/supported_crcs.yaml @@ -30,17 +30,17 @@ # Use bash command "env LC_COLLATE=C sort -u" if not clear. # https://logs.fd.io/production/vex-yul-rot-jenkins-1 -# /vpp-beta-merge-master-ubuntu1804/3788/archives/build-root/ -20.01-rc0~375: - acl_add_replace: '0x13bc8539' # perf - acl_add_replace_reply: '0xac407b0c' # perf +# /vpp-beta-merge-master-ubuntu1804/3866/archives/build-root/ +20.01-rc0~453: + acl_add_replace: '0x13bc8539' # dev + acl_add_replace_reply: '0xac407b0c' # dev acl_details: '0xf89d7a88' # perf teardown acl_dump: '0xef34fea4' # perf teardown acl_interface_list_details: '0xd5e80809' # perf teardown acl_interface_list_dump: '0x529cb13f' # perf teardown acl_interface_set_acl_list: '0x8baece38' # perf acl_interface_set_acl_list_reply: '0xe8d4e804' # perf - # 8x^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr + # 6x^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr # tc01-64B-1c-ethip4udp-ip4base-iacl1sl-10kflows-mrr # ^^ ip4fwdANDiaclANDacl10AND100_flows avf_create: '0xdaab8ae2' # dev @@ -50,7 +50,7 @@ bond_enslave: '0x076ecfa7' # perf bond_enslave_reply: '0xe8d4e804' # perf # 4x^ tc01-64B-1c-1lbvpplacp-dot1q-l2xcbase-eth-2vhostvr1024-1vm-mrr - # ^ l2xcfwdANDlbond_1l + # ^ l2xcfwdANDlbond_1lANDnf_testpmd bridge_domain_add_del: '0xc6360720' # dev bridge_domain_add_del_reply: '0xe8d4e804' # dev # bridge_domain_dump / details # honeycomb @@ -69,31 +69,31 @@ cop_whitelist_enable_disable_reply: '0xe8d4e804' # dev create_loopback: '0x42bb5d22' # dev create_loopback_reply: '0x5383d31f' # dev - create_subif: '0xcb371063' # virl - create_subif_reply: '0x5383d31f' # virl + create_subif: '0xcb371063' # perf + create_subif_reply: '0x5383d31f' # perf + # ^^ tc01-64B-1c-dot1ad-l2xcbase-mrr + # ^ 3n: dot1adANDl2xcfwd create_vhost_user_if: '0xa3438cd4' # dev create_vhost_user_if_reply: '0xfda5941f' # dev create_vlan_subif: '0xaf34ac8b' # dev create_vlan_subif_reply: '0x5383d31f' # dev # dhcp_proxy_dump / details # honeycomb - gbp_bridge_domain_add: '0x70f1069c' # perf - gbp_bridge_domain_add_reply: '0xe8d4e804' # perf - gbp_contract_add_del: '0x5b63d90a' # perf - gbp_contract_add_del_reply: '0x1992deab' # perf - gbp_endpoint_add: '0xf0efa120' # perf - gbp_endpoint_add_reply: '0x1dd3ff3e' # perf - gbp_endpoint_group_add: '0x1031b376' # perf - gbp_endpoint_group_add_reply: '0xe8d4e804' # perf - gbp_ext_itf_add_del: '0x6995e85f' # perf - gbp_ext_itf_add_del_reply: '0xe8d4e804' # perf - gbp_route_domain_add: '0x1560adc7' # perf - gbp_route_domain_add_reply: '0xe8d4e804' # perf - gbp_subnet_add_del: '0xf42b9430' # perf - gbp_subnet_add_del_reply: '0xe8d4e804' # perf - # 14x^ tc01-64B-1c-avf-dot1q-l2bdbasemaclrn-gbp-mrr - # ^ dot1qANDdrv_avfANDgbp - gpe_enable_disable: '0xeb0e943b' # virl - gpe_enable_disable_reply: '0xe8d4e804' # virl + gbp_bridge_domain_add: '0x70f1069c' # dev + gbp_bridge_domain_add_reply: '0xe8d4e804' # dev + gbp_contract_add_del: '0x5b63d90a' # dev + gbp_contract_add_del_reply: '0x1992deab' # dev + gbp_endpoint_add: '0xf0efa120' # dev + gbp_endpoint_add_reply: '0x1dd3ff3e' # dev + gbp_endpoint_group_add: '0x1031b376' # dev + gbp_endpoint_group_add_reply: '0xe8d4e804' # dev + gbp_ext_itf_add_del: '0x6995e85f' # dev + gbp_ext_itf_add_del_reply: '0xe8d4e804' # dev + gbp_route_domain_add: '0x1560adc7' # dev + gbp_route_domain_add_reply: '0xe8d4e804' # dev + gbp_subnet_add_del: '0xf42b9430' # dev + gbp_subnet_add_del_reply: '0xe8d4e804' # dev + gpe_enable_disable: '0xeb0e943b' # dev + gpe_enable_disable_reply: '0xe8d4e804' # dev # gre_tunnel_add_del / reply # unused L1 keyword: create_gre_tunnel_interface hw_interface_set_mtu: '0xe6746899' # dev hw_interface_set_mtu_reply: '0xe8d4e804' # dev @@ -103,8 +103,7 @@ ip_address_dump: '0x6b7bcd0a' # dev ip_neighbor_add_del: '0x029dad44' # dev ip_neighbor_add_del_reply: '0x1992deab' # dev - ip_probe_neighbor: '0x37bc128d' # virl - ip_probe_neighbor_reply: '0xe8d4e804' # virl + # ip_probe_neighbor / reply # unused L1 keyword vpp_ip_probe ip_route_add_del: '0x5ceee41c' # dev ip_route_add_del_reply: '0x1992deab' # dev # ip_source_check_interface_add_del / reply # unused L1 keyword vpp_ip_source_check_setup @@ -116,8 +115,8 @@ ipsec_sad_entry_add_del_reply: '0x9ffac24b' # dev ipsec_select_backend: '0x4fd24836' # perf ipsec_select_backend_reply: '0xe8d4e804' # perf - # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-policy-aes256gcm-mrr - # ^ nic_intel-xl710ANDipsechwANDtnl_1ANDaes_256_gcm + # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-int-aes256gcm-mrr + # ^ nic_intel-xl710ANDipsechwANDipsecintANDtnl_1ANDaes_256_gcm ipsec_spd_add_del: '0x9ffdf5da' # dev ipsec_spd_add_del_reply: '0xe8d4e804' # dev ipsec_spd_entry_add_del: '0xdb217840' # dev @@ -125,33 +124,45 @@ ipsec_tunnel_if_add_del: '0xd5a98274' # perf ipsec_tunnel_if_add_del_reply: '0xfda5941f' # perf # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-int-aes256gcm-mrr - # ^ See select_backend, the tag expression selects both -int- and -policy-. + # ^ See select_backend. # l2_fib_table_dump / details # honeycomb l2_interface_vlan_tag_rewrite: '0xb90be6b4' # dev l2_interface_vlan_tag_rewrite_reply: '0xe8d4e804' # dev l2_patch_add_del: '0x62506e63' # perf l2_patch_add_del_reply: '0xe8d4e804' # perf - # ^^ tc01-64B-1c-avf-eth-l2patch-mrr + # ^^ tc01-64B-1c-avf-eth-l2patch-mrr # currently failing after calls # ^ l2patchANDdrv_avf # l2fib_add_del / reply # unused L1 keyword: vpp_add_l2fib_entry - lisp_add_del_adjacency: '0xf047390d' # virl - lisp_add_del_adjacency_reply: '0xe8d4e804' # virl - lisp_add_del_local_eid: '0xe6d00717' # virl - lisp_add_del_local_eid_reply: '0xe8d4e804' # virl - lisp_add_del_locator: '0x006a4240' # virl - lisp_add_del_locator_reply: '0xe8d4e804' # virl - lisp_add_del_locator_set: '0x06968e38' # virl - lisp_add_del_locator_set_reply: '0xb6666db4' # virl + lb_conf: '0x22ddb739' # perf + lb_conf_reply: '0xe8d4e804' # perf + lb_add_del_vip: '0xd15b7ddc' # perf + lb_add_del_vip_reply: '0xe8d4e804' # perf + lb_add_del_as: '0x78628987' # perf + lb_add_del_as_reply: '0xe8d4e804' # perf + # 6x^ tc01-64B-1c-ethip4-loadbalancer-maglev + # ^ 2n: loadbalancer + lb_add_del_intf_nat4: '0x47d6e753' # perf + lb_add_del_intf_nat4_reply: '0xe8d4e804' # perf + # ^^ tc01-64B-1c-ethip4-loadbalancer-nat4 + # ^^ loadbalancer-nat4 + lisp_add_del_adjacency: '0xf047390d' # dev + lisp_add_del_adjacency_reply: '0xe8d4e804' # dev + lisp_add_del_local_eid: '0xe6d00717' # dev + lisp_add_del_local_eid_reply: '0xe8d4e804' # dev + lisp_add_del_locator: '0x006a4240' # dev + lisp_add_del_locator_reply: '0xe8d4e804' # dev + lisp_add_del_locator_set: '0x06968e38' # dev + lisp_add_del_locator_set_reply: '0xb6666db4' # dev # lisp_add_del_map_resolver / reply # unused L2 keyword: Configure LISP map resolver address - lisp_add_del_remote_mapping: '0xb879c3a9' # virl - lisp_add_del_remote_mapping_reply: '0xe8d4e804' # virl + lisp_add_del_remote_mapping: '0xb879c3a9' # dev + lisp_add_del_remote_mapping_reply: '0xe8d4e804' # dev lisp_eid_table_add_del_map: '0x59e9975e' # perf lisp_eid_table_add_del_map_reply: '0xe8d4e804' # perf - # ^^ tc01-64B-1c-ethip4ipsectptlispgpe-ip4base-aes128cbc-hmac256sha-ndrpdr + # ^^ tc01-64B-1c-ethip4ipsectptlispgpe-ip4base-aes128cbc-hmac256sha-ndrpdr # currently failing before calls # ^ ipsecANDlispgpe # lisp_eid_table_dump / details # unused L2 keyword: LISP eid address should be set correctly to eid table - lisp_enable_disable: '0xeb0e943b' # virl - lisp_enable_disable_reply: '0xe8d4e804' # virl + lisp_enable_disable: '0xeb0e943b' # dev + lisp_enable_disable_reply: '0xe8d4e804' # dev # lisp_locator_set_dump / details # unused L2 keyword: LISP locator_set shpuld be configured correctly # lisp_map_server_dump / details # honeycomb # lisp_map_resolver_dump / details # unused L2 keyword: LISP map resolver address should be configured correctly @@ -179,6 +190,11 @@ nat_show_config_reply: '0x006a0786' # perf teardown # 6x^ tc01-64B-1c-ethip4udp-ip4base-nat44-mrr # ^ nat44NOTscaleNOTsrc_user_1 + policer_add_del: '0xdfea2be8' # dev + policer_add_del_reply: '0xa177cef2' # dev + policer_classify_set_interface: '0xe09537b0' # dev + policer_classify_set_interface_reply: '0xe8d4e804' # dev + # 4x^ tc01-64B-ethip4-ip4base-ipolicemarkbase-dev # show_lisp_map_register_state / reply # honeycomb # show_lisp_map_request_mode / reply # honeycomb # show_lisp_pitr / reply # honeycomb @@ -203,7 +219,7 @@ sr_steering_add_del_reply: '0xe8d4e804' # perf sr_steering_pol_details: '0x5627d483' # perf teardown sr_steering_pol_dump: '0x51077d14' # perf teardown - # x^ tc01-78B-1c-ethip6ip6-ip6base-srv6enc1sid-mrr + # 14x^ tc01-78B-1c-ethip6ip6-ip6base-srv6enc1sid-mrr # ^ srv6_1sid sw_interface_add_del_address: '0x5803d5c4' # dev sw_interface_add_del_address_reply: '0xe8d4e804' # dev @@ -249,20 +265,8 @@ vxlan_add_del_tunnel_reply: '0xfda5941f' # dev # vxlan_gpe_tunnel_dump / details # honeycomb # vxlan_tunnel_dump /details # unused L2 keyword: Get VXLAN dump - policer_add_del: '0xdfea2be8' # dev - policer_add_del_reply: '0xa177cef2' # dev - policer_classify_set_interface: '0xe09537b0' # dev - policer_classify_set_interface_reply: '0xe8d4e804' # dev - # 4x^ tc01-64B-ethip4-ip4base-ipolicemarkbase-dev - # ^^ tc01-64B-1c-ethip4-loadbalancer-maglev/l3dsr/nat4-mrr - lb_conf: '0x22ddb739' # perf - lb_conf_reply: '0xe8d4e804' # perf - lb_add_del_vip: '0xd15b7ddc' # perf - lb_add_del_vip_reply: '0xe8d4e804' # perf - lb_add_del_as: '0x78628987' # perf - lb_add_del_as_reply: '0xe8d4e804' # perf - lb_add_del_intf_nat4: '0x47d6e753' # perf - lb_add_del_intf_nat4_reply: '0xe8d4e804' # perf +# Please keep alphabetic order. +# Use bash command "env LC_COLLATE=C sort -u" if not clear. # Hint to see the currently used command messages: @@ -278,13 +282,14 @@ # mrrAND1cAND64bANDnic_intel-x710ANDip4fwdANDiaclANDacl10AND100_flows # mrrAND1cAND64bANDnic_intel-x710ANDnat44NOTscaleNOTsrc_user_1 # mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDl2bdmaclrnANDbaseANDmemif -# mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDlbond_1l +# mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDlbond_1lANDnf_testpmd # mrrAND1cAND64bANDnic_intel-x710ANDmacipANDacl1AND100_flows # mrrAND1cAND78bANDnic_intel-x710ANDsrv6_1sid # mrrAND1cAND64bANDnic_intel-xl710ANDipsechwANDtnl_1ANDaes_256_gcm -# mrrAND1cAND64bANDnic_intel-xl710ANDipsecANDlispgpe +# mrrAND1cAND64bANDnic_intel-x710ANDdot1adANDl2xcfwd +# mrrAND1cAND64bANDnic_intel-x710ANDipsecANDlispgpe # csit-2n-skx-perftest -# mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDbaseANDdrv_avfANDgbp +# mrrAND1cAND64bANDnic_intel-x710ANDloadbalancer # mrrAND1cAND64bANDnic_intel-x710ANDl2patchANDdrv_avf # TODO: Once device job has complete API coverage, diff --git a/resources/libraries/python/VppApiCrc.py b/resources/libraries/python/VppApiCrc.py index 9a5247a351..8f2979eaec 100644 --- a/resources/libraries/python/VppApiCrc.py +++ b/resources/libraries/python/VppApiCrc.py @@ -36,7 +36,7 @@ class VppApiCrcChecker(object): """Holder of data related to tracking VPP API CRCs. Both message names and crc hexa strings are tracked as - ordinary Python2 (bytes) str, so _str() is used when input is + ordinary Python2 (bytes) string, so _str() is used when input is possibly unicode or otherwise not safe. Each instance of this class starts with same default state, @@ -67,14 +67,14 @@ class VppApiCrcChecker(object): Order of addition reflects the order colections should be queried. If an incompatible CRC is found, affected collections are removed. A CRC that would remove all does not, added to _reported instead, - while causing a failure in single test.""" + while causing a failure in single test (if fail_on_mismatch).""" self._missing = dict() """Mapping from collection name to mapping from API name to CRC string. Starts the same as _expected, but each time an encountered api,crc pair - fits the expectation, the pair is removed from this mapping. - Ideally, the active mappings will become empty. + fits the expectation, the pair is removed from all collections + within this mapping. Ideally, the active mappings will become empty. If not, it is an error, VPP removed or renamed a message CSIT needs.""" self._found = dict() @@ -114,11 +114,12 @@ class VppApiCrcChecker(object): :param name_to_crc_mapping: Mapping from API names to CRCs. :type collection_name: str or unicode :type name_to_crc_mapping: dict from str/unicode to str/unicode + :raises RuntimeError: If the name of a collection is registered already. """ collection_name = _str(collection_name) if collection_name in self._expected: - raise RuntimeError("Collection {cl!r} already registered.".format( - cl=collection_name)) + raise RuntimeError("Collection {cn!r} already registered.".format( + cn=collection_name)) mapping = {_str(k): _str(v) for k, v in name_to_crc_mapping.items()} self._expected[collection_name] = mapping self._missing[collection_name] = mapping.copy() @@ -173,6 +174,8 @@ class VppApiCrcChecker(object): def _process_crc(self, api_name, crc): """Compare API to verified collections, update class state. + Here, API stands for (message name, CRC) pair. + Conflict is NOT when a collection does not recognize the API. Such APIs are merely added to _found for later reporting. Conflict is when a collection recognizes the API under a different CRC. @@ -225,7 +228,7 @@ class VppApiCrcChecker(object): """Parse every .api.json found under directory, remember conflicts. As several collections are supported, each conflict invalidates - one of them, failure happens only when no collections would be left. + some of them, failure happens only when no collections would be left. In that case, set of collections just before the failure is preserved, the _reported mapping is filled with conflicting APIs. The _found mapping is filled with discovered api names and crcs. @@ -259,6 +262,9 @@ class VppApiCrcChecker(object): Missing reporting is disabled by default, because some messages come from plugins that might not be enabled at runtime. + After the report, clear _reported, so that test cases report them again, + thus tracking which message is actually used (by which test). + :param report_missing: Whether to raise on missing messages. :type report_missing: bool :raises RuntimeError: If CRC mismatch or missing messages are detected, @@ -270,8 +276,9 @@ class VppApiCrcChecker(object): if self._reported: reported_indented = json.dumps( self._reported, indent=1, sort_keys=True, separators=[",", ":"]) + self._reported = dict() self.log_and_raise( - "Dir check found incompatible API CRCs:\n{ri}".format( + "Incompatible API CRCs found in .api.json files:\n{ri}".format( ri=reported_indented)) if not report_missing: return @@ -279,15 +286,16 @@ class VppApiCrcChecker(object): if missing: missing_indented = json.dumps( missing, indent=1, sort_keys=True, separators=[",", ":"]) - self.log_and_raise("Dir check found missing API CRCs:\n{mi}".format( + self.log_and_raise("API CRCs missing from .api.json:\n{mi}".format( mi=missing_indented)) def check_api_name(self, api_name): - """Fail if the api_name has no known CRC associated. + """Fail if the api_name has no, or different from known CRC associated. Do not fail if this particular failure has been already reported. - Intended use: Call everytime an API call is queued or response received. + Intended use: Call during test (not in initialization), + everytime an API call is queued or response received. :param api_name: VPP API messagee name to check. :type api_name: str or unicode @@ -306,8 +314,19 @@ class VppApiCrcChecker(object): if new_expected: # Some collections recognized the message name. self._expected = new_expected - return crc = self._found.get(api_name, None) + matching = False + if crc is not None: + # Regardless of how many collections are remaining, + # verify the known CRC is on one of them. + for name_to_crc_mapping in self._expected.values(): + if api_name not in name_to_crc_mapping: + continue + if name_to_crc_mapping[api_name] == crc: + matching = True + break + if matching: + return self._reported[api_name] = crc - self.log_and_raise("No active collection has API {api!r}" - " CRC found {crc!r}".format(api=api_name, crc=crc)) + self.log_and_raise("No active collection contains API {api!r}" + " with CRC {crc!r}".format(api=api_name, crc=crc)) |