CRCs: Update comments and improve CSIT checks.

+ Restore alphabetic order in yaml.
+ Mark the lisp messages verified to be covered by dev instead of virl.
+ Mark the GBP messages verified by devicetest now.
+ Mark acl_add_replace as dev and create_subif as perf.
+ Mark ip_probe_neighbor as unused L1.
+ Improve loadbalancer comments.
  - Nat4 testcase is described, even if not merged yet.
+ Forget reported CRC after initial dir report,
  and report them again (once) in test cases (if hit).
  With this, single run (with fail on crc off) gives full info.
+ Small improvements to CRC library docstrings and messages.

Change-Id: Ic1cfad45e10d437be827af2a89eac49e25267b59
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

2 files changed, 101 insertions, 77 deletions

diff --git a/resources/api/vpp/supported_crcs.yaml b/resources/api/vpp/supported_crcs.yaml
index c397373081..0370500476 100644
--- a/resources/api/vpp/supported_crcs.yaml
+++ b/resources/api/vpp/supported_crcs.yaml
@@ -30,17 +30,17 @@
 # Use bash command "env LC_COLLATE=C sort -u" if not clear.
 
 # https://logs.fd.io/production/vex-yul-rot-jenkins-1
-# /vpp-beta-merge-master-ubuntu1804/3788/archives/build-root/
-20.01-rc0~375:
-    acl_add_replace: '0x13bc8539'  # perf
-    acl_add_replace_reply: '0xac407b0c'  # perf
+# /vpp-beta-merge-master-ubuntu1804/3866/archives/build-root/
+20.01-rc0~453:
+    acl_add_replace: '0x13bc8539'  # dev
+    acl_add_replace_reply: '0xac407b0c'  # dev
     acl_details: '0xf89d7a88'  # perf teardown
     acl_dump: '0xef34fea4'  # perf teardown
     acl_interface_list_details: '0xd5e80809'  # perf teardown
     acl_interface_list_dump: '0x529cb13f'  # perf teardown
     acl_interface_set_acl_list: '0x8baece38'  # perf
     acl_interface_set_acl_list_reply: '0xe8d4e804'  # perf
-    # 8x^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr
+    # 6x^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr
     #     tc01-64B-1c-ethip4udp-ip4base-iacl1sl-10kflows-mrr
     # ^^ ip4fwdANDiaclANDacl10AND100_flows
     avf_create: '0xdaab8ae2'  # dev
@@ -50,7 +50,7 @@
     bond_enslave: '0x076ecfa7'  # perf
     bond_enslave_reply: '0xe8d4e804'  # perf
     # 4x^ tc01-64B-1c-1lbvpplacp-dot1q-l2xcbase-eth-2vhostvr1024-1vm-mrr
-    # ^ l2xcfwdANDlbond_1l
+    # ^ l2xcfwdANDlbond_1lANDnf_testpmd
     bridge_domain_add_del: '0xc6360720'  # dev
     bridge_domain_add_del_reply: '0xe8d4e804'  # dev
     # bridge_domain_dump / details # honeycomb
@@ -69,31 +69,31 @@
     cop_whitelist_enable_disable_reply: '0xe8d4e804'  # dev
     create_loopback: '0x42bb5d22'  # dev
     create_loopback_reply: '0x5383d31f'  # dev
-    create_subif: '0xcb371063'  # virl
-    create_subif_reply: '0x5383d31f'  # virl
+    create_subif: '0xcb371063'  # perf
+    create_subif_reply: '0x5383d31f'  # perf
+    # ^^ tc01-64B-1c-dot1ad-l2xcbase-mrr
+    # ^ 3n: dot1adANDl2xcfwd
     create_vhost_user_if: '0xa3438cd4'  # dev
     create_vhost_user_if_reply: '0xfda5941f'  # dev
     create_vlan_subif: '0xaf34ac8b'  # dev
     create_vlan_subif_reply: '0x5383d31f'  # dev
     # dhcp_proxy_dump / details # honeycomb
-    gbp_bridge_domain_add: '0x70f1069c'  # perf
-    gbp_bridge_domain_add_reply: '0xe8d4e804'  # perf
-    gbp_contract_add_del: '0x5b63d90a'  # perf
-    gbp_contract_add_del_reply: '0x1992deab'  # perf
-    gbp_endpoint_add: '0xf0efa120'  # perf
-    gbp_endpoint_add_reply: '0x1dd3ff3e'  # perf
-    gbp_endpoint_group_add: '0x1031b376'  # perf
-    gbp_endpoint_group_add_reply: '0xe8d4e804'  # perf
-    gbp_ext_itf_add_del: '0x6995e85f'  # perf
-    gbp_ext_itf_add_del_reply: '0xe8d4e804'  # perf
-    gbp_route_domain_add: '0x1560adc7'  # perf
-    gbp_route_domain_add_reply: '0xe8d4e804'  # perf
-    gbp_subnet_add_del: '0xf42b9430'  # perf
-    gbp_subnet_add_del_reply: '0xe8d4e804'  # perf
-    # 14x^ tc01-64B-1c-avf-dot1q-l2bdbasemaclrn-gbp-mrr
-    # ^ dot1qANDdrv_avfANDgbp
-    gpe_enable_disable: '0xeb0e943b'  # virl
-    gpe_enable_disable_reply: '0xe8d4e804'  # virl
+    gbp_bridge_domain_add: '0x70f1069c'  # dev
+    gbp_bridge_domain_add_reply: '0xe8d4e804'  # dev
+    gbp_contract_add_del: '0x5b63d90a'  # dev
+    gbp_contract_add_del_reply: '0x1992deab'  # dev
+    gbp_endpoint_add: '0xf0efa120'  # dev
+    gbp_endpoint_add_reply: '0x1dd3ff3e'  # dev
+    gbp_endpoint_group_add: '0x1031b376'  # dev
+    gbp_endpoint_group_add_reply: '0xe8d4e804'  # dev
+    gbp_ext_itf_add_del: '0x6995e85f'  # dev
+    gbp_ext_itf_add_del_reply: '0xe8d4e804'  # dev
+    gbp_route_domain_add: '0x1560adc7'  # dev
+    gbp_route_domain_add_reply: '0xe8d4e804'  # dev
+    gbp_subnet_add_del: '0xf42b9430'  # dev
+    gbp_subnet_add_del_reply: '0xe8d4e804'  # dev
+    gpe_enable_disable: '0xeb0e943b'  # dev
+    gpe_enable_disable_reply: '0xe8d4e804'  # dev
     # gre_tunnel_add_del / reply # unused L1 keyword: create_gre_tunnel_interface
     hw_interface_set_mtu: '0xe6746899'  # dev
     hw_interface_set_mtu_reply: '0xe8d4e804'  # dev
@@ -103,8 +103,7 @@
     ip_address_dump: '0x6b7bcd0a'  # dev
     ip_neighbor_add_del: '0x029dad44'  # dev
     ip_neighbor_add_del_reply: '0x1992deab'  # dev
-    ip_probe_neighbor: '0x37bc128d'  # virl
-    ip_probe_neighbor_reply: '0xe8d4e804'  # virl
+    # ip_probe_neighbor / reply # unused L1 keyword vpp_ip_probe
     ip_route_add_del: '0x5ceee41c'  # dev
     ip_route_add_del_reply: '0x1992deab'  # dev
     # ip_source_check_interface_add_del / reply # unused L1 keyword vpp_ip_source_check_setup
@@ -116,8 +115,8 @@
     ipsec_sad_entry_add_del_reply: '0x9ffac24b'  # dev
     ipsec_select_backend: '0x4fd24836'  # perf
     ipsec_select_backend_reply: '0xe8d4e804'  # perf
-    # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-policy-aes256gcm-mrr
-    # ^ nic_intel-xl710ANDipsechwANDtnl_1ANDaes_256_gcm
+    # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-int-aes256gcm-mrr
+    # ^ nic_intel-xl710ANDipsechwANDipsecintANDtnl_1ANDaes_256_gcm
     ipsec_spd_add_del: '0x9ffdf5da'  # dev
     ipsec_spd_add_del_reply: '0xe8d4e804'  # dev
     ipsec_spd_entry_add_del: '0xdb217840'  # dev
@@ -125,33 +124,45 @@
     ipsec_tunnel_if_add_del: '0xd5a98274'  # perf
     ipsec_tunnel_if_add_del_reply: '0xfda5941f'  # perf
     # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-int-aes256gcm-mrr
-    # ^ See select_backend, the tag expression selects both -int- and -policy-.
+    # ^ See select_backend.
     # l2_fib_table_dump / details # honeycomb
     l2_interface_vlan_tag_rewrite: '0xb90be6b4'  # dev
     l2_interface_vlan_tag_rewrite_reply: '0xe8d4e804'  # dev
     l2_patch_add_del: '0x62506e63'  # perf
     l2_patch_add_del_reply: '0xe8d4e804'  # perf
-    # ^^ tc01-64B-1c-avf-eth-l2patch-mrr
+    # ^^ tc01-64B-1c-avf-eth-l2patch-mrr # currently failing after calls
     # ^ l2patchANDdrv_avf
     # l2fib_add_del / reply # unused L1 keyword: vpp_add_l2fib_entry
-    lisp_add_del_adjacency: '0xf047390d'  # virl
-    lisp_add_del_adjacency_reply: '0xe8d4e804'  # virl
-    lisp_add_del_local_eid: '0xe6d00717'  # virl
-    lisp_add_del_local_eid_reply: '0xe8d4e804'  # virl
-    lisp_add_del_locator: '0x006a4240'  # virl
-    lisp_add_del_locator_reply: '0xe8d4e804'  # virl
-    lisp_add_del_locator_set: '0x06968e38'  # virl
-    lisp_add_del_locator_set_reply: '0xb6666db4'  # virl
+    lb_conf: '0x22ddb739'  # perf
+    lb_conf_reply: '0xe8d4e804'  # perf
+    lb_add_del_vip: '0xd15b7ddc'  # perf
+    lb_add_del_vip_reply: '0xe8d4e804'  # perf
+    lb_add_del_as: '0x78628987'  # perf
+    lb_add_del_as_reply: '0xe8d4e804'  # perf
+    # 6x^ tc01-64B-1c-ethip4-loadbalancer-maglev
+    # ^ 2n: loadbalancer
+    lb_add_del_intf_nat4: '0x47d6e753' # perf
+    lb_add_del_intf_nat4_reply: '0xe8d4e804' # perf
+    # ^^ tc01-64B-1c-ethip4-loadbalancer-nat4
+    # ^^ loadbalancer-nat4
+    lisp_add_del_adjacency: '0xf047390d'  # dev
+    lisp_add_del_adjacency_reply: '0xe8d4e804'  # dev
+    lisp_add_del_local_eid: '0xe6d00717'  # dev
+    lisp_add_del_local_eid_reply: '0xe8d4e804'  # dev
+    lisp_add_del_locator: '0x006a4240'  # dev
+    lisp_add_del_locator_reply: '0xe8d4e804'  # dev
+    lisp_add_del_locator_set: '0x06968e38'  # dev
+    lisp_add_del_locator_set_reply: '0xb6666db4'  # dev
     # lisp_add_del_map_resolver / reply # unused L2 keyword: Configure LISP map resolver address
-    lisp_add_del_remote_mapping: '0xb879c3a9'  # virl
-    lisp_add_del_remote_mapping_reply: '0xe8d4e804'  # virl
+    lisp_add_del_remote_mapping: '0xb879c3a9'  # dev
+    lisp_add_del_remote_mapping_reply: '0xe8d4e804'  # dev
     lisp_eid_table_add_del_map: '0x59e9975e'  # perf
     lisp_eid_table_add_del_map_reply: '0xe8d4e804'  # perf
-    # ^^ tc01-64B-1c-ethip4ipsectptlispgpe-ip4base-aes128cbc-hmac256sha-ndrpdr
+    # ^^ tc01-64B-1c-ethip4ipsectptlispgpe-ip4base-aes128cbc-hmac256sha-ndrpdr # currently failing before calls
     # ^ ipsecANDlispgpe
     # lisp_eid_table_dump / details # unused L2 keyword: LISP eid address should be set correctly to eid table
-    lisp_enable_disable: '0xeb0e943b'  # virl
-    lisp_enable_disable_reply: '0xe8d4e804'  # virl
+    lisp_enable_disable: '0xeb0e943b'  # dev
+    lisp_enable_disable_reply: '0xe8d4e804'  # dev
     # lisp_locator_set_dump / details # unused L2 keyword: LISP locator_set shpuld be configured correctly
     # lisp_map_server_dump / details # honeycomb
     # lisp_map_resolver_dump / details # unused L2 keyword: LISP map resolver address should be configured correctly
@@ -179,6 +190,11 @@
     nat_show_config_reply: '0x006a0786'  # perf teardown
     # 6x^ tc01-64B-1c-ethip4udp-ip4base-nat44-mrr
     # ^ nat44NOTscaleNOTsrc_user_1
+    policer_add_del: '0xdfea2be8'  # dev
+    policer_add_del_reply: '0xa177cef2'  # dev
+    policer_classify_set_interface: '0xe09537b0' # dev
+    policer_classify_set_interface_reply: '0xe8d4e804' # dev
+    # 4x^ tc01-64B-ethip4-ip4base-ipolicemarkbase-dev
     # show_lisp_map_register_state / reply # honeycomb
     # show_lisp_map_request_mode / reply # honeycomb
     # show_lisp_pitr / reply # honeycomb
@@ -203,7 +219,7 @@
     sr_steering_add_del_reply: '0xe8d4e804'  # perf
     sr_steering_pol_details: '0x5627d483'  # perf teardown
     sr_steering_pol_dump: '0x51077d14'  # perf teardown
-    # x^ tc01-78B-1c-ethip6ip6-ip6base-srv6enc1sid-mrr
+    # 14x^ tc01-78B-1c-ethip6ip6-ip6base-srv6enc1sid-mrr
     # ^ srv6_1sid
     sw_interface_add_del_address: '0x5803d5c4'  # dev
     sw_interface_add_del_address_reply: '0xe8d4e804'  # dev
@@ -249,20 +265,8 @@
     vxlan_add_del_tunnel_reply: '0xfda5941f'  # dev
     # vxlan_gpe_tunnel_dump / details # honeycomb
     # vxlan_tunnel_dump /details # unused L2 keyword: Get VXLAN dump
-    policer_add_del: '0xdfea2be8'  # dev
-    policer_add_del_reply: '0xa177cef2'  # dev
-    policer_classify_set_interface: '0xe09537b0' # dev
-    policer_classify_set_interface_reply: '0xe8d4e804' # dev
-    # 4x^ tc01-64B-ethip4-ip4base-ipolicemarkbase-dev
-    # ^^ tc01-64B-1c-ethip4-loadbalancer-maglev/l3dsr/nat4-mrr
-    lb_conf: '0x22ddb739'  # perf
-    lb_conf_reply: '0xe8d4e804'  # perf
-    lb_add_del_vip: '0xd15b7ddc'  # perf
-    lb_add_del_vip_reply: '0xe8d4e804'  # perf
-    lb_add_del_as: '0x78628987'  # perf
-    lb_add_del_as_reply: '0xe8d4e804'  # perf
-    lb_add_del_intf_nat4: '0x47d6e753' # perf
-    lb_add_del_intf_nat4_reply: '0xe8d4e804' # perf
+# Please keep alphabetic order.
+# Use bash command "env LC_COLLATE=C sort -u" if not clear.
 
 
 # Hint to see the currently used command messages:
@@ -278,13 +282,14 @@
 # mrrAND1cAND64bANDnic_intel-x710ANDip4fwdANDiaclANDacl10AND100_flows
 # mrrAND1cAND64bANDnic_intel-x710ANDnat44NOTscaleNOTsrc_user_1
 # mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDl2bdmaclrnANDbaseANDmemif
-# mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDlbond_1l
+# mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDlbond_1lANDnf_testpmd
 # mrrAND1cAND64bANDnic_intel-x710ANDmacipANDacl1AND100_flows
 # mrrAND1cAND78bANDnic_intel-x710ANDsrv6_1sid
 # mrrAND1cAND64bANDnic_intel-xl710ANDipsechwANDtnl_1ANDaes_256_gcm
-# mrrAND1cAND64bANDnic_intel-xl710ANDipsecANDlispgpe
+# mrrAND1cAND64bANDnic_intel-x710ANDdot1adANDl2xcfwd
+# mrrAND1cAND64bANDnic_intel-x710ANDipsecANDlispgpe
 # csit-2n-skx-perftest
-# mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDbaseANDdrv_avfANDgbp
+# mrrAND1cAND64bANDnic_intel-x710ANDloadbalancer
 # mrrAND1cAND64bANDnic_intel-x710ANDl2patchANDdrv_avf
 
 # TODO: Once device job has complete API coverage,
diff --git a/resources/libraries/python/VppApiCrc.py b/resources/libraries/python/VppApiCrc.py
index 9a5247a351..8f2979eaec 100644
--- a/resources/libraries/python/VppApiCrc.py
+++ b/resources/libraries/python/VppApiCrc.py
@@ -36,7 +36,7 @@ class VppApiCrcChecker(object):
     """Holder of data related to tracking VPP API CRCs.
 
     Both message names and crc hexa strings are tracked as
-    ordinary Python2 (bytes) str, so _str() is used when input is
+    ordinary Python2 (bytes) string, so _str() is used when input is
     possibly unicode or otherwise not safe.
 
     Each instance of this class starts with same default state,
@@ -67,14 +67,14 @@ class VppApiCrcChecker(object):
         Order of addition reflects the order colections should be queried.
         If an incompatible CRC is found, affected collections are removed.
         A CRC that would remove all does not, added to _reported instead,
-        while causing a failure in single test."""
+        while causing a failure in single test (if fail_on_mismatch)."""
 
         self._missing = dict()
         """Mapping from collection name to mapping from API name to CRC string.
 
         Starts the same as _expected, but each time an encountered api,crc pair
-        fits the expectation, the pair is removed from this mapping.
-        Ideally, the active mappings will become empty.
+        fits the expectation, the pair is removed from all collections
+        within this mapping. Ideally, the active mappings will become empty.
         If not, it is an error, VPP removed or renamed a message CSIT needs."""
 
         self._found = dict()
@@ -114,11 +114,12 @@ class VppApiCrcChecker(object):
         :param name_to_crc_mapping: Mapping from API names to CRCs.
         :type collection_name: str or unicode
         :type name_to_crc_mapping: dict from str/unicode to str/unicode
+        :raises RuntimeError: If the name of a collection is registered already.
         """
         collection_name = _str(collection_name)
         if collection_name in self._expected:
-            raise RuntimeError("Collection {cl!r} already registered.".format(
-                cl=collection_name))
+            raise RuntimeError("Collection {cn!r} already registered.".format(
+                cn=collection_name))
         mapping = {_str(k): _str(v) for k, v in name_to_crc_mapping.items()}
         self._expected[collection_name] = mapping
         self._missing[collection_name] = mapping.copy()
@@ -173,6 +174,8 @@ class VppApiCrcChecker(object):
     def _process_crc(self, api_name, crc):
         """Compare API to verified collections, update class state.
 
+        Here, API stands for (message name, CRC) pair.
+
         Conflict is NOT when a collection does not recognize the API.
         Such APIs are merely added to _found for later reporting.
         Conflict is when a collection recognizes the API under a different CRC.
@@ -225,7 +228,7 @@ class VppApiCrcChecker(object):
         """Parse every .api.json found under directory, remember conflicts.
 
         As several collections are supported, each conflict invalidates
-        one of them, failure happens only when no collections would be left.
+        some of them, failure happens only when no collections would be left.
         In that case, set of collections just before the failure is preserved,
         the _reported mapping is filled with conflicting APIs.
         The _found mapping is filled with discovered api names and crcs.
@@ -259,6 +262,9 @@ class VppApiCrcChecker(object):
         Missing reporting is disabled by default, because some messages
         come from plugins that might not be enabled at runtime.
 
+        After the report, clear _reported, so that test cases report them again,
+        thus tracking which message is actually used (by which test).
+
         :param report_missing: Whether to raise on missing messages.
         :type report_missing: bool
         :raises RuntimeError: If CRC mismatch or missing messages are detected,
@@ -270,8 +276,9 @@ class VppApiCrcChecker(object):
         if self._reported:
             reported_indented = json.dumps(
                 self._reported, indent=1, sort_keys=True, separators=[",", ":"])
+            self._reported = dict()
             self.log_and_raise(
-                "Dir check found incompatible API CRCs:\n{ri}".format(
+                "Incompatible API CRCs found in .api.json files:\n{ri}".format(
                     ri=reported_indented))
         if not report_missing:
             return
@@ -279,15 +286,16 @@ class VppApiCrcChecker(object):
         if missing:
             missing_indented = json.dumps(
                 missing, indent=1, sort_keys=True, separators=[",", ":"])
-            self.log_and_raise("Dir check found missing API CRCs:\n{mi}".format(
+            self.log_and_raise("API CRCs missing from .api.json:\n{mi}".format(
                 mi=missing_indented))
 
     def check_api_name(self, api_name):
-        """Fail if the api_name has no known CRC associated.
+        """Fail if the api_name has no, or different from known CRC associated.
 
         Do not fail if this particular failure has been already reported.
 
-        Intended use: Call everytime an API call is queued or response received.
+        Intended use: Call during test (not in initialization),
+        everytime an API call is queued or response received.
 
         :param api_name: VPP API messagee name to check.
         :type api_name: str or unicode
@@ -306,8 +314,19 @@ class VppApiCrcChecker(object):
         if new_expected:
             # Some collections recognized the message name.
             self._expected = new_expected
-            return
         crc = self._found.get(api_name, None)
+        matching = False
+        if crc is not None:
+            # Regardless of how many collections are remaining,
+            # verify the known CRC is on one of them.
+            for name_to_crc_mapping in self._expected.values():
+                if api_name not in name_to_crc_mapping:
+                    continue
+                if name_to_crc_mapping[api_name] == crc:
+                    matching = True
+                    break
+        if matching:
+            return
         self._reported[api_name] = crc
-        self.log_and_raise("No active collection has API {api!r}"
-                           " CRC found {crc!r}".format(api=api_name, crc=crc))
+        self.log_and_raise("No active collection contains API {api!r}"
+                           " with CRC {crc!r}".format(api=api_name, crc=crc))