CRCs: Update comments and improve CSIT checks.

+ Restore alphabetic order in yaml.
+ Mark the lisp messages verified to be covered by dev instead of virl.
+ Mark the GBP messages verified by devicetest now.
+ Mark acl_add_replace as dev and create_subif as perf.
+ Mark ip_probe_neighbor as unused L1.
+ Improve loadbalancer comments.
  - Nat4 testcase is described, even if not merged yet.
+ Forget reported CRC after initial dir report,
  and report them again (once) in test cases (if hit).
  With this, single run (with fail on crc off) gives full info.
+ Small improvements to CRC library docstrings and messages.

Change-Id: Ic1cfad45e10d437be827af2a89eac49e25267b59
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

1 files changed, 68 insertions, 63 deletions

diff --git a/resources/api/vpp/supported_crcs.yaml b/resources/api/vpp/supported_crcs.yaml
index c397373081..0370500476 100644
--- a/resources/api/vpp/supported_crcs.yaml
+++ b/resources/api/vpp/supported_crcs.yaml
@@ -30,17 +30,17 @@
 # Use bash command "env LC_COLLATE=C sort -u" if not clear.
 
 # https://logs.fd.io/production/vex-yul-rot-jenkins-1
-# /vpp-beta-merge-master-ubuntu1804/3788/archives/build-root/
-20.01-rc0~375:
-    acl_add_replace: '0x13bc8539'  # perf
-    acl_add_replace_reply: '0xac407b0c'  # perf
+# /vpp-beta-merge-master-ubuntu1804/3866/archives/build-root/
+20.01-rc0~453:
+    acl_add_replace: '0x13bc8539'  # dev
+    acl_add_replace_reply: '0xac407b0c'  # dev
     acl_details: '0xf89d7a88'  # perf teardown
     acl_dump: '0xef34fea4'  # perf teardown
     acl_interface_list_details: '0xd5e80809'  # perf teardown
     acl_interface_list_dump: '0x529cb13f'  # perf teardown
     acl_interface_set_acl_list: '0x8baece38'  # perf
     acl_interface_set_acl_list_reply: '0xe8d4e804'  # perf
-    # 8x^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr
+    # 6x^ tc01-64B-1c-ethip4udp-ip4base-iacl1sf-10kflows-mrr
     #     tc01-64B-1c-ethip4udp-ip4base-iacl1sl-10kflows-mrr
     # ^^ ip4fwdANDiaclANDacl10AND100_flows
     avf_create: '0xdaab8ae2'  # dev
@@ -50,7 +50,7 @@
     bond_enslave: '0x076ecfa7'  # perf
     bond_enslave_reply: '0xe8d4e804'  # perf
     # 4x^ tc01-64B-1c-1lbvpplacp-dot1q-l2xcbase-eth-2vhostvr1024-1vm-mrr
-    # ^ l2xcfwdANDlbond_1l
+    # ^ l2xcfwdANDlbond_1lANDnf_testpmd
     bridge_domain_add_del: '0xc6360720'  # dev
     bridge_domain_add_del_reply: '0xe8d4e804'  # dev
     # bridge_domain_dump / details # honeycomb
@@ -69,31 +69,31 @@
     cop_whitelist_enable_disable_reply: '0xe8d4e804'  # dev
     create_loopback: '0x42bb5d22'  # dev
     create_loopback_reply: '0x5383d31f'  # dev
-    create_subif: '0xcb371063'  # virl
-    create_subif_reply: '0x5383d31f'  # virl
+    create_subif: '0xcb371063'  # perf
+    create_subif_reply: '0x5383d31f'  # perf
+    # ^^ tc01-64B-1c-dot1ad-l2xcbase-mrr
+    # ^ 3n: dot1adANDl2xcfwd
     create_vhost_user_if: '0xa3438cd4'  # dev
     create_vhost_user_if_reply: '0xfda5941f'  # dev
     create_vlan_subif: '0xaf34ac8b'  # dev
     create_vlan_subif_reply: '0x5383d31f'  # dev
     # dhcp_proxy_dump / details # honeycomb
-    gbp_bridge_domain_add: '0x70f1069c'  # perf
-    gbp_bridge_domain_add_reply: '0xe8d4e804'  # perf
-    gbp_contract_add_del: '0x5b63d90a'  # perf
-    gbp_contract_add_del_reply: '0x1992deab'  # perf
-    gbp_endpoint_add: '0xf0efa120'  # perf
-    gbp_endpoint_add_reply: '0x1dd3ff3e'  # perf
-    gbp_endpoint_group_add: '0x1031b376'  # perf
-    gbp_endpoint_group_add_reply: '0xe8d4e804'  # perf
-    gbp_ext_itf_add_del: '0x6995e85f'  # perf
-    gbp_ext_itf_add_del_reply: '0xe8d4e804'  # perf
-    gbp_route_domain_add: '0x1560adc7'  # perf
-    gbp_route_domain_add_reply: '0xe8d4e804'  # perf
-    gbp_subnet_add_del: '0xf42b9430'  # perf
-    gbp_subnet_add_del_reply: '0xe8d4e804'  # perf
-    # 14x^ tc01-64B-1c-avf-dot1q-l2bdbasemaclrn-gbp-mrr
-    # ^ dot1qANDdrv_avfANDgbp
-    gpe_enable_disable: '0xeb0e943b'  # virl
-    gpe_enable_disable_reply: '0xe8d4e804'  # virl
+    gbp_bridge_domain_add: '0x70f1069c'  # dev
+    gbp_bridge_domain_add_reply: '0xe8d4e804'  # dev
+    gbp_contract_add_del: '0x5b63d90a'  # dev
+    gbp_contract_add_del_reply: '0x1992deab'  # dev
+    gbp_endpoint_add: '0xf0efa120'  # dev
+    gbp_endpoint_add_reply: '0x1dd3ff3e'  # dev
+    gbp_endpoint_group_add: '0x1031b376'  # dev
+    gbp_endpoint_group_add_reply: '0xe8d4e804'  # dev
+    gbp_ext_itf_add_del: '0x6995e85f'  # dev
+    gbp_ext_itf_add_del_reply: '0xe8d4e804'  # dev
+    gbp_route_domain_add: '0x1560adc7'  # dev
+    gbp_route_domain_add_reply: '0xe8d4e804'  # dev
+    gbp_subnet_add_del: '0xf42b9430'  # dev
+    gbp_subnet_add_del_reply: '0xe8d4e804'  # dev
+    gpe_enable_disable: '0xeb0e943b'  # dev
+    gpe_enable_disable_reply: '0xe8d4e804'  # dev
     # gre_tunnel_add_del / reply # unused L1 keyword: create_gre_tunnel_interface
     hw_interface_set_mtu: '0xe6746899'  # dev
     hw_interface_set_mtu_reply: '0xe8d4e804'  # dev
@@ -103,8 +103,7 @@
     ip_address_dump: '0x6b7bcd0a'  # dev
     ip_neighbor_add_del: '0x029dad44'  # dev
     ip_neighbor_add_del_reply: '0x1992deab'  # dev
-    ip_probe_neighbor: '0x37bc128d'  # virl
-    ip_probe_neighbor_reply: '0xe8d4e804'  # virl
+    # ip_probe_neighbor / reply # unused L1 keyword vpp_ip_probe
     ip_route_add_del: '0x5ceee41c'  # dev
     ip_route_add_del_reply: '0x1992deab'  # dev
     # ip_source_check_interface_add_del / reply # unused L1 keyword vpp_ip_source_check_setup
@@ -116,8 +115,8 @@
     ipsec_sad_entry_add_del_reply: '0x9ffac24b'  # dev
     ipsec_select_backend: '0x4fd24836'  # perf
     ipsec_select_backend_reply: '0xe8d4e804'  # perf
-    # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-policy-aes256gcm-mrr
-    # ^ nic_intel-xl710ANDipsechwANDtnl_1ANDaes_256_gcm
+    # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-int-aes256gcm-mrr
+    # ^ nic_intel-xl710ANDipsechwANDipsecintANDtnl_1ANDaes_256_gcm
     ipsec_spd_add_del: '0x9ffdf5da'  # dev
     ipsec_spd_add_del_reply: '0xe8d4e804'  # dev
     ipsec_spd_entry_add_del: '0xdb217840'  # dev
@@ -125,33 +124,45 @@
     ipsec_tunnel_if_add_del: '0xd5a98274'  # perf
     ipsec_tunnel_if_add_del_reply: '0xfda5941f'  # perf
     # ^^ tc01-64B-1c-ethip4ipsec1tnlhw-ip4base-int-aes256gcm-mrr
-    # ^ See select_backend, the tag expression selects both -int- and -policy-.
+    # ^ See select_backend.
     # l2_fib_table_dump / details # honeycomb
     l2_interface_vlan_tag_rewrite: '0xb90be6b4'  # dev
     l2_interface_vlan_tag_rewrite_reply: '0xe8d4e804'  # dev
     l2_patch_add_del: '0x62506e63'  # perf
     l2_patch_add_del_reply: '0xe8d4e804'  # perf
-    # ^^ tc01-64B-1c-avf-eth-l2patch-mrr
+    # ^^ tc01-64B-1c-avf-eth-l2patch-mrr # currently failing after calls
     # ^ l2patchANDdrv_avf
     # l2fib_add_del / reply # unused L1 keyword: vpp_add_l2fib_entry
-    lisp_add_del_adjacency: '0xf047390d'  # virl
-    lisp_add_del_adjacency_reply: '0xe8d4e804'  # virl
-    lisp_add_del_local_eid: '0xe6d00717'  # virl
-    lisp_add_del_local_eid_reply: '0xe8d4e804'  # virl
-    lisp_add_del_locator: '0x006a4240'  # virl
-    lisp_add_del_locator_reply: '0xe8d4e804'  # virl
-    lisp_add_del_locator_set: '0x06968e38'  # virl
-    lisp_add_del_locator_set_reply: '0xb6666db4'  # virl
+    lb_conf: '0x22ddb739'  # perf
+    lb_conf_reply: '0xe8d4e804'  # perf
+    lb_add_del_vip: '0xd15b7ddc'  # perf
+    lb_add_del_vip_reply: '0xe8d4e804'  # perf
+    lb_add_del_as: '0x78628987'  # perf
+    lb_add_del_as_reply: '0xe8d4e804'  # perf
+    # 6x^ tc01-64B-1c-ethip4-loadbalancer-maglev
+    # ^ 2n: loadbalancer
+    lb_add_del_intf_nat4: '0x47d6e753' # perf
+    lb_add_del_intf_nat4_reply: '0xe8d4e804' # perf
+    # ^^ tc01-64B-1c-ethip4-loadbalancer-nat4
+    # ^^ loadbalancer-nat4
+    lisp_add_del_adjacency: '0xf047390d'  # dev
+    lisp_add_del_adjacency_reply: '0xe8d4e804'  # dev
+    lisp_add_del_local_eid: '0xe6d00717'  # dev
+    lisp_add_del_local_eid_reply: '0xe8d4e804'  # dev
+    lisp_add_del_locator: '0x006a4240'  # dev
+    lisp_add_del_locator_reply: '0xe8d4e804'  # dev
+    lisp_add_del_locator_set: '0x06968e38'  # dev
+    lisp_add_del_locator_set_reply: '0xb6666db4'  # dev
     # lisp_add_del_map_resolver / reply # unused L2 keyword: Configure LISP map resolver address
-    lisp_add_del_remote_mapping: '0xb879c3a9'  # virl
-    lisp_add_del_remote_mapping_reply: '0xe8d4e804'  # virl
+    lisp_add_del_remote_mapping: '0xb879c3a9'  # dev
+    lisp_add_del_remote_mapping_reply: '0xe8d4e804'  # dev
     lisp_eid_table_add_del_map: '0x59e9975e'  # perf
     lisp_eid_table_add_del_map_reply: '0xe8d4e804'  # perf
-    # ^^ tc01-64B-1c-ethip4ipsectptlispgpe-ip4base-aes128cbc-hmac256sha-ndrpdr
+    # ^^ tc01-64B-1c-ethip4ipsectptlispgpe-ip4base-aes128cbc-hmac256sha-ndrpdr # currently failing before calls
     # ^ ipsecANDlispgpe
     # lisp_eid_table_dump / details # unused L2 keyword: LISP eid address should be set correctly to eid table
-    lisp_enable_disable: '0xeb0e943b'  # virl
-    lisp_enable_disable_reply: '0xe8d4e804'  # virl
+    lisp_enable_disable: '0xeb0e943b'  # dev
+    lisp_enable_disable_reply: '0xe8d4e804'  # dev
     # lisp_locator_set_dump / details # unused L2 keyword: LISP locator_set shpuld be configured correctly
     # lisp_map_server_dump / details # honeycomb
     # lisp_map_resolver_dump / details # unused L2 keyword: LISP map resolver address should be configured correctly
@@ -179,6 +190,11 @@
     nat_show_config_reply: '0x006a0786'  # perf teardown
     # 6x^ tc01-64B-1c-ethip4udp-ip4base-nat44-mrr
     # ^ nat44NOTscaleNOTsrc_user_1
+    policer_add_del: '0xdfea2be8'  # dev
+    policer_add_del_reply: '0xa177cef2'  # dev
+    policer_classify_set_interface: '0xe09537b0' # dev
+    policer_classify_set_interface_reply: '0xe8d4e804' # dev
+    # 4x^ tc01-64B-ethip4-ip4base-ipolicemarkbase-dev
     # show_lisp_map_register_state / reply # honeycomb
     # show_lisp_map_request_mode / reply # honeycomb
     # show_lisp_pitr / reply # honeycomb
@@ -203,7 +219,7 @@
     sr_steering_add_del_reply: '0xe8d4e804'  # perf
     sr_steering_pol_details: '0x5627d483'  # perf teardown
     sr_steering_pol_dump: '0x51077d14'  # perf teardown
-    # x^ tc01-78B-1c-ethip6ip6-ip6base-srv6enc1sid-mrr
+    # 14x^ tc01-78B-1c-ethip6ip6-ip6base-srv6enc1sid-mrr
     # ^ srv6_1sid
     sw_interface_add_del_address: '0x5803d5c4'  # dev
     sw_interface_add_del_address_reply: '0xe8d4e804'  # dev
@@ -249,20 +265,8 @@
     vxlan_add_del_tunnel_reply: '0xfda5941f'  # dev
     # vxlan_gpe_tunnel_dump / details # honeycomb
     # vxlan_tunnel_dump /details # unused L2 keyword: Get VXLAN dump
-    policer_add_del: '0xdfea2be8'  # dev
-    policer_add_del_reply: '0xa177cef2'  # dev
-    policer_classify_set_interface: '0xe09537b0' # dev
-    policer_classify_set_interface_reply: '0xe8d4e804' # dev
-    # 4x^ tc01-64B-ethip4-ip4base-ipolicemarkbase-dev
-    # ^^ tc01-64B-1c-ethip4-loadbalancer-maglev/l3dsr/nat4-mrr
-    lb_conf: '0x22ddb739'  # perf
-    lb_conf_reply: '0xe8d4e804'  # perf
-    lb_add_del_vip: '0xd15b7ddc'  # perf
-    lb_add_del_vip_reply: '0xe8d4e804'  # perf
-    lb_add_del_as: '0x78628987'  # perf
-    lb_add_del_as_reply: '0xe8d4e804'  # perf
-    lb_add_del_intf_nat4: '0x47d6e753' # perf
-    lb_add_del_intf_nat4_reply: '0xe8d4e804' # perf
+# Please keep alphabetic order.
+# Use bash command "env LC_COLLATE=C sort -u" if not clear.
 
 
 # Hint to see the currently used command messages:
@@ -278,13 +282,14 @@
 # mrrAND1cAND64bANDnic_intel-x710ANDip4fwdANDiaclANDacl10AND100_flows
 # mrrAND1cAND64bANDnic_intel-x710ANDnat44NOTscaleNOTsrc_user_1
 # mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDl2bdmaclrnANDbaseANDmemif
-# mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDlbond_1l
+# mrrAND1cAND64bANDnic_intel-x710ANDl2xcfwdANDlbond_1lANDnf_testpmd
 # mrrAND1cAND64bANDnic_intel-x710ANDmacipANDacl1AND100_flows
 # mrrAND1cAND78bANDnic_intel-x710ANDsrv6_1sid
 # mrrAND1cAND64bANDnic_intel-xl710ANDipsechwANDtnl_1ANDaes_256_gcm
-# mrrAND1cAND64bANDnic_intel-xl710ANDipsecANDlispgpe
+# mrrAND1cAND64bANDnic_intel-x710ANDdot1adANDl2xcfwd
+# mrrAND1cAND64bANDnic_intel-x710ANDipsecANDlispgpe
 # csit-2n-skx-perftest
-# mrrAND1cAND64bANDnic_intel-x710ANDdot1qANDbaseANDdrv_avfANDgbp
+# mrrAND1cAND64bANDnic_intel-x710ANDloadbalancer
 # mrrAND1cAND64bANDnic_intel-x710ANDl2patchANDdrv_avf
 
 # TODO: Once device job has complete API coverage,