tests: add enhanced packet counter verification

Add support for inline packet counter verification to send_and_* functions.

Diff dictionary is a dictionary of dictionaries of interesting stats:

    diff_dictionary =
    {
        "err" : { '/error/counter1' : 4, },
        sw_if_index1 : { '/stat/segment/counter1' : 5,
                         '/stat/segment/counter2' : 6,
                       },
        sw_if_index2 : { '/stat/segment/counter1' : 7,
                       },
    }

It describes a per sw-if-index diffset, where each key is stat segment
path and value is the expected change for that counter for sw-if-index.
Special case string "err" is used for error counters.

This then allows more precise packet counter verification by first
defining a "zero" dictionary, e.g. for ED NAT:

        cls.no_diff = StatsDiff({
            pg.sw_if_index: {
                '/nat44-ed/in2out/fastpath/tcp': 0,
                '/nat44-ed/in2out/fastpath/udp': 0,
                '/nat44-ed/in2out/fastpath/icmp': 0,
                '/nat44-ed/in2out/fastpath/drops': 0,
                '/nat44-ed/in2out/slowpath/tcp': 0,
                '/nat44-ed/in2out/slowpath/udp': 0,
                '/nat44-ed/in2out/slowpath/icmp': 0,
                '/nat44-ed/in2out/slowpath/drops': 0,
                '/nat44-ed/in2out/fastpath/tcp': 0,
                '/nat44-ed/in2out/fastpath/udp': 0,
                '/nat44-ed/in2out/fastpath/icmp': 0,
                '/nat44-ed/in2out/fastpath/drops': 0,
                '/nat44-ed/in2out/slowpath/tcp': 0,
                '/nat44-ed/in2out/slowpath/udp': 0,
                '/nat44-ed/in2out/slowpath/icmp': 0,
                '/nat44-ed/in2out/slowpath/drops': 0,
            }
            for pg in cls.pg_interfaces
        })

and then to specify only changed counters directly when calling
one of send_and_* functions:

        self.send_and_assert_no_replies(
            self.pg0, pkts, msg="i2o pkts",
            stats_diff=self.no_diff | {
                "err": {
                    '/err/nat44-ed-in2out-slowpath/out of ports': len(pkts),
                },
                self.pg0.sw_if_index: {
                    '/nat44-ed/in2out/slowpath/drops': len(pkts),
                },
            }
        )

operator | is overloaded by StatsDiff class to perform a deep merge operation,
so in above case, dictionaries for "err" and self.pg0.sw_if_index do not
overwrite whole sub-dictionaries, rather the contents are merged,
assuring that all the remaining counters are verified to be zero.

Type: improvement
Signed-off-by: Klement Sekera <klement.sekera@gmail.com>
Change-Id: I2b87f7bd58a7d4b34ee72344e2f871b2f372e2d9

3 files changed, 175 insertions, 40 deletions

diff --git a/test/framework.py b/test/framework.py
index f0d916fd9a6..1c81e8afabc 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 from __future__ import print_function
-import gc
 import logging
 import sys
 import os
@@ -1264,26 +1263,82 @@ class VppTestCase(CPUInterface, unittest.TestCase):
         self.pg_enable_capture(self.pg_interfaces)
         self.pg_start(trace=trace)
 
+    def snapshot_stats(self, stats_diff):
+        """Return snapshot of interesting stats based on diff dictionary."""
+        stats_snapshot = {}
+        for sw_if_index in stats_diff:
+            for counter in stats_diff[sw_if_index]:
+                stats_snapshot[counter] = self.statistics[counter]
+        self.logger.debug(f"Took statistics stats_snapshot: {stats_snapshot}")
+        return stats_snapshot
+
+    def compare_stats_with_snapshot(self, stats_diff, stats_snapshot):
+        """Assert appropriate difference between current stats and snapshot."""
+        for sw_if_index in stats_diff:
+            for cntr, diff in stats_diff[sw_if_index].items():
+                if sw_if_index == "err":
+                    self.assert_equal(
+                        self.statistics[cntr].sum(),
+                        stats_snapshot[cntr].sum() + diff,
+                        f"'{cntr}' counter value (previous value: "
+                        f"{stats_snapshot[cntr].sum()}, "
+                        f"expected diff: {diff})")
+                else:
+                    try:
+                        self.assert_equal(
+                            self.statistics[cntr][:, sw_if_index].sum(),
+                            stats_snapshot[cntr][:, sw_if_index].sum() + diff,
+                            f"'{cntr}' counter value (previous value: "
+                            f"{stats_snapshot[cntr][:, sw_if_index].sum()}, "
+                            f"expected diff: {diff})")
+                    except IndexError:
+                        # if diff is 0, then this most probably a case where
+                        # test declares multiple interfaces but traffic hasn't
+                        # passed through this one yet - which means the counter
+                        # value is 0 and can be ignored
+                        if 0 != diff:
+                            raise
+
     def send_and_assert_no_replies(self, intf, pkts, remark="", timeout=None,
-                                   trace=True):
+                                   stats_diff=None, trace=True, msg=None):
+        if stats_diff:
+            stats_snapshot = self.snapshot_stats(stats_diff)
+
         self.pg_send(intf, pkts)
-        if not timeout:
-            timeout = 1
-        for i in self.pg_interfaces:
-            i.get_capture(0, timeout=timeout)
-            i.assert_nothing_captured(remark=remark)
-            timeout = 0.1
-        if trace:
-            self.logger.debug(self.vapi.cli("show trace"))
+
+        try:
+            if not timeout:
+                timeout = 1
+            for i in self.pg_interfaces:
+                i.get_capture(0, timeout=timeout)
+                i.assert_nothing_captured(remark=remark)
+                timeout = 0.1
+        finally:
+            if trace:
+                if msg:
+                    self.logger.debug(f"send_and_assert_no_replies: {msg}")
+                self.logger.debug(self.vapi.cli("show trace"))
+
+        if stats_diff:
+            self.compare_stats_with_snapshot(stats_diff, stats_snapshot)
 
     def send_and_expect(self, intf, pkts, output, n_rx=None, worker=None,
-                        trace=True):
+                        trace=True, msg=None, stats_diff=None):
+        if stats_diff:
+            stats_snapshot = self.snapshot_stats(stats_diff)
+
         if not n_rx:
             n_rx = 1 if isinstance(pkts, Packet) else len(pkts)
         self.pg_send(intf, pkts, worker=worker, trace=trace)
         rx = output.get_capture(n_rx)
         if trace:
+            if msg:
+                self.logger.debug(f"send_and_expect: {msg}")
             self.logger.debug(self.vapi.cli("show trace"))
+
+        if stats_diff:
+            self.compare_stats_with_snapshot(stats_diff, stats_snapshot)
+
         return rx
 
     def send_and_expect_load_balancing(self, input, pkts, outputs,
@@ -1298,7 +1353,11 @@ class VppTestCase(CPUInterface, unittest.TestCase):
             self.logger.debug(self.vapi.cli("show trace"))
         return rxs
 
-    def send_and_expect_only(self, intf, pkts, output, timeout=None):
+    def send_and_expect_only(self, intf, pkts, output, timeout=None,
+                             stats_diff=None):
+        if stats_diff:
+            stats_snapshot = self.snapshot_stats(stats_diff)
+
         self.pg_send(intf, pkts)
         rx = output.get_capture(len(pkts))
         outputs = [output]
@@ -1310,6 +1369,9 @@ class VppTestCase(CPUInterface, unittest.TestCase):
                 i.assert_nothing_captured()
                 timeout = 0.1
 
+        if stats_diff:
+            self.compare_stats_with_snapshot(stats_diff, stats_snapshot)
+
         return rx
 
 
diff --git a/test/test_nat44_ed.py b/test/test_nat44_ed.py
index 9bb803e4435..764693d636a 100644
--- a/test/test_nat44_ed.py
+++ b/test/test_nat44_ed.py
@@ -2,7 +2,7 @@
 
 import unittest
 from io import BytesIO
-from random import randint, shuffle, choice
+from random import randint, choice
 
 import scapy.compat
 from framework import VppTestCase, VppTestRunner
@@ -17,6 +17,7 @@ from util import ppp, ip4_range
 from vpp_acl import AclRule, VppAcl, VppAclInterface
 from vpp_ip_route import VppIpRoute, VppRoutePath
 from vpp_papi import VppEnum
+from util import StatsDiff
 
 
 class TestNAT44ED(VppTestCase):
@@ -213,6 +214,28 @@ class TestNAT44ED(VppTestCase):
         for r in rl:
             r.add_vpp_config()
 
+        cls.no_diff = StatsDiff({
+            pg.sw_if_index: {
+                '/nat44-ed/in2out/fastpath/tcp': 0,
+                '/nat44-ed/in2out/fastpath/udp': 0,
+                '/nat44-ed/in2out/fastpath/icmp': 0,
+                '/nat44-ed/in2out/fastpath/drops': 0,
+                '/nat44-ed/in2out/slowpath/tcp': 0,
+                '/nat44-ed/in2out/slowpath/udp': 0,
+                '/nat44-ed/in2out/slowpath/icmp': 0,
+                '/nat44-ed/in2out/slowpath/drops': 0,
+                '/nat44-ed/in2out/fastpath/tcp': 0,
+                '/nat44-ed/in2out/fastpath/udp': 0,
+                '/nat44-ed/in2out/fastpath/icmp': 0,
+                '/nat44-ed/in2out/fastpath/drops': 0,
+                '/nat44-ed/in2out/slowpath/tcp': 0,
+                '/nat44-ed/in2out/slowpath/udp': 0,
+                '/nat44-ed/in2out/slowpath/icmp': 0,
+                '/nat44-ed/in2out/slowpath/drops': 0,
+            }
+            for pg in cls.pg_interfaces
+        })
+
     def get_err_counter(self, path):
         return self.statistics.get_err_counter(path)
 
@@ -2622,37 +2645,41 @@ class TestNAT44EDMW(TestNAT44ED):
         self.nat_add_outside_interface(self.pg1)
 
         # in2out and no NAT addresses added
-        err_old = self.statistics.get_err_counter(
-            '/err/nat44-ed-in2out-slowpath/out of ports')
-
         pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        self.pg1.get_capture(0, timeout=1)
 
-        err_new = self.statistics.get_err_counter(
-            '/err/nat44-ed-in2out-slowpath/out of ports')
-
-        self.assertEqual(err_new - err_old, len(pkts))
+        self.send_and_assert_no_replies(
+            self.pg0, pkts, msg="i2o pkts",
+            stats_diff=self.no_diff | {
+                "err": {
+                    '/err/nat44-ed-in2out-slowpath/out of ports': len(pkts),
+                },
+                self.pg0.sw_if_index: {
+                    '/nat44-ed/in2out/slowpath/drops': len(pkts),
+                },
+            }
+        )
 
         # in2out after NAT addresses added
         self.nat_add_address(self.nat_addr)
 
-        err_old = self.statistics.get_err_counter(
-            '/err/nat44-ed-in2out-slowpath/out of ports')
-
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture, ignore_port=True)
-
-        err_new = self.statistics.get_err_counter(
-            '/err/nat44-ed-in2out-slowpath/out of ports')
-
-        self.assertEqual(err_new, err_old)
+        tcpn, udpn, icmpn = (sum(x) for x in
+                             zip(*((TCP in p, UDP in p, ICMP in p)
+                                 for p in pkts)))
+
+        self.send_and_expect(
+            self.pg0, pkts, self.pg1, msg="i2o pkts",
+            stats_diff=self.no_diff | {
+                "err": {
+                    '/err/nat44-ed-in2out-slowpath/out of ports': 0,
+                },
+                self.pg0.sw_if_index: {
+                    '/nat44-ed/in2out/slowpath/drops': 0,
+                    '/nat44-ed/in2out/slowpath/tcp': tcpn,
+                    '/nat44-ed/in2out/slowpath/udp': udpn,
+                    '/nat44-ed/in2out/slowpath/icmp': icmpn,
+                },
+            }
+        )
 
     def test_unknown_proto(self):
         """ NAT44ED translate packet with unknown protocol """
diff --git a/test/util.py b/test/util.py
index e21fdb81026..653b667eb6c 100644
--- a/test/util.py
+++ b/test/util.py
@@ -1,12 +1,11 @@
 """ test framework utilities """
 
-import abc
 import ipaddress
 import logging
 import socket
 from socket import AF_INET6
-import sys
 import os.path
+from copy import deepcopy
 
 import scapy.compat
 from scapy.layers.l2 import Ether
@@ -452,3 +451,50 @@ def reassemble4_ether(listoffragments):
 
 def reassemble4(listoffragments):
     return reassemble4_core(listoffragments, True)
+
+
+def recursive_dict_merge(dict_base, dict_update):
+    """Recursively merge base dict with update dict, return merged dict"""
+    for key in dict_update:
+        if key in dict_base:
+            if type(dict_update[key]) is dict:
+                dict_base[key] = recursive_dict_merge(dict_base[key],
+                                                      dict_update[key])
+            else:
+                dict_base[key] = dict_update[key]
+        else:
+            dict_base[key] = dict_update[key]
+    return dict_base
+
+
+class StatsDiff:
+    """
+    Diff dictionary is a dictionary of dictionaries of interesting stats:
+
+        diff_dictionary =
+        {
+            "err" : { '/error/counter1' : 4, },
+            sw_if_index1 : { '/stat/segment/counter1' : 5,
+                             '/stat/segment/counter2' : 6,
+                           },
+            sw_if_index2 : { '/stat/segment/counter1' : 7,
+                           },
+        }
+
+    It describes a per sw-if-index diffset, where each key is stat segment
+    path and value is the expected change for that counter for sw-if-index.
+    Special case string "err" is used for error counters, which are not per
+    sw-if-index.
+    """
+
+    def __init__(self, stats_diff={}):
+        self.stats_diff = stats_diff
+
+    def update(self, sw_if_index, key, value):
+        if sw_if_index in self.stats_diff:
+            self.stats_diff[sw_if_index][key] = value
+        else:
+            self.stats_diff[sw_if_index] = {key: value}
+
+    def __or__(self, other):
+        return recursive_dict_merge(deepcopy(self.stats_diff), other)