diff options
| -rw-r--r-- | test/framework.py | 86 | ||||
| -rw-r--r-- | test/test_nat44_ed.py | 79 | ||||
| -rw-r--r-- | test/util.py | 50 |
3 files changed, 175 insertions, 40 deletions
diff --git a/test/framework.py b/test/framework.py index f0d916fd9a6..1c81e8afabc 100644 --- a/test/framework.py +++ b/test/framework.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 from __future__ import print_function -import gc import logging import sys import os @@ -1264,26 +1263,82 @@ class VppTestCase(CPUInterface, unittest.TestCase): self.pg_enable_capture(self.pg_interfaces) self.pg_start(trace=trace) + def snapshot_stats(self, stats_diff): + """Return snapshot of interesting stats based on diff dictionary.""" + stats_snapshot = {} + for sw_if_index in stats_diff: + for counter in stats_diff[sw_if_index]: + stats_snapshot[counter] = self.statistics[counter] + self.logger.debug(f"Took statistics stats_snapshot: {stats_snapshot}") + return stats_snapshot + + def compare_stats_with_snapshot(self, stats_diff, stats_snapshot): + """Assert appropriate difference between current stats and snapshot.""" + for sw_if_index in stats_diff: + for cntr, diff in stats_diff[sw_if_index].items(): + if sw_if_index == "err": + self.assert_equal( + self.statistics[cntr].sum(), + stats_snapshot[cntr].sum() + diff, + f"'{cntr}' counter value (previous value: " + f"{stats_snapshot[cntr].sum()}, " + f"expected diff: {diff})") + else: + try: + self.assert_equal( + self.statistics[cntr][:, sw_if_index].sum(), + stats_snapshot[cntr][:, sw_if_index].sum() + diff, + f"'{cntr}' counter value (previous value: " + f"{stats_snapshot[cntr][:, sw_if_index].sum()}, " + f"expected diff: {diff})") + except IndexError: + # if diff is 0, then this most probably a case where + # test declares multiple interfaces but traffic hasn't + # passed through this one yet - which means the counter + # value is 0 and can be ignored + if 0 != diff: + raise + def send_and_assert_no_replies(self, intf, pkts, remark="", timeout=None, - trace=True): + stats_diff=None, trace=True, msg=None): + if stats_diff: + stats_snapshot = self.snapshot_stats(stats_diff) + self.pg_send(intf, pkts) - if not timeout: - timeout = 1 - for i in self.pg_interfaces: - i.get_capture(0, timeout=timeout) - i.assert_nothing_captured(remark=remark) - timeout = 0.1 - if trace: - self.logger.debug(self.vapi.cli("show trace")) + + try: + if not timeout: + timeout = 1 + for i in self.pg_interfaces: + i.get_capture(0, timeout=timeout) + i.assert_nothing_captured(remark=remark) + timeout = 0.1 + finally: + if trace: + if msg: + self.logger.debug(f"send_and_assert_no_replies: {msg}") + self.logger.debug(self.vapi.cli("show trace")) + + if stats_diff: + self.compare_stats_with_snapshot(stats_diff, stats_snapshot) def send_and_expect(self, intf, pkts, output, n_rx=None, worker=None, - trace=True): + trace=True, msg=None, stats_diff=None): + if stats_diff: + stats_snapshot = self.snapshot_stats(stats_diff) + if not n_rx: n_rx = 1 if isinstance(pkts, Packet) else len(pkts) self.pg_send(intf, pkts, worker=worker, trace=trace) rx = output.get_capture(n_rx) if trace: + if msg: + self.logger.debug(f"send_and_expect: {msg}") self.logger.debug(self.vapi.cli("show trace")) + + if stats_diff: + self.compare_stats_with_snapshot(stats_diff, stats_snapshot) + return rx def send_and_expect_load_balancing(self, input, pkts, outputs, @@ -1298,7 +1353,11 @@ class VppTestCase(CPUInterface, unittest.TestCase): self.logger.debug(self.vapi.cli("show trace")) return rxs - def send_and_expect_only(self, intf, pkts, output, timeout=None): + def send_and_expect_only(self, intf, pkts, output, timeout=None, + stats_diff=None): + if stats_diff: + stats_snapshot = self.snapshot_stats(stats_diff) + self.pg_send(intf, pkts) rx = output.get_capture(len(pkts)) outputs = [output] @@ -1310,6 +1369,9 @@ class VppTestCase(CPUInterface, unittest.TestCase): i.assert_nothing_captured() timeout = 0.1 + if stats_diff: + self.compare_stats_with_snapshot(stats_diff, stats_snapshot) + return rx diff --git a/test/test_nat44_ed.py b/test/test_nat44_ed.py index 9bb803e4435..764693d636a 100644 --- a/test/test_nat44_ed.py +++ b/test/test_nat44_ed.py @@ -2,7 +2,7 @@ import unittest from io import BytesIO -from random import randint, shuffle, choice +from random import randint, choice import scapy.compat from framework import VppTestCase, VppTestRunner @@ -17,6 +17,7 @@ from util import ppp, ip4_range from vpp_acl import AclRule, VppAcl, VppAclInterface from vpp_ip_route import VppIpRoute, VppRoutePath from vpp_papi import VppEnum +from util import StatsDiff class TestNAT44ED(VppTestCase): @@ -213,6 +214,28 @@ class TestNAT44ED(VppTestCase): for r in rl: r.add_vpp_config() + cls.no_diff = StatsDiff({ + pg.sw_if_index: { + '/nat44-ed/in2out/fastpath/tcp': 0, + '/nat44-ed/in2out/fastpath/udp': 0, + '/nat44-ed/in2out/fastpath/icmp': 0, + '/nat44-ed/in2out/fastpath/drops': 0, + '/nat44-ed/in2out/slowpath/tcp': 0, + '/nat44-ed/in2out/slowpath/udp': 0, + '/nat44-ed/in2out/slowpath/icmp': 0, + '/nat44-ed/in2out/slowpath/drops': 0, + '/nat44-ed/in2out/fastpath/tcp': 0, + '/nat44-ed/in2out/fastpath/udp': 0, + '/nat44-ed/in2out/fastpath/icmp': 0, + '/nat44-ed/in2out/fastpath/drops': 0, + '/nat44-ed/in2out/slowpath/tcp': 0, + '/nat44-ed/in2out/slowpath/udp': 0, + '/nat44-ed/in2out/slowpath/icmp': 0, + '/nat44-ed/in2out/slowpath/drops': 0, + } + for pg in cls.pg_interfaces + }) + def get_err_counter(self, path): return self.statistics.get_err_counter(path) @@ -2622,37 +2645,41 @@ class TestNAT44EDMW(TestNAT44ED): self.nat_add_outside_interface(self.pg1) # in2out and no NAT addresses added - err_old = self.statistics.get_err_counter( - '/err/nat44-ed-in2out-slowpath/out of ports') - pkts = self.create_stream_in(self.pg0, self.pg1) - self.pg0.add_stream(pkts) - self.pg_enable_capture(self.pg_interfaces) - self.pg_start() - self.pg1.get_capture(0, timeout=1) - err_new = self.statistics.get_err_counter( - '/err/nat44-ed-in2out-slowpath/out of ports') - - self.assertEqual(err_new - err_old, len(pkts)) + self.send_and_assert_no_replies( + self.pg0, pkts, msg="i2o pkts", + stats_diff=self.no_diff | { + "err": { + '/err/nat44-ed-in2out-slowpath/out of ports': len(pkts), + }, + self.pg0.sw_if_index: { + '/nat44-ed/in2out/slowpath/drops': len(pkts), + }, + } + ) # in2out after NAT addresses added self.nat_add_address(self.nat_addr) - err_old = self.statistics.get_err_counter( - '/err/nat44-ed-in2out-slowpath/out of ports') - - pkts = self.create_stream_in(self.pg0, self.pg1) - self.pg0.add_stream(pkts) - self.pg_enable_capture(self.pg_interfaces) - self.pg_start() - capture = self.pg1.get_capture(len(pkts)) - self.verify_capture_out(capture, ignore_port=True) - - err_new = self.statistics.get_err_counter( - '/err/nat44-ed-in2out-slowpath/out of ports') - - self.assertEqual(err_new, err_old) + tcpn, udpn, icmpn = (sum(x) for x in + zip(*((TCP in p, UDP in p, ICMP in p) + for p in pkts))) + + self.send_and_expect( + self.pg0, pkts, self.pg1, msg="i2o pkts", + stats_diff=self.no_diff | { + "err": { + '/err/nat44-ed-in2out-slowpath/out of ports': 0, + }, + self.pg0.sw_if_index: { + '/nat44-ed/in2out/slowpath/drops': 0, + '/nat44-ed/in2out/slowpath/tcp': tcpn, + '/nat44-ed/in2out/slowpath/udp': udpn, + '/nat44-ed/in2out/slowpath/icmp': icmpn, + }, + } + ) def test_unknown_proto(self): """ NAT44ED translate packet with unknown protocol """ diff --git a/test/util.py b/test/util.py index e21fdb81026..653b667eb6c 100644 --- a/test/util.py +++ b/test/util.py @@ -1,12 +1,11 @@ """ test framework utilities """ -import abc import ipaddress import logging import socket from socket import AF_INET6 -import sys import os.path +from copy import deepcopy import scapy.compat from scapy.layers.l2 import Ether @@ -452,3 +451,50 @@ def reassemble4_ether(listoffragments): def reassemble4(listoffragments): return reassemble4_core(listoffragments, True) + + +def recursive_dict_merge(dict_base, dict_update): + """Recursively merge base dict with update dict, return merged dict""" + for key in dict_update: + if key in dict_base: + if type(dict_update[key]) is dict: + dict_base[key] = recursive_dict_merge(dict_base[key], + dict_update[key]) + else: + dict_base[key] = dict_update[key] + else: + dict_base[key] = dict_update[key] + return dict_base + + +class StatsDiff: + """ + Diff dictionary is a dictionary of dictionaries of interesting stats: + + diff_dictionary = + { + "err" : { '/error/counter1' : 4, }, + sw_if_index1 : { '/stat/segment/counter1' : 5, + '/stat/segment/counter2' : 6, + }, + sw_if_index2 : { '/stat/segment/counter1' : 7, + }, + } + + It describes a per sw-if-index diffset, where each key is stat segment + path and value is the expected change for that counter for sw-if-index. + Special case string "err" is used for error counters, which are not per + sw-if-index. + """ + + def __init__(self, stats_diff={}): + self.stats_diff = stats_diff + + def update(self, sw_if_index, key, value): + if sw_if_index in self.stats_diff: + self.stats_diff[sw_if_index][key] = value + else: + self.stats_diff[sw_if_index] = {key: value} + + def __or__(self, other): + return recursive_dict_merge(deepcopy(self.stats_diff), other) |