ipsec: Test and fix IPSec worker hand-off

Type: fix

Change-Id: I5cb9a3845ddbc5f4de4eb4e9c481f606fe5cec9a
Signed-off-by: Neale Ranns <nranns@cisco.com>

9 files changed, 138 insertions, 27 deletions

diff --git a/src/vnet/ipsec/ah_decrypt.c b/src/vnet/ipsec/ah_decrypt.c
index 22f9a09453e..682f6cc91f0 100644
--- a/src/vnet/ipsec/ah_decrypt.c
+++ b/src/vnet/ipsec/ah_decrypt.c
@@ -440,7 +440,7 @@ VLIB_REGISTER_NODE (ah4_decrypt_node) = {
     [AH_DECRYPT_NEXT_DROP] = "ip4-drop",
     [AH_DECRYPT_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
     [AH_DECRYPT_NEXT_IP6_INPUT] = "ip6-input",
-    [AH_DECRYPT_NEXT_HANDOFF] = "esp4-decrypt-tun-handoff",
+    [AH_DECRYPT_NEXT_HANDOFF] = "ah4-decrypt-handoff",
   },
 };
 /* *INDENT-ON* */
@@ -467,7 +467,7 @@ VLIB_REGISTER_NODE (ah6_decrypt_node) = {
     [AH_DECRYPT_NEXT_DROP] = "ip6-drop",
     [AH_DECRYPT_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
     [AH_DECRYPT_NEXT_IP6_INPUT] = "ip6-input",
-    [AH_DECRYPT_NEXT_HANDOFF] = "esp6-decrypt-handoff",
+    [AH_DECRYPT_NEXT_HANDOFF] = "ah6-decrypt-handoff",
   },
 };
 /* *INDENT-ON* */
diff --git a/src/vnet/ipsec/esp_decrypt.c b/src/vnet/ipsec/esp_decrypt.c
index 8afea9b32a0..16ae3a3d9eb 100644
--- a/src/vnet/ipsec/esp_decrypt.c
+++ b/src/vnet/ipsec/esp_decrypt.c
@@ -685,7 +685,7 @@ VLIB_REGISTER_NODE (esp4_decrypt_tun_node) = {
     [ESP_DECRYPT_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
     [ESP_DECRYPT_NEXT_IP6_INPUT] = "ip6-input",
     [ESP_DECRYPT_NEXT_L2_INPUT] = "l2-input",
-    [ESP_DECRYPT_NEXT_HANDOFF] = "esp4-decrypt-handoff",
+    [ESP_DECRYPT_NEXT_HANDOFF] = "esp4-decrypt-tun-handoff",
   },
 };
 
@@ -702,7 +702,7 @@ VLIB_REGISTER_NODE (esp6_decrypt_tun_node) = {
     [ESP_DECRYPT_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
     [ESP_DECRYPT_NEXT_IP6_INPUT] = "ip6-input",
     [ESP_DECRYPT_NEXT_L2_INPUT] = "l2-input",
-    [ESP_DECRYPT_NEXT_HANDOFF]=  "esp6-decrypt-handoff",
+    [ESP_DECRYPT_NEXT_HANDOFF]=  "esp6-decrypt-tun-handoff",
   },
 };
 /* *INDENT-ON* */
diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c
index 58e25f6b85e..4f1bb802bcb 100644
--- a/src/vnet/ipsec/esp_encrypt.c
+++ b/src/vnet/ipsec/esp_encrypt.c
@@ -642,7 +642,7 @@ VLIB_REGISTER_NODE (esp4_encrypt_tun_node) = {
   .n_next_nodes = ESP_ENCRYPT_N_NEXT,
   .next_nodes = {
     [ESP_ENCRYPT_NEXT_DROP] = "ip4-drop",
-    [ESP_ENCRYPT_NEXT_HANDOFF] = "esp4-encrypt-handoff",
+    [ESP_ENCRYPT_NEXT_HANDOFF] = "esp4-encrypt-tun-handoff",
     [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "error-drop",
   },
 };
@@ -689,7 +689,7 @@ VLIB_REGISTER_NODE (esp6_encrypt_tun_node) = {
   .n_next_nodes = ESP_ENCRYPT_N_NEXT,
   .next_nodes = {
     [ESP_ENCRYPT_NEXT_DROP] = "ip6-drop",
-    [ESP_ENCRYPT_NEXT_HANDOFF] = "esp6-encrypt-handoff",
+    [ESP_ENCRYPT_NEXT_HANDOFF] = "esp6-encrypt-tun-handoff",
     [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "error-drop",
   },
 };
diff --git a/test/framework.py b/test/framework.py
index 7ff7978a055..7d1955f5c7a 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -1133,9 +1133,9 @@ class VppTestCase(unittest.TestCase):
                 "Finished sleep (%s) - slept %es (wanted %es)",
                 remark, after - before, timeout)
 
-    def pg_send(self, intf, pkts):
+    def pg_send(self, intf, pkts, worker=None):
         self.vapi.cli("clear trace")
-        intf.add_stream(pkts)
+        intf.add_stream(pkts, worker=worker)
         self.pg_enable_capture(self.pg_interfaces)
         self.pg_start()
 
@@ -1148,10 +1148,10 @@ class VppTestCase(unittest.TestCase):
             i.assert_nothing_captured(remark=remark)
             timeout = 0.1
 
-    def send_and_expect(self, intf, pkts, output, n_rx=None):
+    def send_and_expect(self, intf, pkts, output, n_rx=None, worker=None):
         if not n_rx:
             n_rx = len(pkts)
-        self.pg_send(intf, pkts)
+        self.pg_send(intf, pkts, worker=worker)
         rx = output.get_capture(n_rx)
         return rx
 
diff --git a/test/template_ipsec.py b/test/template_ipsec.py
index 034bc8ecf45..bcfc0d9253a 100644
--- a/test/template_ipsec.py
+++ b/test/template_ipsec.py
@@ -660,21 +660,21 @@ class IpsecTra46Tests(IpsecTra4Tests, IpsecTra6Tests):
 
 class IpsecTun4(object):
     """ verify methods for Tunnel v4 """
-    def verify_counters4(self, p, count, n_frags=None):
+    def verify_counters4(self, p, count, n_frags=None, worker=None):
         if not n_frags:
             n_frags = count
         if (hasattr(p, "spd_policy_in_any")):
-            pkts = p.spd_policy_in_any.get_stats()['packets']
+            pkts = p.spd_policy_in_any.get_stats(worker)['packets']
             self.assertEqual(pkts, count,
                              "incorrect SPD any policy: expected %d != %d" %
                              (count, pkts))
 
         if (hasattr(p, "tun_sa_in")):
-            pkts = p.tun_sa_in.get_stats()['packets']
+            pkts = p.tun_sa_in.get_stats(worker)['packets']
             self.assertEqual(pkts, count,
                              "incorrect SA in counts: expected %d != %d" %
                              (count, pkts))
-            pkts = p.tun_sa_out.get_stats()['packets']
+            pkts = p.tun_sa_out.get_stats(worker)['packets']
             self.assertEqual(pkts, count,
                              "incorrect SA out counts: expected %d != %d" %
                              (count, pkts))
@@ -840,14 +840,14 @@ class IpsecTun4Tests(IpsecTun4):
 
 class IpsecTun6(object):
     """ verify methods for Tunnel v6 """
-    def verify_counters6(self, p_in, p_out, count):
+    def verify_counters6(self, p_in, p_out, count, worker=None):
         if (hasattr(p_in, "tun_sa_in")):
-            pkts = p_in.tun_sa_in.get_stats()['packets']
+            pkts = p_in.tun_sa_in.get_stats(worker)['packets']
             self.assertEqual(pkts, count,
                              "incorrect SA in counts: expected %d != %d" %
                              (count, pkts))
         if (hasattr(p_out, "tun_sa_out")):
-            pkts = p_out.tun_sa_out.get_stats()['packets']
+            pkts = p_out.tun_sa_out.get_stats(worker)['packets']
             self.assertEqual(pkts, count,
                              "incorrect SA out counts: expected %d != %d" %
                              (count, pkts))
@@ -908,8 +908,7 @@ class IpsecTun6(object):
                                        dst=p_out.remote_tun_if_host,
                                        count=count,
                                        payload_size=payload_size)
-            recv_pkts = self.send_and_expect(self.pg1, send_pkts,
-                                             self.tun_if)
+            recv_pkts = self.send_and_expect(self.pg1, send_pkts, self.tun_if)
             self.verify_encrypted6(p_out, p_out.vpp_tun_sa, recv_pkts)
 
         finally:
@@ -1002,6 +1001,68 @@ class IpsecTun6Tests(IpsecTun6):
         self.verify_tun_66(self.params[socket.AF_INET6], count=257)
 
 
+class IpsecTun6HandoffTests(IpsecTun6):
+    """ UT test methods for Tunnel v6 with multiple workers """
+    worker_config = "workers 2"
+
+    def test_tun_handoff_66(self):
+        """ ipsec 6o6 tunnel worker hand-off test """
+        N_PKTS = 15
+        p = self.params[socket.AF_INET6]
+
+        # inject alternately on worker 0 and 1. all counts on the SA
+        # should be against worker 0
+        for worker in [0, 1, 0, 1]:
+            send_pkts = self.gen_encrypt_pkts6(p.scapy_tun_sa, self.tun_if,
+                                               src=p.remote_tun_if_host,
+                                               dst=self.pg1.remote_ip6,
+                                               count=N_PKTS)
+            recv_pkts = self.send_and_expect(self.tun_if, send_pkts,
+                                             self.pg1, worker=worker)
+            self.verify_decrypted6(p, recv_pkts)
+
+            send_pkts = self.gen_pkts6(self.pg1, src=self.pg1.remote_ip6,
+                                       dst=p.remote_tun_if_host,
+                                       count=N_PKTS)
+            recv_pkts = self.send_and_expect(self.pg1, send_pkts,
+                                             self.tun_if, worker=worker)
+            self.verify_encrypted6(p, p.vpp_tun_sa, recv_pkts)
+
+        # all counts against the first worker that was used
+        self.verify_counters6(p, p, 4*N_PKTS, worker=0)
+
+
+class IpsecTun4HandoffTests(IpsecTun4):
+    """ UT test methods for Tunnel v4 with multiple workers """
+    worker_config = "workers 2"
+
+    def test_tun_handooff_44(self):
+        """ ipsec 4o4 tunnel worker hand-off test """
+        N_PKTS = 15
+        p = self.params[socket.AF_INET]
+
+        # inject alternately on worker 0 and 1. all counts on the SA
+        # should be against worker 0
+        for worker in [0, 1, 0, 1]:
+            send_pkts = self.gen_encrypt_pkts(p.scapy_tun_sa, self.tun_if,
+                                              src=p.remote_tun_if_host,
+                                              dst=self.pg1.remote_ip4,
+                                              count=N_PKTS)
+            recv_pkts = self.send_and_expect(self.tun_if, send_pkts,
+                                             self.pg1, worker=worker)
+            self.verify_decrypted(p, recv_pkts)
+
+            send_pkts = self.gen_pkts(self.pg1, src=self.pg1.remote_ip4,
+                                      dst=p.remote_tun_if_host,
+                                      count=N_PKTS)
+            recv_pkts = self.send_and_expect(self.pg1, send_pkts,
+                                             self.tun_if, worker=worker)
+            self.verify_encrypted(p, p.vpp_tun_sa, recv_pkts)
+
+        # all counts against the first worker that was used
+        self.verify_counters4(p, 4*N_PKTS, worker=0)
+
+
 class IpsecTun46Tests(IpsecTun4Tests, IpsecTun6Tests):
     """ UT test methods for Tunnel v6 & v4 """
     pass
diff --git a/test/test_ipsec_ah.py b/test/test_ipsec_ah.py
index 73577cd2197..bc539a2bd03 100644
--- a/test/test_ipsec_ah.py
+++ b/test/test_ipsec_ah.py
@@ -6,7 +6,8 @@ from scapy.layers.ipsec import AH
 from framework import VppTestRunner
 from template_ipsec import TemplateIpsec, IpsecTra46Tests, IpsecTun46Tests, \
     config_tun_params, config_tra_params, IPsecIPv4Params, IPsecIPv6Params, \
-    IpsecTra4, IpsecTun4, IpsecTra6, IpsecTun6
+    IpsecTra4, IpsecTun4, IpsecTra6, IpsecTun6, \
+    IpsecTun6HandoffTests, IpsecTun4HandoffTests
 from template_ipsec import IpsecTcpTests
 from vpp_ipsec import VppIpsecSA, VppIpsecSpd, VppIpsecSpdEntry,\
         VppIpsecSpdItfBinding
@@ -301,6 +302,13 @@ class TestIpsecAh2(TemplateIpsecAh, IpsecTra46Tests, IpsecTun46Tests):
     pass
 
 
+class TestIpsecAhHandoff(TemplateIpsecAh,
+                         IpsecTun6HandoffTests,
+                         IpsecTun4HandoffTests):
+    """ Ipsec AH Handoff """
+    pass
+
+
 class TestIpsecAhAll(ConfigIpsecAH,
                      IpsecTra4, IpsecTra6,
                      IpsecTun4, IpsecTun6):
diff --git a/test/test_ipsec_esp.py b/test/test_ipsec_esp.py
index 69b48ae8e13..82346d64708 100644
--- a/test/test_ipsec_esp.py
+++ b/test/test_ipsec_esp.py
@@ -8,7 +8,8 @@ from framework import VppTestRunner
 from template_ipsec import IpsecTra46Tests, IpsecTun46Tests, TemplateIpsec, \
     IpsecTcpTests, IpsecTun4Tests, IpsecTra4Tests, config_tra_params, \
     config_tun_params, IPsecIPv4Params, IPsecIPv6Params, \
-    IpsecTra4, IpsecTun4, IpsecTra6, IpsecTun6
+    IpsecTra4, IpsecTun4, IpsecTra6, IpsecTun6, \
+    IpsecTun6HandoffTests, IpsecTun4HandoffTests
 from vpp_ipsec import VppIpsecSpd, VppIpsecSpdEntry, VppIpsecSA,\
     VppIpsecSpdItfBinding
 from vpp_ip_route import VppIpRoute, VppRoutePath
@@ -295,6 +296,13 @@ class TestIpsecEsp2(TemplateIpsecEsp, IpsecTcpTests):
     pass
 
 
+class TestIpsecEspHandoff(TemplateIpsecEsp,
+                          IpsecTun6HandoffTests,
+                          IpsecTun4HandoffTests):
+    """ Ipsec ESP - handoff tests """
+    pass
+
+
 class TemplateIpsecEspUdp(ConfigIpsecESP):
     """
     UDP encapped ESP
diff --git a/test/test_ipsec_tun_if_esp.py b/test/test_ipsec_tun_if_esp.py
index 68d6b588b3c..eefd477c71d 100644
--- a/test/test_ipsec_tun_if_esp.py
+++ b/test/test_ipsec_tun_if_esp.py
@@ -9,7 +9,8 @@ from scapy.layers.inet import IP, UDP
 from scapy.layers.inet6 import IPv6
 from framework import VppTestRunner
 from template_ipsec import TemplateIpsec, IpsecTun4Tests, IpsecTun6Tests, \
-    IpsecTun4, IpsecTun6,  IpsecTcpTests, mk_scapy_crypt_key
+    IpsecTun4, IpsecTun6,  IpsecTcpTests, mk_scapy_crypt_key, \
+    IpsecTun6HandoffTests, IpsecTun4HandoffTests
 from vpp_ipsec_tun_interface import VppIpsecTunInterface
 from vpp_gre_interface import VppGreInterface
 from vpp_ipip_tun_interface import VppIpIpTunInterface
@@ -243,7 +244,8 @@ class TemplateIpsec6TunIfEsp(TemplateIpsec):
         super(TemplateIpsec6TunIfEsp, self).tearDown()
 
 
-class TestIpsec6TunIfEsp1(TemplateIpsec6TunIfEsp, IpsecTun6Tests):
+class TestIpsec6TunIfEsp1(TemplateIpsec6TunIfEsp,
+                          IpsecTun6Tests):
     """ Ipsec ESP - TUN tests """
     tun6_encrypt_node_name = "esp6-encrypt-tun"
     tun6_decrypt_node_name = "esp6-decrypt-tun"
@@ -259,6 +261,20 @@ class TestIpsec6TunIfEsp1(TemplateIpsec6TunIfEsp, IpsecTun6Tests):
         self.verify_tun_46(self.params[socket.AF_INET6], count=257)
 
 
+class TestIpsec6TunIfEspHandoff(TemplateIpsec6TunIfEsp,
+                                IpsecTun6HandoffTests):
+    """ Ipsec ESP 6 Handoff tests """
+    tun6_encrypt_node_name = "esp6-encrypt-tun"
+    tun6_decrypt_node_name = "esp6-decrypt-tun"
+
+
+class TestIpsec4TunIfEspHandoff(TemplateIpsec4TunIfEsp,
+                                IpsecTun4HandoffTests):
+    """ Ipsec ESP 4 Handoff tests """
+    tun4_encrypt_node_name = "esp4-encrypt-tun"
+    tun4_decrypt_node_name = "esp4-decrypt-tun"
+
+
 class TestIpsec4MultiTunIfEsp(TemplateIpsec, IpsecTun4):
     """ IPsec IPv4 Multi Tunnel interface """
 
diff --git a/test/vpp_ipsec.py b/test/vpp_ipsec.py
index 0df8cb2a88a..8144ea27c8f 100644
--- a/test/vpp_ipsec.py
+++ b/test/vpp_ipsec.py
@@ -8,6 +8,10 @@ except NameError:
     text_type = str
 
 
+def mk_counter():
+    return {'packets': 0, 'bytes': 0}
+
+
 class VppIpsecSpd(VppObject):
     """
     VPP SPD DB
@@ -163,9 +167,16 @@ class VppIpsecSpdEntry(VppObject):
                 return True
         return False
 
-    def get_stats(self):
+    def get_stats(self, worker=None):
         c = self.test.statistics.get_counter("/net/ipsec/policy")
-        return c[0][self.stat_index]
+        if worker is None:
+            total = mk_counter()
+            for t in c:
+                total['packets'] += t[self.stat_index]['packets']
+            return total
+        else:
+            # +1 to skip main thread
+            return c[worker+1][self.stat_index]
 
 
 class VppIpsecSA(VppObject):
@@ -244,9 +255,16 @@ class VppIpsecSA(VppObject):
                 return True
         return False
 
-    def get_stats(self):
+    def get_stats(self, worker=None):
         c = self.test.statistics.get_counter("/net/ipsec/sa")
-        return c[0][self.stat_index]
+        if worker is None:
+            total = mk_counter()
+            for t in c:
+                total['packets'] += t[self.stat_index]['packets']
+            return total
+        else:
+            # +1 to skip main thread
+            return c[worker+1][self.stat_index]
 
 
 class VppIpsecTunProtect(VppObject):