vlib: Punt reason allocation listener enable/disable callback

Type: improvement

allow clients that allocate punt reasons to pass a callback function
that is invoked when the first/last client registers to use/listen on
that punt reason. This allows the client to perform some necessary
configs that might not otherwise be enabled.
IPSec uses this callback to register the ESP proto and UDP handling
nodes, that would not otherwise be enabled unless a tunnel was present.

Change-Id: I9759349903f21ffeeb253d4271e619e6bf46054b
Signed-off-by: Neale Ranns <nranns@cisco.com>

9 files changed, 158 insertions, 53 deletions

diff --git a/src/plugins/unittest/punt_test.c b/src/plugins/unittest/punt_test.c
index 4a7503687db..431624fbea0 100644
--- a/src/plugins/unittest/punt_test.c
+++ b/src/plugins/unittest/punt_test.c
@@ -317,9 +317,11 @@ punt_test (vlib_main_t * vm,
 	  punt_hdl = vlib_punt_client_register ("test");
 
 	  rc =
-	    vlib_punt_reason_alloc (punt_hdl, "reason-v4", &punt_reason_v4);
+	    vlib_punt_reason_alloc (punt_hdl, "reason-v4",
+				    NULL, NULL, &punt_reason_v4);
 	  rc |=
-	    vlib_punt_reason_alloc (punt_hdl, "reason-v6", &punt_reason_v6);
+	    vlib_punt_reason_alloc (punt_hdl, "reason-v6",
+				    NULL, NULL, &punt_reason_v6);
 	  ASSERT (!rc);
 
 	  vnet_feature_enable_disable ("ip4-unicast",
diff --git a/src/vlib/punt.c b/src/vlib/punt.c
index 7c2daf25ee4..d5e13004caf 100644
--- a/src/vlib/punt.c
+++ b/src/vlib/punt.c
@@ -47,6 +47,21 @@ typedef struct punt_reason_data_t_
    * Clients/owners that have registered this reason
    */
   u32 *pd_owners;
+
+  /**
+   * clients interested/listening to this reason
+   */
+  u32 pd_users;
+
+  /**
+   * function to invoke if a client becomes interested in the code.
+   */
+  punt_interested_listener_t pd_fn;
+
+  /**
+   * Data to pass to the callback
+   */
+  void *pd_data;
 } punt_reason_data_t;
 
 /**
@@ -249,8 +264,8 @@ punt_reg_mk_dp (vlib_punt_reason_t reason)
 }
 
 int
-vlib_punt_register (vlib_punt_hdl_t client, vlib_punt_reason_t reason,
-		    const char *node_name)
+vlib_punt_register (vlib_punt_hdl_t client,
+		    vlib_punt_reason_t reason, const char *node_name)
 {
   vlib_node_t *punt_to, *punt_from;
   punt_client_t *pc;
@@ -298,6 +313,11 @@ vlib_punt_register (vlib_punt_hdl_t client, vlib_punt_reason_t reason,
 
       pri = pr - punt_reg_pool;
 
+      if (0 == punt_reason_data[reason].pd_users++ &&
+	  NULL != punt_reason_data[reason].pd_fn)
+	punt_reason_data[reason].pd_fn (VLIB_ENABLE,
+					punt_reason_data[reason].pd_data);
+
       punt_reg_add (pr);
     }
 
@@ -353,6 +373,10 @@ vlib_punt_unregister (vlib_punt_hdl_t client,
 
       if (0 == pr->pr_locks)
 	{
+	  if (0 == --punt_reason_data[reason].pd_users &&
+	      NULL != punt_reason_data[reason].pd_fn)
+	    punt_reason_data[reason].pd_fn (VLIB_DISABLE,
+					    punt_reason_data[reason].pd_data);
 	  punt_reg_remove (pr);
 	  pool_put (punt_reg_pool, pr);
 	}
@@ -377,7 +401,9 @@ vlib_punt_reason_validate (vlib_punt_reason_t reason)
 
 int
 vlib_punt_reason_alloc (vlib_punt_hdl_t client,
-			const char *reason_name, vlib_punt_reason_t * reason)
+			const char *reason_name,
+			punt_interested_listener_t fn,
+			void *data, vlib_punt_reason_t * reason)
 {
   vlib_punt_reason_t new;
 
@@ -388,6 +414,8 @@ vlib_punt_reason_alloc (vlib_punt_hdl_t client,
   vec_validate (punt_reason_data, new);
   punt_reason_data[new].pd_name = format (NULL, "%s", reason_name);
   punt_reason_data[new].pd_reason = new;
+  punt_reason_data[new].pd_fn = fn;
+  punt_reason_data[new].pd_data = data;
   vec_add1 (punt_reason_data[new].pd_owners, client);
 
   vlib_validate_combined_counter (&punt_counters, new);
diff --git a/src/vlib/punt.h b/src/vlib/punt.h
index 7a3e5da2da6..d93b5eac599 100644
--- a/src/vlib/punt.h
+++ b/src/vlib/punt.h
@@ -56,12 +56,19 @@ typedef int vlib_punt_hdl_t;
  */
 vlib_punt_hdl_t vlib_punt_client_register (const char *who);
 
+typedef void (*punt_interested_listener_t) (vlib_enable_or_disable_t i,
+					    void *data);
+
 /**
  * Allocate a new punt reason
+ * @param fn     - A callback to invoke when an entity becomes [un]interested
+ *                 in the punt code.
+ * @param data   - To be passed in the callback function.
  */
 extern int vlib_punt_reason_alloc (vlib_punt_hdl_t client,
 				   const char *reason_name,
-				   vlib_punt_reason_t * reason);
+				   punt_interested_listener_t fn,
+				   void *data, vlib_punt_reason_t * reason);
 
 /**
  * Validate that a punt reason is assigned
diff --git a/src/vnet/ip/ip_types.h b/src/vnet/ip/ip_types.h
index dc07d23087d..720ef1a5edf 100644
--- a/src/vnet/ip/ip_types.h
+++ b/src/vnet/ip/ip_types.h
@@ -24,6 +24,8 @@ typedef enum ip_address_family_t_
   AF_IP6,
 } ip_address_family_t;
 
+#define N_AF (AF_IP6+1)
+
 extern uword unformat_ip_address_family (unformat_input_t * input,
 					 va_list * args);
 extern u8 *format_ip_address_family (u8 * s, va_list * args);
diff --git a/src/vnet/ipsec/ipsec_punt.c b/src/vnet/ipsec/ipsec_punt.c
index 357e9596cfc..a08231ab299 100644
--- a/src/vnet/ipsec/ipsec_punt.c
+++ b/src/vnet/ipsec/ipsec_punt.c
@@ -17,11 +17,27 @@
 
 #include <vnet/ipsec/ipsec.h>
 #include <vnet/ipsec/ipsec_punt.h>
+#include <vnet/ipsec/ipsec_tun.h>
 
 static vlib_punt_hdl_t punt_hdl;
 
 vlib_punt_reason_t ipsec_punt_reason[IPSEC_PUNT_N_REASONS];
 
+static void
+ipsec_punt_interested_listener (vlib_enable_or_disable_t action, void *data)
+{
+  if (action == VLIB_ENABLE)
+    {
+      ipsec_tun_register_nodes (AF_IP4);
+      ipsec_tun_register_nodes (AF_IP6);
+    }
+  else
+    {
+      ipsec_tun_unregister_nodes (AF_IP4);
+      ipsec_tun_unregister_nodes (AF_IP6);
+    }
+}
+
 static clib_error_t *
 ipsec_punt_init (vlib_main_t * vm)
 {
@@ -33,6 +49,8 @@ ipsec_punt_init (vlib_main_t * vm)
   punt_hdl = vlib_punt_client_register ("ipsec");
 
 #define _(s,v)  vlib_punt_reason_alloc (punt_hdl, v,                    \
+                                        ipsec_punt_interested_listener, \
+                                        NULL,                           \
                                         &ipsec_punt_reason[IPSEC_PUNT_##s]);
   foreach_ipsec_punt_reason
 #undef _
diff --git a/src/vnet/ipsec/ipsec_tun.c b/src/vnet/ipsec/ipsec_tun.c
index 769229721f6..07dd9ea409b 100644
--- a/src/vnet/ipsec/ipsec_tun.c
+++ b/src/vnet/ipsec/ipsec_tun.c
@@ -81,6 +81,43 @@ const static ipsec_tun_protect_itf_db_t IPSEC_TUN_PROTECT_DEFAULT_DB_ENTRY = {
                  _fmt, ##_args);                        \
 }
 
+static u32 ipsec_tun_node_regs[N_AF];
+
+void
+ipsec_tun_register_nodes (ip_address_family_t af)
+{
+  if (0 == ipsec_tun_node_regs[af]++)
+    {
+      if (AF_IP4 == af)
+	{
+	  udp_register_dst_port (vlib_get_main (),
+				 UDP_DST_PORT_ipsec,
+				 ipsec4_tun_input_node.index, 1);
+	  ip4_register_protocol (IP_PROTOCOL_IPSEC_ESP,
+				 ipsec4_tun_input_node.index);
+	}
+      else
+	ip6_register_protocol (IP_PROTOCOL_IPSEC_ESP,
+			       ipsec6_tun_input_node.index);
+    }
+}
+
+void
+ipsec_tun_unregister_nodes (ip_address_family_t af)
+{
+  ASSERT (0 != ipsec_tun_node_regs[af]);
+  if (0 == --ipsec_tun_node_regs[af])
+    {
+      if (AF_IP4 == af)
+	{
+	  udp_unregister_dst_port (vlib_get_main (), UDP_DST_PORT_ipsec, 1);
+	  ip4_unregister_protocol (IP_PROTOCOL_IPSEC_ESP);
+	}
+      else
+	ip6_unregister_protocol (IP_PROTOCOL_IPSEC_ESP);
+    }
+}
+
 static void
 ipsec_tun_protect_add_adj (adj_index_t ai, index_t sai)
 {
@@ -208,14 +245,7 @@ ipsec_tun_protect_rx_db_add (ipsec_main_t * im,
             .spi = clib_host_to_net_u32 (sa->spi),
           };
           hash_set (im->tun4_protect_by_key, key.as_u64, res.as_u64);
-          if (1 == hash_elts(im->tun4_protect_by_key))
-            {
-              udp_register_dst_port (vlib_get_main(),
-                                     UDP_DST_PORT_ipsec,
-                                     ipsec4_tun_input_node.index, 1);
-              ip4_register_protocol (IP_PROTOCOL_IPSEC_ESP,
-                                     ipsec4_tun_input_node.index);
-            }
+          ipsec_tun_register_nodes(AF_IP4);
         }
       else
         {
@@ -224,10 +254,7 @@ ipsec_tun_protect_rx_db_add (ipsec_main_t * im,
             .spi = clib_host_to_net_u32 (sa->spi),
           };
           hash_set_mem_alloc (&im->tun6_protect_by_key, &key, res.as_u64);
-
-          if (1 == hash_elts (im->tun6_protect_by_key))
-            ip6_register_protocol (IP_PROTOCOL_IPSEC_ESP,
-                                   ipsec6_tun_input_node.index);
+          ipsec_tun_register_nodes(AF_IP6);
         }
   }))
   /* *INDENT-ON* */
@@ -295,6 +322,9 @@ ipsec_tun_protect_tx_db_add (ipsec_tun_protect_t * itp)
 
       adj_nbr_walk_nh (itp->itp_sw_if_index,
 		       nh_proto, &nh, ipsec_tun_protect_adj_add, itp);
+
+      ipsec_tun_register_nodes (FIB_PROTOCOL_IP6 == nh_proto ?
+				AF_IP6 : AF_IP4);
     }
 }
 
@@ -313,11 +343,11 @@ ipsec_tun_protect_rx_db_remove (ipsec_main_t * im,
             .remote_ip = itp->itp_crypto.dst.ip4,
             .spi = clib_host_to_net_u32 (sa->spi),
           };
-          hash_unset (im->tun4_protect_by_key, key.as_u64);
-          if (0 == hash_elts(im->tun4_protect_by_key))
-            udp_unregister_dst_port (vlib_get_main(),
-                                     UDP_DST_PORT_ipsec,
-                                     1);
+          if (hash_get(im->tun4_protect_by_key, key.as_u64))
+            {
+              hash_unset (im->tun4_protect_by_key, key.as_u64);
+              ipsec_tun_unregister_nodes(AF_IP4);
+            }
         }
       else
         {
@@ -325,7 +355,11 @@ ipsec_tun_protect_rx_db_remove (ipsec_main_t * im,
             .remote_ip = itp->itp_crypto.dst.ip6,
             .spi = clib_host_to_net_u32 (sa->spi),
           };
-          hash_unset_mem_free (&im->tun6_protect_by_key, &key);
+          if (hash_get_mem(im->tun6_protect_by_key, &key))
+            {
+              hash_unset_mem_free (&im->tun6_protect_by_key, &key);
+              ipsec_tun_unregister_nodes(AF_IP6);
+            }
         }
   }))
   /* *INDENT-ON* */
@@ -372,6 +406,8 @@ ipsec_tun_protect_tx_db_remove (ipsec_tun_protect_t * itp)
 	  hash_free (idi->id_hash);
 	  idi->id_hash = NULL;
 	}
+      ipsec_tun_unregister_nodes (FIB_PROTOCOL_IP6 == nh_proto ?
+				  AF_IP6 : AF_IP4);
     }
 }
 
@@ -732,11 +768,6 @@ ipsec_tun_protect_del (u32 sw_if_index, const ip_address_t * nh)
   clib_mem_free (itp->itp_key);
   pool_put (ipsec_tun_protect_pool, itp);
 
-  if (0 == hash_elts (im->tun4_protect_by_key))
-    ip4_unregister_protocol (IP_PROTOCOL_IPSEC_ESP);
-  if (0 == hash_elts (im->tun6_protect_by_key))
-    ip6_unregister_protocol (IP_PROTOCOL_IPSEC_ESP);
-
   return (0);
 }
 
diff --git a/src/vnet/ipsec/ipsec_tun.h b/src/vnet/ipsec/ipsec_tun.h
index 56d3de14704..863afdbba5a 100644
--- a/src/vnet/ipsec/ipsec_tun.h
+++ b/src/vnet/ipsec/ipsec_tun.h
@@ -121,6 +121,9 @@ extern void ipsec_tun_protect_walk_itf (u32 sw_if_index,
 extern u8 *format_ipsec_tun_protect (u8 * s, va_list * args);
 extern u8 *format_ipsec_tun_protect_index (u8 * s, va_list * args);
 
+extern void ipsec_tun_register_nodes (ip_address_family_t af);
+extern void ipsec_tun_unregister_nodes (ip_address_family_t af);
+
 // FIXME
 extern vlib_node_registration_t ipsec4_tun_input_node;
 extern vlib_node_registration_t ipsec6_tun_input_node;
diff --git a/src/vnet/vxlan-gbp/vxlan_gbp.c b/src/vnet/vxlan-gbp/vxlan_gbp.c
index d17a2c82b02..a061cc17d8f 100644
--- a/src/vnet/vxlan-gbp/vxlan_gbp.c
+++ b/src/vnet/vxlan-gbp/vxlan_gbp.c
@@ -1165,9 +1165,11 @@ vxlan_gbp_init (vlib_main_t * vm)
 
   vlib_punt_reason_alloc (punt_hdl,
 			  "VXLAN-GBP-no-such-v4-tunnel",
+			  NULL, NULL,
 			  &vxm->punt_no_such_tunnel[FIB_PROTOCOL_IP4]);
   vlib_punt_reason_alloc (punt_hdl,
 			  "VXLAN-GBP-no-such-v6-tunnel",
+			  NULL, NULL,
 			  &vxm->punt_no_such_tunnel[FIB_PROTOCOL_IP6]);
 
   return (0);
diff --git a/test/test_punt.py b/test/test_punt.py
index 9627e542c2c..8ebf44767f3 100644
--- a/test/test_punt.py
+++ b/test/test_punt.py
@@ -794,30 +794,6 @@ class TestExceptionPuntSocket(TestPuntSocket):
         }
 
         #
-        # we need an IPSec tunnels for this to work otherwise ESP gets dropped
-        # due to unknown IP proto
-        #
-        VppIpsecTunInterface(self, self.pg0, 1000, 1000,
-                             (VppEnum.vl_api_ipsec_crypto_alg_t.
-                              IPSEC_API_CRYPTO_ALG_AES_CBC_128),
-                             b"0123456701234567",
-                             b"0123456701234567",
-                             (VppEnum.vl_api_ipsec_integ_alg_t.
-                              IPSEC_API_INTEG_ALG_SHA1_96),
-                             b"0123456701234567",
-                             b"0123456701234567").add_vpp_config()
-        VppIpsecTunInterface(self, self.pg1, 1000, 1000,
-                             (VppEnum.vl_api_ipsec_crypto_alg_t.
-                              IPSEC_API_CRYPTO_ALG_AES_CBC_128),
-                             b"0123456701234567",
-                             b"0123456701234567",
-                             (VppEnum.vl_api_ipsec_integ_alg_t.
-                              IPSEC_API_INTEG_ALG_SHA1_96),
-                             b"0123456701234567",
-                             b"0123456701234567",
-                             udp_encap=True).add_vpp_config()
-
-        #
         # we're dealing with IPSec tunnels punting for no-such-tunnel
         # adn SPI=0
         #
@@ -880,6 +856,42 @@ class TestExceptionPuntSocket(TestPuntSocket):
                                  cfg['spi'], cfg['udp'])
 
         #
+        # add some tunnels, make sure it still punts
+        #
+        VppIpsecTunInterface(self, self.pg0, 1000, 1000,
+                             (VppEnum.vl_api_ipsec_crypto_alg_t.
+                              IPSEC_API_CRYPTO_ALG_AES_CBC_128),
+                             b"0123456701234567",
+                             b"0123456701234567",
+                             (VppEnum.vl_api_ipsec_integ_alg_t.
+                              IPSEC_API_INTEG_ALG_SHA1_96),
+                             b"0123456701234567",
+                             b"0123456701234567").add_vpp_config()
+        VppIpsecTunInterface(self, self.pg1, 1000, 1000,
+                             (VppEnum.vl_api_ipsec_crypto_alg_t.
+                              IPSEC_API_CRYPTO_ALG_AES_CBC_128),
+                             b"0123456701234567",
+                             b"0123456701234567",
+                             (VppEnum.vl_api_ipsec_integ_alg_t.
+                              IPSEC_API_INTEG_ALG_SHA1_96),
+                             b"0123456701234567",
+                             b"0123456701234567",
+                             udp_encap=True).add_vpp_config()
+
+        #
+        # send packets for each SPI we expect to be punted
+        #
+        for cfg in cfgs.values():
+            self.send_and_assert_no_replies(cfg['itf'], cfg['pkts'])
+
+        #
+        # verify the punted packets arrived on the associated socket
+        #
+        for cfg in cfgs.values():
+            rx = cfg['sock'].close()
+            self.verify_esp_pkts(rx, len(cfg['pkts']),
+                                 cfg['spi'], cfg['udp'])
+        #
         # socket deregister
         #
         for cfg in cfgs.values():