linux-cp: handle ipv4 routes when link goes down

Type: improvement

Currently, when the link goes down on an interface, routes that resolve
through that interface and created with Nexthop API are removed by the
kernel. However, IPv4 routes remain in the FIB because the kernel
doesn't send any notifications about that.

And for the plugin working with user-space applications that create
routes in the kernel using Nexthop API there should be a mechanism to
synchronize the FIB and the kernel in this case.

With this change, add two new startup configuration options to the
plugin to be able to control what should happen with static and dynamic
routes managed by the plugin on link down:
  - del-static-on-link-down (disabled by default, delete routes created
    with the linux-cp static FIB source on link down),
  - del-dynamic-on-link-down (disabled by default, delete routes created
    with the linux-cp dynamic FIB source on link down).

Then, monitor link state changes on interfaces for which a linux-cp pair
exists. If the link goes down on one of the interfaces, process routes
that resolve through that interface according to the new configurations.

Signed-off-by: Alexander Chernavin <achernavin@netgate.com>
Change-Id: I0fbaeeca3f3d1fcd22e8eebb08a0a4a3d0dfe5b8

5 files changed, 163 insertions, 0 deletions

diff --git a/src/plugins/linux-cp/lcp.c b/src/plugins/linux-cp/lcp.c
index 69a7a6e42ba..33c71e44143 100644
--- a/src/plugins/linux-cp/lcp.c
+++ b/src/plugins/linux-cp/lcp.c
@@ -113,6 +113,38 @@ lcp_auto_subint (void)
   return lcpm->lcp_auto_subint;
 }
 
+void
+lcp_set_del_static_on_link_down (u8 is_del)
+{
+  lcp_main_t *lcpm = &lcp_main;
+
+  lcpm->del_static_on_link_down = (is_del != 0);
+}
+
+u8
+lcp_get_del_static_on_link_down (void)
+{
+  lcp_main_t *lcpm = &lcp_main;
+
+  return lcpm->del_static_on_link_down;
+}
+
+void
+lcp_set_del_dynamic_on_link_down (u8 is_del)
+{
+  lcp_main_t *lcpm = &lcp_main;
+
+  lcpm->del_dynamic_on_link_down = (is_del != 0);
+}
+
+u8
+lcp_get_del_dynamic_on_link_down (void)
+{
+  lcp_main_t *lcpm = &lcp_main;
+
+  return lcpm->del_dynamic_on_link_down;
+}
+
 /*
  * fd.io coding-style-patch-verification: ON
  *
diff --git a/src/plugins/linux-cp/lcp.h b/src/plugins/linux-cp/lcp.h
index 14c1a6e2a75..4ddaa3898d6 100644
--- a/src/plugins/linux-cp/lcp.h
+++ b/src/plugins/linux-cp/lcp.h
@@ -26,6 +26,8 @@ typedef struct lcp_main_s
   int default_ns_fd;
   u8 lcp_auto_subint; /* Automatically create/delete LCP sub-interfaces */
   u8 lcp_sync;	      /* Automatically sync VPP changes to LCP */
+  u8 del_static_on_link_down;  /* Delete static routes when link goes down */
+  u8 del_dynamic_on_link_down; /* Delete dynamic routes when link goes down */
   u8 test_mode;	      /* Set when Unit testing */
 } lcp_main_t;
 
@@ -38,6 +40,18 @@ int lcp_set_default_ns (u8 *ns);
 u8 *lcp_get_default_ns (void); /* Returns NULL or shared string */
 int lcp_get_default_ns_fd (void);
 
+/**
+ * Get/Set whether to delete static routes when the link goes down.
+ */
+void lcp_set_del_static_on_link_down (u8 is_del);
+u8 lcp_get_del_static_on_link_down (void);
+
+/**
+ * Get/Set whether to delete dynamic routes when the link goes down.
+ */
+void lcp_set_del_dynamic_on_link_down (u8 is_del);
+u8 lcp_get_del_dynamic_on_link_down (void);
+
 #endif
 
 /*
diff --git a/src/plugins/linux-cp/lcp_cli.c b/src/plugins/linux-cp/lcp_cli.c
index 8f2d17ab209..ff84e74809a 100644
--- a/src/plugins/linux-cp/lcp_cli.c
+++ b/src/plugins/linux-cp/lcp_cli.c
@@ -174,6 +174,55 @@ VLIB_CLI_COMMAND (lcp_auto_subint_command, static) = {
 };
 
 static clib_error_t *
+lcp_param_command_fn (vlib_main_t *vm, unformat_input_t *input,
+		      vlib_cli_command_t *cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "del-static-on-link-down"))
+	{
+	  if (unformat (line_input, "on") || unformat (line_input, "enable"))
+	    lcp_set_del_static_on_link_down (1 /* is_del */);
+	  else if (unformat (line_input, "off") ||
+		   unformat (line_input, "disable"))
+	    lcp_set_del_static_on_link_down (0 /* is_del */);
+	  else
+	    return clib_error_return (0, "unknown input `%U'",
+				      format_unformat_error, line_input);
+	}
+      else if (unformat (line_input, "del-dynamic-on-link-down"))
+	{
+	  if (unformat (line_input, "on") || unformat (line_input, "enable"))
+	    lcp_set_del_dynamic_on_link_down (1 /* is_del */);
+	  else if (unformat (line_input, "off") ||
+		   unformat (line_input, "disable"))
+	    lcp_set_del_dynamic_on_link_down (0 /* is_del */);
+	  else
+	    return clib_error_return (0, "unknown input `%U'",
+				      format_unformat_error, line_input);
+	}
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, line_input);
+    }
+
+  unformat_free (line_input);
+  return 0;
+}
+
+VLIB_CLI_COMMAND (lcp_param_command, static) = {
+  .path = "lcp param",
+  .short_help = "lcp param [del-static-on-link-down (on|enable|off|disable)] "
+		"[del-dynamic-on-link-down (on|enable|off|disable)]",
+  .function = lcp_param_command_fn,
+};
+
+static clib_error_t *
 lcp_default_netns_command_fn (vlib_main_t *vm, unformat_input_t *input,
 			      vlib_cli_command_t *cmd)
 {
diff --git a/src/plugins/linux-cp/lcp_interface.c b/src/plugins/linux-cp/lcp_interface.c
index a822a5498d8..142ac505fcc 100644
--- a/src/plugins/linux-cp/lcp_interface.c
+++ b/src/plugins/linux-cp/lcp_interface.c
@@ -134,6 +134,10 @@ lcp_itf_pair_show (u32 phy_sw_if_index)
   vlib_cli_output (vm, "lcp lcp-auto-subint %s\n",
 		   lcp_auto_subint () ? "on" : "off");
   vlib_cli_output (vm, "lcp lcp-sync %s\n", lcp_sync () ? "on" : "off");
+  vlib_cli_output (vm, "lcp del-static-on-link-down %s\n",
+		   lcp_get_del_static_on_link_down () ? "on" : "off");
+  vlib_cli_output (vm, "lcp del-dynamic-on-link-down %s\n",
+		   lcp_get_del_dynamic_on_link_down () ? "on" : "off");
 
   if (phy_sw_if_index == ~0)
     {
@@ -571,6 +575,10 @@ lcp_itf_pair_config (vlib_main_t *vm, unformat_input_t *input)
 	lcp_set_auto_subint (1 /* is_auto */);
       else if (unformat (input, "lcp-sync"))
 	lcp_set_sync (1 /* is_auto */);
+      else if (unformat (input, "del-static-on-link-down"))
+	lcp_set_del_static_on_link_down (1 /* is_del */);
+      else if (unformat (input, "del-dynamic-on-link-down"))
+	lcp_set_del_dynamic_on_link_down (1 /* is_del */);
       else
 	return clib_error_return (0, "interfaces not found");
     }
diff --git a/src/plugins/linux-cp/lcp_router.c b/src/plugins/linux-cp/lcp_router.c
index 34d0500b0c5..4fc824ca5ad 100644
--- a/src/plugins/linux-cp/lcp_router.c
+++ b/src/plugins/linux-cp/lcp_router.c
@@ -468,6 +468,66 @@ lcp_router_link_sync_end (void)
   LCP_ROUTER_INFO ("End synchronization of interface configurations");
 }
 
+static clib_error_t *
+lcp_router_link_up_down (vnet_main_t *vnm, u32 hw_if_index, u32 flags)
+{
+  vnet_hw_interface_t *hi;
+  index_t lipi;
+
+  hi = vnet_get_hw_interface_or_null (vnm, hw_if_index);
+  if (!hi)
+    return 0;
+
+  lipi = lcp_itf_pair_find_by_phy (hi->sw_if_index);
+  if (lipi == INDEX_INVALID)
+    return 0;
+
+  /* When the link goes down on an interface, the kernel processes routes which
+   * resolve through that interface depending on how they were created:
+   *   - Legacy Route API: the kernel retains the routes and marks them as
+   *     "linkdown";
+   *   - Nexthop API: the kernel removes the next-hop objects and the routes
+   *     which reference them.
+   *
+   * For IPv4 routes created with Nexthop API, the kernel will not send any
+   * explicit RTM_DELROUTE messages about removing them. In order to
+   * synchronize with the kernel, affected routes need to be manually removed
+   * from the FIB.
+   *
+   * The behavior is different for IPv6 routes created with Nexthop API. The
+   * kernel will send explicit RTM_DELROUTE messages about IPv6 routes being
+   * removed.
+   */
+  if (!(flags & VNET_HW_INTERFACE_FLAG_LINK_UP) &&
+      (lcp_get_del_static_on_link_down () ||
+       lcp_get_del_dynamic_on_link_down ()))
+    {
+      u32 fib_index;
+      lcp_router_table_t *nlt;
+
+      fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
+						       hi->sw_if_index);
+
+      pool_foreach (nlt, lcp_router_table_pool)
+	{
+	  if (fib_index == nlt->nlt_fib_index &&
+	      FIB_PROTOCOL_IP4 == nlt->nlt_proto)
+	    {
+	      if (lcp_get_del_static_on_link_down ())
+		lcp_router_table_flush (nlt, hi->sw_if_index, lcp_rt_fib_src);
+	      if (lcp_get_del_dynamic_on_link_down ())
+		lcp_router_table_flush (nlt, hi->sw_if_index,
+					lcp_rt_fib_src_dynamic);
+	      break;
+	    }
+	}
+    }
+
+  return 0;
+}
+
+VNET_HW_INTERFACE_LINK_UP_DOWN_FUNCTION (lcp_router_link_up_down);
+
 static fib_protocol_t
 lcp_router_proto_k2f (uint32_t k)
 {