Improve MTU handling

- setting MTU on an interface updates the L3 max bytes too
- value cached in the adjacency is also updated
- MTU exceeded generates ICMP to sender

Change-Id: I343ec71d8e903b529594c4bd0543f04bc7f370b3
Signed-off-by: Neale Ranns <neale.ranns@cisco.com>

9 files changed, 106 insertions, 28 deletions

diff --git a/src/vnet/adj/adj.c b/src/vnet/adj/adj.c
index 5f7fe74cd43..d28d5195330 100644
--- a/src/vnet/adj/adj.c
+++ b/src/vnet/adj/adj.c
@@ -338,6 +338,43 @@ adj_feature_update (u32 sw_if_index,
     adj_walk (sw_if_index, adj_feature_update_walk_cb, &ctx);
 }
 
+static adj_walk_rc_t
+adj_mtu_update_walk_cb (adj_index_t ai,
+                        void *arg)
+{
+    ip_adjacency_t *adj;
+
+    adj = adj_get(ai);
+
+    vnet_rewrite_update_mtu (vnet_get_main(),
+                             &adj->rewrite_header);
+
+    return (ADJ_WALK_RC_CONTINUE);
+}
+
+static void
+adj_sw_mtu_update (vnet_main_t * vnm,
+                   u32 sw_if_index,
+                   void *ctx)
+{
+    /*
+     * Walk all the adjacencies on the interface to update the cached MTU
+     */
+    adj_walk (sw_if_index, adj_mtu_update_walk_cb, NULL);
+}
+
+void
+adj_mtu_update (u32 hw_if_index)
+{
+    /*
+     * Walk all the SW interfaces on the HW interface to update the cached MTU
+     */
+    vnet_hw_interface_walk_sw(vnet_get_main(),
+                              hw_if_index,
+                              adj_sw_mtu_update,
+                              NULL);
+}
+
 /**
  * @brief Walk the Adjacencies on a given interface
  */
diff --git a/src/vnet/adj/adj.h b/src/vnet/adj/adj.h
index fe77d1634e0..bcf6c041209 100644
--- a/src/vnet/adj/adj.h
+++ b/src/vnet/adj/adj.h
@@ -345,6 +345,12 @@ extern const u8* adj_get_rewrite (adj_index_t ai);
 extern void adj_feature_update (u32 sw_if_index, u8 arc_index, u8 is_enable);
 
 /**
+ * @brief Notify the adjacency subsystem that the MTU settings for
+ * an HW interface have changed
+ */
+extern void adj_mtu_update (u32 hw_if_index);
+
+/**
  * @brief
  * The global adjacnecy pool. Exposed for fast/inline data-plane access
  */
diff --git a/src/vnet/adj/rewrite.c b/src/vnet/adj/rewrite.c
index 9150f2c6c37..c21495a9644 100644
--- a/src/vnet/adj/rewrite.c
+++ b/src/vnet/adj/rewrite.c
@@ -77,15 +77,14 @@ format_vnet_rewrite (u8 * s, va_list * args)
       vnet_sw_interface_t *si;
       si = vnet_get_sw_interface_safe (vnm, rw->sw_if_index);
       if (NULL != si)
-	s = format (s, "%U: ", format_vnet_sw_interface_name, vnm, si);
+	s = format (s, "%U:", format_vnet_sw_interface_name, vnm, si);
       else
 	s = format (s, "DELETED:%d", rw->sw_if_index);
     }
 
   /* Format rewrite string. */
   if (rw->data_bytes > 0)
-
-    s = format (s, "%U",
+    s = format (s, " %U",
 		format_hex_bytes,
 		rw->data + max_data_bytes - rw->data_bytes, rw->data_bytes);
 
@@ -111,6 +110,13 @@ vnet_rewrite_init (vnet_main_t * vnm,
 }
 
 void
+vnet_rewrite_update_mtu (vnet_main_t * vnm, vnet_rewrite_header_t * rw)
+{
+  rw->max_l3_packet_bytes =
+    vnet_sw_interface_get_mtu (vnm, rw->sw_if_index, VLIB_TX);
+}
+
+void
 vnet_rewrite_for_sw_interface (vnet_main_t * vnm,
 			       vnet_link_t link_type,
 			       u32 sw_if_index,
diff --git a/src/vnet/adj/rewrite.h b/src/vnet/adj/rewrite.h
index 1dea72f5ed8..005ac41fe72 100644
--- a/src/vnet/adj/rewrite.h
+++ b/src/vnet/adj/rewrite.h
@@ -328,6 +328,9 @@ void vnet_rewrite_init (struct vnet_main_t *vnm,
 			u32 this_node,
 			u32 next_node, vnet_rewrite_header_t * rw);
 
+void vnet_rewrite_update_mtu (struct vnet_main_t *vnm,
+			      vnet_rewrite_header_t * rw);
+
 u8 *vnet_build_rewrite_for_sw_interface (struct vnet_main_t *vnm,
 					 u32 sw_if_index,
 					 vnet_link_t packet_type,
diff --git a/src/vnet/interface.c b/src/vnet/interface.c
index a34bb7931ae..7516aec0e71 100644
--- a/src/vnet/interface.c
+++ b/src/vnet/interface.c
@@ -122,6 +122,22 @@ unserialize_vnet_sw_interface_set_flags (serialize_main_t * m, va_list * va)
      /* helper_flags no redistribution */ 0);
 }
 
+void
+vnet_hw_interface_set_mtu (vnet_main_t * vnm, u32 hw_if_index, u32 mtu)
+{
+  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
+
+  if (hi->max_packet_bytes != mtu)
+    {
+      u16 l3_pad = hi->max_packet_bytes - hi->max_l3_packet_bytes[VLIB_TX];
+      hi->max_packet_bytes = mtu;
+      hi->max_l3_packet_bytes[VLIB_TX] =
+	hi->max_l3_packet_bytes[VLIB_RX] = mtu - l3_pad;
+      ethernet_set_flags (vnm, hw_if_index, ETHERNET_INTERFACE_FLAG_MTU);
+      adj_mtu_update (hw_if_index);
+    }
+}
+
 static void
 unserialize_vnet_hw_interface_set_flags (serialize_main_t * m, va_list * va)
 {
diff --git a/src/vnet/interface_api.c b/src/vnet/interface_api.c
index 0541f311bc2..ed116bc88d9 100644
--- a/src/vnet/interface_api.c
+++ b/src/vnet/interface_api.c
@@ -98,7 +98,6 @@ vl_api_sw_interface_set_mtu_t_handler (vl_api_sw_interface_set_mtu_t * mp)
 {
   vl_api_sw_interface_set_mtu_reply_t *rmp;
   vnet_main_t *vnm = vnet_get_main ();
-  u32 flags = ETHERNET_INTERFACE_FLAG_MTU;
   u32 sw_if_index = ntohl (mp->sw_if_index);
   u16 mtu = ntohs (mp->mtu);
   ethernet_main_t *em = &ethernet_main;
@@ -134,11 +133,7 @@ vl_api_sw_interface_set_mtu_t_handler (vl_api_sw_interface_set_mtu_t * mp)
       goto bad_sw_if_index;
     }
 
-  if (hi->max_packet_bytes != mtu)
-    {
-      hi->max_packet_bytes = mtu;
-      ethernet_set_flags (vnm, si->hw_if_index, flags);
-    }
+  vnet_hw_interface_set_mtu (vnm, si->hw_if_index, mtu);
 
   BAD_SW_IF_INDEX_LABEL;
   REPLY_MACRO (VL_API_SW_INTERFACE_SET_MTU_REPLY);
diff --git a/src/vnet/interface_cli.c b/src/vnet/interface_cli.c
index 5509e3ffa57..8880d9749b8 100644
--- a/src/vnet/interface_cli.c
+++ b/src/vnet/interface_cli.c
@@ -1132,7 +1132,6 @@ mtu_cmd (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd)
 {
   vnet_main_t *vnm = vnet_get_main ();
   u32 hw_if_index, mtu;
-  u32 flags = ETHERNET_INTERFACE_FLAG_MTU;
   ethernet_main_t *em = &ethernet_main;
 
   if (unformat (input, "%d %U", &mtu,
@@ -1153,11 +1152,7 @@ mtu_cmd (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd)
 	return clib_error_return (0, "Invalid mtu (%d): must be <= (%d)", mtu,
 				  hi->max_supported_packet_bytes);
 
-      if (hi->max_packet_bytes != mtu)
-	{
-	  hi->max_packet_bytes = mtu;
-	  ethernet_set_flags (vnm, hw_if_index, flags);
-	}
+      vnet_hw_interface_set_mtu (vnm, hw_if_index, mtu);
     }
   else
     return clib_error_return (0, "unknown input `%U'",
diff --git a/src/vnet/interface_funcs.h b/src/vnet/interface_funcs.h
index ac50e7cd6ca..5aecaff9b27 100644
--- a/src/vnet/interface_funcs.h
+++ b/src/vnet/interface_funcs.h
@@ -292,6 +292,9 @@ clib_error_t *set_hw_interface_change_rx_mode (vnet_main_t * vnm,
 					       vnet_hw_interface_rx_mode
 					       mode);
 
+/* Set the MTU on the HW interface */
+void vnet_hw_interface_set_mtu (vnet_main_t * vnm, u32 hw_if_index, u32 mtu);
+
 /* Formats sw/hw interface. */
 format_function_t format_vnet_hw_interface;
 format_function_t format_vnet_hw_interface_rx_mode;
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index ae4510626d0..b9875d72b8e 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -2109,16 +2109,26 @@ ip4_rewrite_inline (vlib_main_t * vm,
 	  vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
 
 	  /* Check MTU of outgoing interface. */
-	  error0 =
-	    (vlib_buffer_length_in_chain (vm, p0) >
-	     adj0[0].
-	     rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
-	     error0);
-	  error1 =
-	    (vlib_buffer_length_in_chain (vm, p1) >
-	     adj1[0].
-	     rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
-	     error1);
+	  if (vlib_buffer_length_in_chain (vm, p0) >
+	      adj0[0].rewrite_header.max_l3_packet_bytes)
+	    {
+	      error0 = IP4_ERROR_MTU_EXCEEDED;
+	      next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
+	      icmp4_error_set_vnet_buffer
+		(p0, ICMP4_destination_unreachable,
+		 ICMP4_destination_unreachable_fragmentation_needed_and_dont_fragment_set,
+		 0);
+	    }
+	  if (vlib_buffer_length_in_chain (vm, p1) >
+	      adj1[0].rewrite_header.max_l3_packet_bytes)
+	    {
+	      error1 = IP4_ERROR_MTU_EXCEEDED;
+	      next1 = IP4_REWRITE_NEXT_ICMP_ERROR;
+	      icmp4_error_set_vnet_buffer
+		(p1, ICMP4_destination_unreachable,
+		 ICMP4_destination_unreachable_fragmentation_needed_and_dont_fragment_set,
+		 0);
+	    }
 
 	  if (is_mcast)
 	    {
@@ -2290,9 +2300,16 @@ ip4_rewrite_inline (vlib_main_t * vm,
 	       vlib_buffer_length_in_chain (vm, p0) + rw_len0);
 
 	  /* Check MTU of outgoing interface. */
-	  error0 = (vlib_buffer_length_in_chain (vm, p0)
-		    > adj0[0].rewrite_header.max_l3_packet_bytes
-		    ? IP4_ERROR_MTU_EXCEEDED : error0);
+	  if (vlib_buffer_length_in_chain (vm, p0) >
+	      adj0[0].rewrite_header.max_l3_packet_bytes)
+	    {
+	      error0 = IP4_ERROR_MTU_EXCEEDED;
+	      next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
+	      icmp4_error_set_vnet_buffer
+		(p0, ICMP4_destination_unreachable,
+		 ICMP4_destination_unreachable_fragmentation_needed_and_dont_fragment_set,
+		 0);
+	    }
 	  if (is_mcast)
 	    {
 	      error0 = ((adj0[0].rewrite_header.sw_if_index ==