SNAT: in2out translation as an output feature (VPP-903)

in2out translation as an output feature on the outside interface (postrouting)

Change-Id: I32c0311be09bdf102b9a0885b8b89c7588cb558f
Signed-off-by: Matus Fabian <matfabia@cisco.com>

5 files changed, 495 insertions, 38 deletions

diff --git a/src/plugins/snat/in2out.c b/src/plugins/snat/in2out.c
index 9b09902a..9ee2ebd4 100644
--- a/src/plugins/snat/in2out.c
+++ b/src/plugins/snat/in2out.c
@@ -88,6 +88,9 @@ vlib_node_registration_t snat_in2out_slowpath_node;
 vlib_node_registration_t snat_in2out_fast_node;
 vlib_node_registration_t snat_in2out_worker_handoff_node;
 vlib_node_registration_t snat_det_in2out_node;
+vlib_node_registration_t snat_in2out_output_node;
+vlib_node_registration_t snat_in2out_output_slowpath_node;
+vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 
 #define foreach_snat_in2out_error                       \
 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
@@ -515,8 +518,13 @@ u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
   clib_bihash_kv_8_8_t kv0, value0;
   u32 next0 = ~0;
   int err;
+  u32 iph_offset0 = 0;
 
-  ip0 = vlib_buffer_get_current (b0);
+  if (PREDICT_FALSE(vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0))
+    {
+      iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
+    }
+  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + iph_offset0);
   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
@@ -535,7 +543,8 @@ u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
   if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
     {
       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
-          IP_PROTOCOL_ICMP, rx_fib_index0)))
+          IP_PROTOCOL_ICMP, rx_fib_index0) &&
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0))
         {
           dont_translate = 1;
           goto out;
@@ -704,7 +713,8 @@ static inline u32 icmp_in2out (snat_main_t *sm,
 
   old_addr0 = ip0->src_address.as_u32;
   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
-  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
+  if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
+    vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 
   sum0 = ip0->checksum;
   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
@@ -1276,7 +1286,8 @@ create_ses:
       kv.key = m_key.as_u64;
       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
         {
-          vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
+          if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
+            vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
           return;
         }
 
@@ -1298,7 +1309,8 @@ create_ses:
 static inline uword
 snat_in2out_node_fn_inline (vlib_main_t * vm,
                             vlib_node_runtime_t * node,
-                            vlib_frame_t * frame, int is_slow_path)
+                            vlib_frame_t * frame, int is_slow_path,
+                            int is_output_feature)
 {
   u32 n_left_from, * from, * to_next;
   snat_in2out_next_t next_index;
@@ -1340,6 +1352,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
           u32 proto0, proto1;
           snat_session_t * s0 = 0, * s1 = 0;
           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
+          u32 iph_offset0 = 0, iph_offset1 = 0;
           
 	  /* Prefetch next iteration. */
 	  {
@@ -1366,7 +1379,12 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
 	  b0 = vlib_get_buffer (vm, bi0);
 	  b1 = vlib_get_buffer (vm, bi1);
 
-          ip0 = vlib_buffer_get_current (b0);
+          if (is_output_feature)
+            iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
+
+          ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
+                 iph_offset0);
+
           udp0 = ip4_next_header (ip0);
           tcp0 = (tcp_header_t *) udp0;
           icmp0 = (icmp46_header_t *) udp0;
@@ -1427,8 +1445,8 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
             {
               if (is_slow_path)
                 {
-                  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
-                      proto0, rx_fib_index0)))
+                  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
+                      ip0, proto0, rx_fib_index0)) && !is_output_feature)
                     goto trace00;
 
                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
@@ -1449,7 +1467,8 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
           old_addr0 = ip0->src_address.as_u32;
           ip0->src_address = s0->out2in.addr;
           new_addr0 = ip0->src_address.as_u32;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
+          if (!is_output_feature)
+            vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 
           sum0 = ip0->checksum;
           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
@@ -1512,7 +1531,12 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
 
           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
 
-          ip1 = vlib_buffer_get_current (b1);
+          if (is_output_feature)
+            iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
+
+          ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
+                 iph_offset1);
+
           udp1 = ip4_next_header (ip1);
           tcp1 = (tcp_header_t *) udp1;
           icmp1 = (icmp46_header_t *) udp1;
@@ -1571,8 +1595,8 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
             {
               if (is_slow_path)
                 {
-                  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1, ip1,
-                      proto1, rx_fib_index1)))
+                  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
+                      ip1, proto1, rx_fib_index1)) && !is_output_feature)
                     goto trace01;
 
                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
@@ -1593,7 +1617,8 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
           old_addr1 = ip1->src_address.as_u32;
           ip1->src_address = s1->out2in.addr;
           new_addr1 = ip1->src_address.as_u32;
-          vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
+          if (!is_output_feature)
+            vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
 
           sum1 = ip1->checksum;
           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
@@ -1679,7 +1704,8 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
           u32 proto0;
           snat_session_t * s0 = 0;
           clib_bihash_kv_8_8_t kv0, value0;
-          
+          u32 iph_offset0 = 0;
+
           /* speculatively enqueue b0 to the current next frame */
 	  bi0 = from[0];
 	  to_next[0] = bi0;
@@ -1691,7 +1717,12 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
 	  b0 = vlib_get_buffer (vm, bi0);
           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
 
-          ip0 = vlib_buffer_get_current (b0);
+          if (is_output_feature)
+            iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
+
+          ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
+                 iph_offset0);
+
           udp0 = ip4_next_header (ip0);
           tcp0 = (tcp_header_t *) udp0;
           icmp0 = (icmp46_header_t *) udp0;
@@ -1750,8 +1781,8 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
             {
               if (is_slow_path)
                 {
-                  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
-                      proto0, rx_fib_index0)))
+                  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
+                      ip0, proto0, rx_fib_index0)) && !is_output_feature)
                     goto trace0;
 
                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
@@ -1773,7 +1804,8 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
           old_addr0 = ip0->src_address.as_u32;
           ip0->src_address = s0->out2in.addr;
           new_addr0 = ip0->src_address.as_u32;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
+          if (!is_output_feature)
+            vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 
           sum0 = ip0->checksum;
           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
@@ -1856,7 +1888,7 @@ snat_in2out_fast_path_fn (vlib_main_t * vm,
                           vlib_node_runtime_t * node,
                           vlib_frame_t * frame)
 {
-  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */);
+  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
 }
 
 VLIB_REGISTER_NODE (snat_in2out_node) = {
@@ -1865,12 +1897,12 @@ VLIB_REGISTER_NODE (snat_in2out_node) = {
   .vector_size = sizeof (u32),
   .format_trace = format_snat_in2out_trace,
   .type = VLIB_NODE_TYPE_INTERNAL,
-  
+
   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
   .error_strings = snat_in2out_error_strings,
 
   .runtime_data_bytes = sizeof (snat_runtime_t),
-  
+
   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
 
   /* edit / add dispositions here */
@@ -1885,11 +1917,45 @@ VLIB_REGISTER_NODE (snat_in2out_node) = {
 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
 
 static uword
+snat_in2out_output_fast_path_fn (vlib_main_t * vm,
+                                 vlib_node_runtime_t * node,
+                                 vlib_frame_t * frame)
+{
+  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
+}
+
+VLIB_REGISTER_NODE (snat_in2out_output_node) = {
+  .function = snat_in2out_output_fast_path_fn,
+  .name = "snat-in2out-output",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_in2out_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
+  .error_strings = snat_in2out_error_strings,
+
+  .runtime_data_bytes = sizeof (snat_runtime_t),
+
+  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
+    [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
+    [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-output-slowpath",
+    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
+                              snat_in2out_output_fast_path_fn);
+
+static uword
 snat_in2out_slow_path_fn (vlib_main_t * vm,
                           vlib_node_runtime_t * node,
                           vlib_frame_t * frame)
 {
-  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */);
+  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
 }
 
 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
@@ -1898,12 +1964,12 @@ VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
   .vector_size = sizeof (u32),
   .format_trace = format_snat_in2out_trace,
   .type = VLIB_NODE_TYPE_INTERNAL,
-  
+
   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
   .error_strings = snat_in2out_error_strings,
 
   .runtime_data_bytes = sizeof (snat_runtime_t),
-  
+
   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
 
   /* edit / add dispositions here */
@@ -1915,7 +1981,42 @@ VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
   },
 };
 
-VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node, snat_in2out_slow_path_fn);
+VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
+                              snat_in2out_slow_path_fn);
+
+static uword
+snat_in2out_output_slow_path_fn (vlib_main_t * vm,
+                                 vlib_node_runtime_t * node,
+                                 vlib_frame_t * frame)
+{
+  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
+}
+
+VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
+  .function = snat_in2out_output_slow_path_fn,
+  .name = "snat-in2out-output-slowpath",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_in2out_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
+  .error_strings = snat_in2out_error_strings,
+
+  .runtime_data_bytes = sizeof (snat_runtime_t),
+
+  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
+    [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
+    [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-output-slowpath",
+    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
+                              snat_in2out_output_slow_path_fn);
 
 /**************************/
 /*** deterministic mode ***/
@@ -2677,10 +2778,11 @@ out:
 /**********************/
 /*** worker handoff ***/
 /**********************/
-static uword
-snat_in2out_worker_handoff_fn (vlib_main_t * vm,
-                               vlib_node_runtime_t * node,
-                               vlib_frame_t * frame)
+static inline uword
+snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
+                                      vlib_node_runtime_t * node,
+                                      vlib_frame_t * frame,
+                                      u8 is_output)
 {
   snat_main_t *sm = &snat_main;
   vlib_thread_main_t *tm = vlib_get_thread_main ();
@@ -2695,9 +2797,22 @@ snat_in2out_worker_handoff_fn (vlib_main_t * vm,
   u32 next_worker_index = 0;
   u32 current_worker_index = ~0;
   u32 thread_index = vlib_get_thread_index ();
+  u32 fq_index;
+  u32 to_node_index;
 
   ASSERT (vec_len (sm->workers));
 
+  if (is_output)
+    {
+      fq_index = sm->fq_in2out_output_index;
+      to_node_index = sm->in2out_output_node_index;
+    }
+  else
+    {
+      fq_index = sm->fq_in2out_index;
+      to_node_index = sm->in2out_node_index;
+    }
+
   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
     {
       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
@@ -2741,7 +2856,7 @@ snat_in2out_worker_handoff_fn (vlib_main_t * vm,
               if (hf)
                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
 
-              hf = vlib_get_worker_handoff_queue_elt (sm->fq_in2out_index,
+              hf = vlib_get_worker_handoff_queue_elt (fq_index,
                                                       next_worker_index,
                                                       handoff_queue_elt_by_worker_index);
 
@@ -2770,7 +2885,7 @@ snat_in2out_worker_handoff_fn (vlib_main_t * vm,
           /* if this is 1st frame */
           if (!f)
             {
-              f = vlib_get_frame_to_node (vm, sm->in2out_node_index);
+              f = vlib_get_frame_to_node (vm, to_node_index);
               to_next = vlib_frame_vector_args (f);
             }
 
@@ -2790,7 +2905,7 @@ snat_in2out_worker_handoff_fn (vlib_main_t * vm,
     }
 
   if (f)
-    vlib_put_frame_to_node (vm, sm->in2out_node_index, f);
+    vlib_put_frame_to_node (vm, to_node_index, f);
 
   if (hf)
     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
@@ -2821,13 +2936,46 @@ snat_in2out_worker_handoff_fn (vlib_main_t * vm,
   return frame->n_vectors;
 }
 
+static uword
+snat_in2out_worker_handoff_fn (vlib_main_t * vm,
+                               vlib_node_runtime_t * node,
+                               vlib_frame_t * frame)
+{
+  return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
+}
+
 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
   .function = snat_in2out_worker_handoff_fn,
   .name = "snat-in2out-worker-handoff",
   .vector_size = sizeof (u32),
   .format_trace = format_snat_in2out_worker_handoff_trace,
   .type = VLIB_NODE_TYPE_INTERNAL,
-  
+
+  .n_next_nodes = 1,
+
+  .next_nodes = {
+    [0] = "error-drop",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
+                              snat_in2out_worker_handoff_fn);
+
+static uword
+snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
+                                      vlib_node_runtime_t * node,
+                                      vlib_frame_t * frame)
+{
+  return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
+}
+
+VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
+  .function = snat_in2out_output_worker_handoff_fn,
+  .name = "snat-in2out-output-worker-handoff",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_in2out_worker_handoff_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
   .n_next_nodes = 1,
 
   .next_nodes = {
@@ -2835,7 +2983,8 @@ VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
   },
 };
 
-VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node, snat_in2out_worker_handoff_fn);
+VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
+                              snat_in2out_output_worker_handoff_fn);
 
 static uword
 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
diff --git a/src/plugins/snat/snat.api b/src/plugins/snat/snat.api
index 62c96058..f68a5aaa 100644
--- a/src/plugins/snat/snat.api
+++ b/src/plugins/snat/snat.api
@@ -96,6 +96,42 @@ define snat_interface_details {
   u32 sw_if_index;
 };
 
+/** \brief Enable/disbale S-NAT as an interface output feature (postrouting
+           in2out translation)
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - 1 if add, 0 if delete
+    @param is_inside - 1 if inside, 0 if outside
+    @param sw_if_index - software index of the interface
+*/
+autoreply define snat_interface_add_del_output_feature {
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
+/** \brief Dump interfaces with S-NAT output feature
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define snat_interface_output_feature_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief S-NAT interface with output feature details response
+    @param context - sender context, to match reply w/ request
+    @param is_inside - 1 if inside, 0 if outside
+    @param sw_if_index - software index of the interface
+*/
+define snat_interface_output_feature_details {
+  u32 context;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
 /** \brief Add/delete S-NAT static mapping
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
diff --git a/src/plugins/snat/snat.c b/src/plugins/snat/snat.c
index 97eb061c..bce7d210 100644
--- a/src/plugins/snat/snat.c
+++ b/src/plugins/snat/snat.c
@@ -73,6 +73,20 @@ VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
   .runs_before = VNET_FEATURES ("ip4-lookup"),
 };
 
+/* Hook up output features */
+VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
+  .arc_name = "ip4-output",
+  .node_name = "snat-in2out-output",
+  .runs_before = VNET_FEATURES ("interface-output"),
+};
+
+VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
+  .arc_name = "ip4-output",
+  .node_name = "snat-in2out-output-worker-handoff",
+  .runs_before = VNET_FEATURES ("interface-output"),
+};
+
+
 /* *INDENT-OFF* */
 VLIB_PLUGIN_REGISTER () = {
     .version = VPP_BUILD_VER,
@@ -157,6 +171,14 @@ void snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id)
     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
     break;
   }));
+  pool_foreach (i, sm->output_feature_interfaces,
+  ({
+    if (i->is_inside)
+      continue;
+
+    snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
+    break;
+  }));
 }
 
 static int is_snat_address_used_in_static_mapping (snat_main_t *sm,
@@ -542,6 +564,14 @@ delete:
     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
     break;
   }));
+  pool_foreach (interface, sm->output_feature_interfaces,
+  ({
+    if (interface->is_inside)
+      continue;
+
+    snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
+    break;
+  }));
 
   return 0;
 }
@@ -668,6 +698,14 @@ int snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm)
     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
     break;
   }));
+  pool_foreach (interface, sm->output_feature_interfaces,
+  ({
+    if (interface->is_inside)
+      continue;
+
+    snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
+    break;
+  }));
 
   return 0;
 }
@@ -746,6 +784,85 @@ fib:
   return 0;
 }
 
+int snat_interface_add_del_output_feature (u32 sw_if_index,
+                                           u8 is_inside,
+                                           int is_del)
+{
+  snat_main_t *sm = &snat_main;
+  snat_interface_t *i;
+  snat_address_t * ap;
+  snat_static_mapping_t * m;
+
+  if (sm->deterministic ||
+      (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
+    return VNET_API_ERROR_UNSUPPORTED;
+
+  if (is_inside)
+    goto find;
+
+  if (sm->num_workers > 1)
+    {
+      vnet_feature_enable_disable ("ip4-unicast", "snat-out2in-worker-handoff",
+                                   sw_if_index, !is_del, 0, 0);
+      vnet_feature_enable_disable ("ip4-output",
+                                   "snat-in2out-output-worker-handoff",
+                                   sw_if_index, !is_del, 0, 0);
+    }
+  else
+    {
+      vnet_feature_enable_disable ("ip4-unicast", "snat-out2in", sw_if_index,
+                                   !is_del, 0, 0);
+      vnet_feature_enable_disable ("ip4-output", "snat-in2out-output",
+                                   sw_if_index, !is_del, 0, 0);
+    }
+
+  if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
+    sm->fq_in2out_output_index =
+      vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
+
+  if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
+    sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
+
+find:
+  pool_foreach (i, sm->output_feature_interfaces,
+  ({
+    if (i->sw_if_index == sw_if_index)
+      {
+        if (is_del)
+          pool_put (sm->output_feature_interfaces, i);
+        else
+          return VNET_API_ERROR_VALUE_EXIST;
+
+        goto fib;
+      }
+  }));
+
+  if (is_del)
+    return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+  pool_get (sm->output_feature_interfaces, i);
+  i->sw_if_index = sw_if_index;
+  i->is_inside = is_inside;
+
+  /* Add/delete external addresses to FIB */
+fib:
+  if (is_inside)
+    return 0;
+
+  vec_foreach (ap, sm->addresses)
+    snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
+
+  pool_foreach (m, sm->static_mappings,
+  ({
+    if (!(m->addr_only))
+      continue;
+
+    snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
+  }));
+
+  return 0;
+}
+
 int snat_set_workers (uword * bitmap)
 {
   snat_main_t *sm = &snat_main;
@@ -1103,6 +1220,7 @@ snat_feature_command_fn (vlib_main_t * vm,
   u32 sw_if_index;
   u32 * inside_sw_if_indices = 0;
   u32 * outside_sw_if_indices = 0;
+  u8 is_output_feature = 0;
   int is_del = 0;
   int i;
 
@@ -1120,6 +1238,8 @@ snat_feature_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "out %U", unformat_vnet_sw_interface,
                          vnm, &sw_if_index))
         vec_add1 (outside_sw_if_indices, sw_if_index);
+      else if (unformat (line_input, "output-feature"))
+        is_output_feature = 1;
       else if (unformat (line_input, "del"))
         is_del = 1;
       else
@@ -1135,7 +1255,30 @@ snat_feature_command_fn (vlib_main_t * vm,
       for (i = 0; i < vec_len(inside_sw_if_indices); i++)
         {
           sw_if_index = inside_sw_if_indices[i];
-          snat_interface_add_del (sw_if_index, 1, is_del);
+          if (is_output_feature)
+            {
+              if (snat_interface_add_del_output_feature (sw_if_index, 1, is_del))
+                {
+                  error = clib_error_return (0, "%s %U failed",
+                                             is_del ? "del" : "add",
+                                             format_vnet_sw_interface_name, vnm,
+                                             vnet_get_sw_interface (vnm,
+                                                                    sw_if_index));
+                  goto done;
+                }
+            }
+          else
+            {
+              if (snat_interface_add_del (sw_if_index, 1, is_del))
+                {
+                  error = clib_error_return (0, "%s %U failed",
+                                             is_del ? "del" : "add",
+                                             format_vnet_sw_interface_name, vnm,
+                                             vnet_get_sw_interface (vnm,
+                                                                    sw_if_index));
+                  goto done;
+                }
+            }
         }
     }
 
@@ -1144,7 +1287,30 @@ snat_feature_command_fn (vlib_main_t * vm,
       for (i = 0; i < vec_len(outside_sw_if_indices); i++)
         {
           sw_if_index = outside_sw_if_indices[i];
-          snat_interface_add_del (sw_if_index, 0, is_del);
+          if (is_output_feature)
+            {
+              if (snat_interface_add_del_output_feature (sw_if_index, 0, is_del))
+                {
+                  error = clib_error_return (0, "%s %U failed",
+                                             is_del ? "del" : "add",
+                                             format_vnet_sw_interface_name, vnm,
+                                             vnet_get_sw_interface (vnm,
+                                                                    sw_if_index));
+                  goto done;
+                }
+            }
+          else
+            {
+              if (snat_interface_add_del (sw_if_index, 0, is_del))
+                {
+                  error = clib_error_return (0, "%s %U failed",
+                                             is_del ? "del" : "add",
+                                             format_vnet_sw_interface_name, vnm,
+                                             vnet_get_sw_interface (vnm,
+                                                                    sw_if_index));
+                  goto done;
+                }
+            }
         }
     }
 
@@ -1159,7 +1325,8 @@ done:
 VLIB_CLI_COMMAND (set_interface_snat_command, static) = {
   .path = "set interface snat",
   .function = snat_feature_command_fn,
-  .short_help = "set interface snat in <intfc> out <intfc> [del]",
+  .short_help = "set interface snat in <intfc> out <intfc> [output-feature] "
+                "[del]",
 };
 
 uword
@@ -1597,6 +1764,7 @@ snat_config (vlib_main_t * vm, unformat_input_t * input)
   if (sm->deterministic)
     {
       sm->in2out_node_index = snat_det_in2out_node.index;
+      sm->in2out_output_node_index = ~0;
       sm->out2in_node_index = snat_det_out2in_node.index;
       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
@@ -1606,6 +1774,7 @@ snat_config (vlib_main_t * vm, unformat_input_t * input)
       sm->worker_in2out_cb = snat_get_worker_in2out_cb;
       sm->worker_out2in_cb = snat_get_worker_out2in_cb;
       sm->in2out_node_index = snat_in2out_node.index;
+      sm->in2out_output_node_index = snat_in2out_output_node.index;
       sm->out2in_node_index = snat_out2in_node.index;
       if (!static_mapping_only ||
           (static_mapping_only && static_mapping_connection_tracking))
@@ -1881,6 +2050,14 @@ show_snat_command_fn (vlib_main_t * vm,
                          i->is_inside ? "in" : "out");
       }));
 
+      pool_foreach (i, sm->output_feature_interfaces,
+      ({
+        vlib_cli_output (vm, "%U output-feature %s",
+                         format_vnet_sw_interface_name, vnm,
+                         vnet_get_sw_interface (vnm, i->sw_if_index),
+                         i->is_inside ? "in" : "out");
+      }));
+
       if (vec_len (sm->auto_add_sw_if_indices))
         {
           vlib_cli_output (vm, "SNAT pool addresses interfaces:");
diff --git a/src/plugins/snat/snat.h b/src/plugins/snat/snat.h
index 1bc5fcd7..2546815b 100644
--- a/src/plugins/snat/snat.h
+++ b/src/plugins/snat/snat.h
@@ -297,6 +297,7 @@ typedef struct snat_main_s {
 
   /* Interface pool */
   snat_interface_t * interfaces;
+  snat_interface_t * output_feature_interfaces;
 
   /* Vector of outside addresses */
   snat_address_t * addresses;
@@ -312,10 +313,12 @@ typedef struct snat_main_s {
 
   /* Worker handoff index */
   u32 fq_in2out_index;
+  u32 fq_in2out_output_index;
   u32 fq_out2in_index;
 
   /* in2out and out2in node index */
   u32 in2out_node_index;
+  u32 in2out_output_node_index;
   u32 out2in_node_index;
 
   /* Deterministic NAT */
@@ -357,10 +360,12 @@ typedef struct snat_main_s {
 
 extern snat_main_t snat_main;
 extern vlib_node_registration_t snat_in2out_node;
+extern vlib_node_registration_t snat_in2out_output_node;
 extern vlib_node_registration_t snat_out2in_node;
 extern vlib_node_registration_t snat_in2out_fast_node;
 extern vlib_node_registration_t snat_out2in_fast_node;
 extern vlib_node_registration_t snat_in2out_worker_handoff_node;
+extern vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 extern vlib_node_registration_t snat_out2in_worker_handoff_node;
 extern vlib_node_registration_t snat_det_in2out_node;
 extern vlib_node_registration_t snat_det_out2in_node;
@@ -477,6 +482,8 @@ int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 clib_error_t * snat_api_init(vlib_main_t * vm, snat_main_t * sm);
 int snat_set_workers (uword * bitmap);
 int snat_interface_add_del(u32 sw_if_index, u8 is_inside, int is_del);
+int snat_interface_add_del_output_feature(u32 sw_if_index, u8 is_inside,
+                                          int is_del);
 int snat_add_interface_address(snat_main_t *sm, u32 sw_if_index, int is_del);
 uword unformat_snat_protocol(unformat_input_t * input, va_list * args);
 u8 * format_snat_protocol(u8 * s, va_list * args);
diff --git a/src/plugins/snat/snat_api.c b/src/plugins/snat/snat_api.c
index 237b080d..ee623d26 100644
--- a/src/plugins/snat/snat_api.c
+++ b/src/plugins/snat/snat_api.c
@@ -253,7 +253,91 @@ static void *vl_api_snat_interface_dump_t_print
   s = format (0, "SCRIPT: snat_interface_dump ");
 
   FINISH;
-} static void
+}
+
+static void
+  vl_api_snat_interface_add_del_output_feature_t_handler
+  (vl_api_snat_interface_add_del_output_feature_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_interface_add_del_output_feature_reply_t *rmp;
+  u8 is_del = mp->is_add == 0;
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  int rv = 0;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  rv = snat_interface_add_del_output_feature (sw_if_index, mp->is_inside,
+					      is_del);
+
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_SNAT_INTERFACE_ADD_DEL_OUTPUT_FEATURE_REPLY);
+}
+
+static void *vl_api_snat_interface_add_del_output_feature_t_print
+  (vl_api_snat_interface_add_del_output_feature_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_interface_add_del_output_feature ");
+  s = format (s, "sw_if_index %d %s %s",
+	      clib_host_to_net_u32 (mp->sw_if_index),
+	      mp->is_inside ? "in" : "out", mp->is_add ? "" : "del");
+
+  FINISH;
+}
+
+static void
+send_snat_interface_output_feature_details (snat_interface_t * i,
+					    unix_shared_memory_queue_t * q,
+					    u32 context)
+{
+  vl_api_snat_interface_output_feature_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id =
+    ntohs (VL_API_SNAT_INTERFACE_OUTPUT_FEATURE_DETAILS + sm->msg_id_base);
+  rmp->sw_if_index = ntohl (i->sw_if_index);
+  rmp->context = context;
+  rmp->is_inside = i->is_inside;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+  vl_api_snat_interface_output_feature_dump_t_handler
+  (vl_api_snat_interface_output_feature_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_interface_t *i;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  pool_foreach (i, sm->output_feature_interfaces,
+  ({
+    send_snat_interface_output_feature_details(i, q, mp->context);
+  }));
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_interface_output_feature_dump_t_print
+  (vl_api_snat_interface_output_feature_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_interface_output_feature_dump ");
+
+  FINISH;
+}
+
+static void
   vl_api_snat_add_static_mapping_t_handler
   (vl_api_snat_add_static_mapping_t * mp)
 {
@@ -1792,6 +1876,10 @@ _(SNAT_INTERFACE_ADDR_DUMP, snat_interface_addr_dump)                   \
 _(SNAT_IPFIX_ENABLE_DISABLE, snat_ipfix_enable_disable)                 \
 _(SNAT_USER_DUMP, snat_user_dump)                                       \
 _(SNAT_USER_SESSION_DUMP, snat_user_session_dump)                       \
+_(SNAT_INTERFACE_ADD_DEL_OUTPUT_FEATURE,                                \
+  snat_interface_add_del_output_feature)                                \
+_(SNAT_INTERFACE_OUTPUT_FEATURE_DUMP,                                   \
+  snat_interface_output_feature_dump)                                   \
 _(SNAT_ADD_DET_MAP, snat_add_det_map)                                   \
 _(SNAT_DET_FORWARD, snat_det_forward)                                   \
 _(SNAT_DET_REVERSE, snat_det_reverse)                                   \