vnet: device flow offload infra

Change-Id: Ibea4a96bdec5e368301a03d8b11a0712fa0265e0
Signed-off-by: Damjan Marion <damarion@cisco.com>

18 files changed, 1335 insertions, 0 deletions

diff --git a/src/plugins/dpdk.am b/src/plugins/dpdk.am
index 10f2fe4aaba..7c6e0058992 100644
--- a/src/plugins/dpdk.am
+++ b/src/plugins/dpdk.am
@@ -38,6 +38,7 @@ dpdk_plugin_la_SOURCES =					\
   dpdk/api/dpdk_api.c						\
   dpdk/device/cli.c						\
   dpdk/device/common.c						\
+  dpdk/device/flow.c						\
   dpdk/device/dpdk_priv.h					\
   dpdk/device/device.c						\
   dpdk/device/format.c						\
diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index 3a9c7813f0c..cdd9785a3f3 100644
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -715,6 +715,8 @@ VNET_DEVICE_CLASS (dpdk_device_class) = {
   .subif_add_del_function = dpdk_subif_add_del_function,
   .rx_redirect_to_node = dpdk_set_interface_next_node,
   .mac_addr_change_function = dpdk_set_mac_address,
+  .format_flow = format_dpdk_flow,
+  .flow_ops_function = dpdk_flow_ops_fn,
 };
 /* *INDENT-ON* */
 
diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h
index 0778659db55..e1966e438c0 100644
--- a/src/plugins/dpdk/device/dpdk.h
+++ b/src/plugins/dpdk/device/dpdk.h
@@ -40,6 +40,7 @@
 #include <rte_sched.h>
 #include <rte_net.h>
 #include <rte_bus_pci.h>
+#include <rte_flow.h>
 
 #include <vnet/unix/pcap.h>
 #include <vnet/devices/devices.h>
@@ -51,6 +52,7 @@
 #endif
 
 #include <vlib/pci/pci.h>
+#include <vnet/flow/flow.h>
 
 #define NB_MBUF   (16<<10)
 
@@ -158,6 +160,7 @@ typedef struct
   _( 8, BOND_SLAVE_UP, "bond-slave-up") \
   _( 9, TX_OFFLOAD, "tx-offload") \
   _(10, INTEL_PHDR_CKSUM, "intel-phdr-cksum") \
+  _(11, RX_FLOW_OFFLOAD, "rx-flow-offload")
 
 enum
 {
@@ -168,6 +171,20 @@ enum
 
 typedef struct
 {
+  u32 flow_index;
+  u32 mark;
+  struct rte_flow *handle;
+} dpdk_flow_entry_t;
+
+typedef struct
+{
+  u32 flow_id;
+  u16 next_index;
+  i16 buffer_advance;
+} dpdk_flow_lookup_entry_t;
+
+typedef struct
+{
   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
   volatile u32 **lockp;
 
@@ -202,6 +219,12 @@ typedef struct
   struct rte_eth_conf port_conf;
   struct rte_eth_txconf tx_conf;
 
+  /* flow related */
+  u32 supported_flow_actions;
+  dpdk_flow_entry_t *flow_entries;	/* pool */
+  dpdk_flow_lookup_entry_t *flow_lookup_entries;	/* vector */
+  struct rte_flow_error last_flow_error;
+
   /* HQoS related */
   dpdk_device_hqos_per_worker_thread_t *hqos_wt;
   dpdk_device_hqos_per_hqos_thread_t *hqos_ht;
@@ -472,7 +495,10 @@ format_function_t format_dpdk_tx_trace;
 format_function_t format_dpdk_rx_trace;
 format_function_t format_dpdk_rte_mbuf;
 format_function_t format_dpdk_rx_rte_mbuf;
+format_function_t format_dpdk_flow;
 unformat_function_t unformat_dpdk_log_level;
+vnet_flow_dev_ops_function_t dpdk_flow_ops_fn;
+
 clib_error_t *unformat_rss_fn (unformat_input_t * input, uword * rss_fn);
 clib_error_t *unformat_hqos (unformat_input_t * input,
 			     dpdk_device_config_hqos_t * hqos);
diff --git a/src/plugins/dpdk/device/flow.c b/src/plugins/dpdk/device/flow.c
new file mode 100644
index 00000000000..a0eb95f8fa0
--- /dev/null
+++ b/src/plugins/dpdk/device/flow.c
@@ -0,0 +1,286 @@
+/*
+ * Copyright (c) 2019 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/vnet.h>
+#include <vppinfra/vec.h>
+#include <vppinfra/format.h>
+#include <vlib/unix/cj.h>
+#include <assert.h>
+
+#include <vnet/ip/ip.h>
+#include <vnet/ethernet/ethernet.h>
+#include <vnet/ethernet/arp_packet.h>
+#include <dpdk/device/dpdk.h>
+
+#include <dpdk/device/dpdk_priv.h>
+#include <vppinfra/error.h>
+
+/* constant structs */
+static const struct rte_flow_attr ingress = {.ingress = 1 };
+static const struct rte_flow_item_eth any_eth[2] = { };
+static const struct rte_flow_item_vlan any_vlan[2] = { };
+
+static int
+dpdk_flow_add_n_touple (dpdk_device_t * xd, vnet_flow_t * f,
+			dpdk_flow_entry_t * fe)
+{
+  struct rte_flow_item_ipv4 ip4[2] = { };
+  struct rte_flow_item_ipv6 ip6[2] = { };
+  struct rte_flow_item_udp udp[2] = { };
+  struct rte_flow_item_tcp tcp[2] = { };
+  struct rte_flow_action_mark mark = { 0 };
+  struct rte_flow_item *item, *items = 0;
+  struct rte_flow_action *action, *actions = 0;
+  u16 src_port, dst_port, src_port_mask, dst_port_mask;
+  u8 protocol;
+  int rv = 0;
+
+  if (f->actions & (~xd->supported_flow_actions))
+    return VNET_FLOW_ERROR_NOT_SUPPORTED;
+
+  /* Ethernet */
+  vec_add2 (items, item, 1);
+  item->type = RTE_FLOW_ITEM_TYPE_ETH;
+  item->spec = any_eth;
+  item->mask = any_eth + 1;
+
+  /* VLAN */
+  vec_add2 (items, item, 1);
+  item->type = RTE_FLOW_ITEM_TYPE_VLAN;
+  item->spec = any_vlan;
+  item->mask = any_vlan + 1;
+
+  /* IP */
+  vec_add2 (items, item, 1);
+  if (f->type == VNET_FLOW_TYPE_IP6_N_TUPLE)
+    {
+      vnet_flow_ip6_n_tuple_t *t6 = &f->ip6_n_tuple;
+      clib_memcpy (ip6[0].hdr.src_addr, &t6->src_addr.addr, 16);
+      clib_memcpy (ip6[1].hdr.src_addr, &t6->src_addr.mask, 16);
+      clib_memcpy (ip6[0].hdr.dst_addr, &t6->dst_addr.addr, 16);
+      clib_memcpy (ip6[1].hdr.dst_addr, &t6->dst_addr.mask, 16);
+      item->type = RTE_FLOW_ITEM_TYPE_IPV6;
+      item->spec = ip6;
+      item->mask = ip6 + 1;
+
+      src_port = t6->src_port.port;
+      dst_port = t6->dst_port.port;
+      src_port_mask = t6->src_port.mask;
+      dst_port_mask = t6->dst_port.mask;
+      protocol = t6->protocol;
+    }
+  else
+    {
+      vnet_flow_ip4_n_tuple_t *t4 = &f->ip4_n_tuple;
+      ASSERT (f->type == VNET_FLOW_TYPE_IP4_N_TUPLE);
+      ip4[0].hdr.src_addr = t4->src_addr.mask.as_u32;
+      ip4[1].hdr.src_addr = t4->src_addr.mask.as_u32;
+      ip4[0].hdr.dst_addr = t4->dst_addr.addr.as_u32;
+      ip4[1].hdr.dst_addr = t4->dst_addr.mask.as_u32;
+      item->type = RTE_FLOW_ITEM_TYPE_IPV4;
+      item->spec = ip4;
+      item->mask = ip4 + 1;
+
+      src_port = t4->src_port.port;
+      dst_port = t4->dst_port.mask;
+      src_port_mask = t4->src_port.mask;
+      dst_port_mask = t4->dst_port.mask;
+      protocol = t4->protocol;
+    }
+
+  /* Layer 4 */
+  vec_add2 (items, item, 1);
+  if (protocol == IP_PROTOCOL_UDP)
+    {
+      udp[0].hdr.src_port = clib_host_to_net_u16 (src_port);
+      udp[1].hdr.src_port = clib_host_to_net_u16 (src_port_mask);
+      udp[0].hdr.dst_port = clib_host_to_net_u16 (dst_port);
+      udp[1].hdr.dst_port = clib_host_to_net_u16 (dst_port_mask);
+      item->type = RTE_FLOW_ITEM_TYPE_UDP;
+      item->spec = udp;
+      item->mask = udp + 1;
+    }
+  else if (protocol == IP_PROTOCOL_TCP)
+    {
+      tcp[0].hdr.src_port = clib_host_to_net_u16 (src_port);
+      tcp[1].hdr.src_port = clib_host_to_net_u16 (src_port_mask);
+      tcp[0].hdr.dst_port = clib_host_to_net_u16 (dst_port);
+      tcp[1].hdr.dst_port = clib_host_to_net_u16 (dst_port_mask);
+      item->type = RTE_FLOW_ITEM_TYPE_TCP;
+      item->spec = tcp;
+      item->mask = tcp + 1;
+    }
+  else
+    {
+      rv = VNET_FLOW_ERROR_NOT_SUPPORTED;
+      goto done;
+    }
+
+  /* The End */
+  vec_add2 (items, item, 1);
+  item->type = RTE_FLOW_ITEM_TYPE_END;
+
+  vec_add2 (actions, action, 1);
+  action->type = RTE_FLOW_ACTION_TYPE_PASSTHRU;
+
+  vec_add2 (actions, action, 1);
+  mark.id = fe->mark;
+  action->type = RTE_FLOW_ACTION_TYPE_MARK;
+  action->conf = &mark;
+
+  vec_add2 (actions, action, 1);
+  action->type = RTE_FLOW_ACTION_TYPE_END;
+
+  fe->handle = rte_flow_create (xd->device_index, &ingress, items, actions,
+				&xd->last_flow_error);
+
+  if (!fe->handle)
+    rv = VNET_FLOW_ERROR_NOT_SUPPORTED;
+
+done:
+  vec_free (items);
+  vec_free (actions);
+  return rv;
+}
+
+int
+dpdk_flow_ops_fn (vnet_main_t * vnm, vnet_flow_dev_op_t op, u32 dev_instance,
+		  u32 flow_index, uword * private_data)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  vnet_flow_t *flow = vnet_get_flow (flow_index);
+  dpdk_device_t *xd = vec_elt_at_index (dm->devices, dev_instance);
+  dpdk_flow_entry_t *fe;
+  dpdk_flow_lookup_entry_t *fle = 0;
+  int rv;
+
+  if (op == VNET_FLOW_DEV_OP_DEL_FLOW)
+    {
+      ASSERT (*private_data >= vec_len (xd->flow_entries));
+
+      fe = vec_elt_at_index (xd->flow_entries, *private_data);
+
+      if ((rv = rte_flow_destroy (xd->device_index, fe->handle,
+				  &xd->last_flow_error)))
+	return VNET_FLOW_ERROR_INTERNAL;
+
+      memset (fe, 0, sizeof (*fe));
+      pool_put (xd->flow_entries, fe);
+      return 0;
+    }
+
+  if (op != VNET_FLOW_DEV_OP_ADD_FLOW)
+    return VNET_FLOW_ERROR_NOT_SUPPORTED;
+
+  pool_get (xd->flow_entries, fe);
+  fe->flow_index = flow->index;
+
+  if (flow->actions == 0)
+    {
+      rv = VNET_FLOW_ERROR_NOT_SUPPORTED;
+      goto done;
+    }
+
+  /* if we need to mark packets, assign one mark */
+  if (flow->actions & (VNET_FLOW_ACTION_MARK |
+		       VNET_FLOW_ACTION_REDIRECT_TO_NODE |
+		       VNET_FLOW_ACTION_BUFFER_ADVANCE))
+    {
+      /* reserve slot 0 */
+      if (xd->flow_lookup_entries == 0)
+	pool_get_aligned (xd->flow_lookup_entries, fle,
+			  CLIB_CACHE_LINE_BYTES);
+      pool_get_aligned (xd->flow_lookup_entries, fle, CLIB_CACHE_LINE_BYTES);
+      fe->mark = fle - xd->flow_lookup_entries;
+    }
+  else
+    fe->mark = 0;
+
+  switch (flow->type)
+    {
+    case VNET_FLOW_TYPE_IP4_N_TUPLE:
+    case VNET_FLOW_TYPE_IP6_N_TUPLE:
+      if ((rv = dpdk_flow_add_n_touple (xd, flow, fe)))
+	goto done;
+      break;
+    default:
+      rv = VNET_FLOW_ERROR_NOT_SUPPORTED;
+      goto done;
+    }
+
+
+  *private_data = fe - xd->flow_entries;
+
+  /* install entry in the lookup table */
+  memset (fle, ~1, sizeof (*fle));
+  if (flow->actions & VNET_FLOW_ACTION_MARK)
+    fle->flow_id = flow->mark_flow_id;
+  if (flow->actions & VNET_FLOW_ACTION_REDIRECT_TO_NODE)
+    fle->next_index = flow->redirect_device_input_next_index;
+  if (flow->actions & VNET_FLOW_ACTION_BUFFER_ADVANCE)
+    fle->buffer_advance = flow->buffer_advance;
+
+done:
+  if (rv)
+    {
+      memset (fe, 0, sizeof (*fe));
+      pool_put (xd->flow_entries, fe);
+      if (fle)
+	{
+	  memset (fle, 0, sizeof (*fle));
+	  pool_put (xd->flow_lookup_entries, fle);
+	}
+    }
+  return rv;
+}
+
+u8 *
+format_dpdk_flow (u8 * s, va_list * args)
+{
+  u32 dev_instance = va_arg (*args, u32);
+  u32 flow_index = va_arg (*args, u32);
+  uword private_data = va_arg (*args, uword);
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd = vec_elt_at_index (dm->devices, dev_instance);
+  dpdk_flow_entry_t *fe;
+
+  if (flow_index == ~0)
+    {
+      s = format (s, "%-25s: %U\n", "supported flow actions",
+		  format_flow_actions, xd->supported_flow_actions);
+      s = format (s, "%-25s: %d\n", "last DPDK error type",
+		  xd->last_flow_error.type);
+      s = format (s, "%-25s: %s\n", "last DPDK error message",
+		  xd->last_flow_error.message ? xd->last_flow_error.message :
+		  "n/a");
+      return s;
+    }
+
+  fe = vec_elt_at_index (xd->flow_entries, private_data);
+
+  if (!fe)
+    return format (s, "unknown flow");
+
+  s = format (s, "mark %u", fe->mark);
+  return s;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c
index 83d26ce13e5..9d4c0fd5d7f 100755
--- a/src/plugins/dpdk/device/init.c
+++ b/src/plugins/dpdk/device/init.c
@@ -385,6 +385,14 @@ dpdk_lib_init (dpdk_main_t * dm)
 	    case VNET_DPDK_PMD_IXGBE:
 	    case VNET_DPDK_PMD_I40E:
 	      xd->port_type = port_type_from_speed_capa (&dev_info);
+#if 0
+	      xd->port_conf.fdir_conf.mode = RTE_FDIR_MODE_PERFECT;
+#endif
+	      xd->supported_flow_actions = VNET_FLOW_ACTION_MARK |
+		VNET_FLOW_ACTION_REDIRECT_TO_NODE |
+		VNET_FLOW_ACTION_BUFFER_ADVANCE |
+		VNET_FLOW_ACTION_COUNT | VNET_FLOW_ACTION_DROP;
+
 	      if (dm->conf->no_tx_checksum_offload == 0)
 		{
 		  xd->tx_conf.txq_flags &= ~ETH_TXQ_FLAGS_NOXSUMS;
diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c
index 7bb1fb3c9c1..7ba4dad8ae9 100644
--- a/src/plugins/dpdk/device/node.c
+++ b/src/plugins/dpdk/device/node.c
@@ -474,6 +474,40 @@ dpdk_set_next_from_etype (vlib_main_t * vm, vlib_node_runtime_t * node,
     }
 }
 
+static_always_inline void
+dpdk_process_flow_offload (dpdk_device_t * xd, dpdk_per_thread_data_t * ptd,
+			   uword n_rx_packets)
+{
+  uword n;
+  dpdk_flow_lookup_entry_t *fle;
+  vlib_buffer_t *b0;
+
+  /* TODO prefetch and quad-loop */
+  for (n = 0; n < n_rx_packets; n++)
+    {
+      if ((ptd->flags[n] & (1 << DPDK_RX_F_FDIR)) == 0)
+	continue;
+
+      fle = vec_elt_at_index (xd->flow_lookup_entries,
+			      ptd->mbufs[n]->hash.fdir.hi);
+
+      if (fle->next_index != (u16) ~ 0)
+	ptd->next[n] = fle->next_index;
+
+      if (fle->flow_id != ~0)
+	{
+	  b0 = vlib_buffer_from_rte_mbuf (ptd->mbufs[n]);
+	  b0->flow_id = fle->flow_id;
+	}
+
+      if (fle->buffer_advance != ~0)
+	{
+	  b0 = vlib_buffer_from_rte_mbuf (ptd->mbufs[n]);
+	  vlib_buffer_advance (b0, fle->buffer_advance);
+	}
+    }
+}
+
 static_always_inline u32
 dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd,
 		   vlib_node_runtime_t * node, u32 thread_index, u16 queue_id)
@@ -549,6 +583,12 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd,
   else
     dpdk_set_next_from_etype (vm, node, ptd, n_rx_packets);
 
+  /* flow offload - process if rx flow offlaod enabled and at least one packet
+     is marked */
+  if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD) &&
+		     (or_flags & (1 << DPDK_RX_F_FDIR))))
+    dpdk_process_flow_offload (xd, ptd, n_rx_packets);
+
   /* is at least one packet marked as ip4 checksum bad? */
   if (PREDICT_FALSE (or_flags & (1 << DPDK_RX_F_CKSUM_BAD)))
     for (n = 0; n < n_rx_packets; n++)
diff --git a/src/vnet.am b/src/vnet.am
index adf93398648..2a5a4b83286 100644
--- a/src/vnet.am
+++ b/src/vnet.am
@@ -35,6 +35,8 @@ libvnet_la_SOURCES +=				\
   vnet/config.c					\
   vnet/devices/devices.c			\
   vnet/devices/netlink.c			\
+  vnet/flow/flow.c				\
+  vnet/flow/flow_cli.c				\
   vnet/handoff.c				\
   vnet/interface.c				\
   vnet/interface_api.c				\
@@ -51,6 +53,7 @@ nobase_include_HEADERS +=			\
   vnet/config.h					\
   vnet/devices/devices.h			\
   vnet/devices/netlink.h			\
+  vnet/flow/flow.h				\
   vnet/global_funcs.h				\
   vnet/handoff.h				\
   vnet/interface.h				\
diff --git a/src/vnet/flow/flow.c b/src/vnet/flow/flow.c
new file mode 100644
index 00000000000..91e51120a7e
--- /dev/null
+++ b/src/vnet/flow/flow.c
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/vnet.h>
+#include <vnet/ip/ip.h>
+#include <vnet/ethernet/ethernet.h>
+#include <vnet/flow/flow.h>
+
+vnet_flow_main_t flow_main;
+
+int
+vnet_flow_get_range (vnet_main_t * vnm, char *owner, u32 count, u32 * start)
+{
+  vnet_flow_main_t *fm = &flow_main;
+  vnet_flow_range_t *r;
+
+  /* skip 0 */
+  if (fm->flows_used == 0)
+    fm->flows_used = 1;
+
+  *start = fm->flows_used;
+  fm->flows_used += count;
+  vec_add2 (fm->ranges, r, 1);
+  r->start = *start;
+  r->count = count;
+  r->owner = format (0, "%s%c", owner, 0);
+  return 0;
+}
+
+int
+vnet_flow_add (vnet_main_t * vnm, vnet_flow_t * flow, u32 * flow_index)
+{
+  vnet_flow_main_t *fm = &flow_main;
+  vnet_flow_t *f;
+
+  pool_get (fm->global_flow_pool, f);
+  *flow_index = f - fm->global_flow_pool;
+  clib_memcpy (f, flow, sizeof (vnet_flow_t));
+  f->private_data = 0;
+  f->index = *flow_index;
+  return 0;
+}
+
+vnet_flow_t *
+vnet_get_flow (u32 flow_index)
+{
+  vnet_flow_main_t *fm = &flow_main;
+  if (pool_is_free_index (fm->global_flow_pool, flow_index))
+    return 0;
+
+  return pool_elt_at_index (fm->global_flow_pool, flow_index);
+}
+
+int
+vnet_flow_del (vnet_main_t * vnm, u32 flow_index)
+{
+  vnet_flow_main_t *fm = &flow_main;
+  vnet_flow_t *f = vnet_get_flow (flow_index);
+  uword hw_if_index;
+  uword private_data;
+
+  if (f == 0)
+    return VNET_FLOW_ERROR_NO_SUCH_ENTRY;
+
+  /* *INDENT-OFF* */
+  hash_foreach (hw_if_index, private_data, f->private_data,
+    ({
+     vnet_flow_disable (vnm, flow_index, hw_if_index);
+    }));
+  /* *INDENT-ON* */
+
+  hash_free (f->private_data);
+  memset (f, 0, sizeof (*f));
+  pool_put (fm->global_flow_pool, f);
+  return 0;
+}
+
+int
+vnet_flow_enable (vnet_main_t * vnm, u32 flow_index, u32 hw_if_index)
+{
+  vnet_flow_t *f = vnet_get_flow (flow_index);
+  vnet_hw_interface_t *hi;
+  vnet_device_class_t *dev_class;
+  uword private_data;
+  int rv;
+
+  if (!vnet_hw_interface_is_valid (vnm, hw_if_index))
+    return VNET_FLOW_ERROR_NO_SUCH_INTERFACE;
+
+  /* don't enable flow twice */
+  if (hash_get (f->private_data, hw_if_index) != 0)
+    return VNET_FLOW_ERROR_ALREADY_DONE;
+
+  hi = vnet_get_hw_interface (vnm, hw_if_index);
+  dev_class = vnet_get_device_class (vnm, hi->dev_class_index);
+
+  if (dev_class->flow_ops_function == 0)
+    return VNET_FLOW_ERROR_NOT_SUPPORTED;
+
+  rv = dev_class->flow_ops_function (vnm, VNET_FLOW_DEV_OP_ADD_FLOW,
+				     hi->dev_instance, flow_index,
+				     &private_data);
+
+  if (rv)
+    return rv;
+
+  hash_set (f->private_data, hw_if_index, private_data);
+  return 0;
+}
+
+int
+vnet_flow_disable (vnet_main_t * vnm, u32 flow_index, u32 hw_if_index)
+{
+  vnet_flow_t *f = vnet_get_flow (flow_index);
+  vnet_hw_interface_t *hi;
+  vnet_device_class_t *dev_class;
+  uword *p;
+  int rv;
+
+  if (!vnet_hw_interface_is_valid (vnm, hw_if_index))
+    return VNET_FLOW_ERROR_NO_SUCH_INTERFACE;
+
+  /* don't disable if not enabled */
+  if ((p = hash_get (f->private_data, hw_if_index)) == 0)
+    return VNET_FLOW_ERROR_ALREADY_DONE;
+
+  hi = vnet_get_hw_interface (vnm, hw_if_index);
+  dev_class = vnet_get_device_class (vnm, hi->dev_class_index);
+
+  rv = dev_class->flow_ops_function (vnm, VNET_FLOW_DEV_OP_DEL_FLOW,
+				     hi->dev_instance, flow_index, p);
+
+  if (rv)
+    return rv;
+
+  hash_unset (f->private_data, hw_if_index);
+  return 0;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/flow/flow.h b/src/vnet/flow/flow.h
new file mode 100644
index 00000000000..bc9b2f19cc3
--- /dev/null
+++ b/src/vnet/flow/flow.h
@@ -0,0 +1,193 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef included_vnet_flow_flow_h
+#define included_vnet_flow_flow_h
+
+#include <vppinfra/clib.h>
+#include <vnet/unix/pcap.h>
+#include <vnet/l3_types.h>
+#include <vnet/ip/ip4_packet.h>
+#include <vnet/ip/ip6_packet.h>
+
+#define foreach_flow_type \
+  _(IP4_N_TUPLE, ip4_n_tuple, "ipv4-n-tuple") \
+  _(IP6_N_TUPLE, ip6_n_tuple, "ipv4-n-tuple") \
+  _(IP4_VXLAN, ip4_vxlan, "ipv4-vxlan") \
+  _(IP6_VXLAN, ip6_vxlan, "ipv6-vxlan")
+
+#define foreach_flow_entry_ip4_n_tuple \
+  _fe(ip4_address_and_mask_t, src_addr) \
+  _fe(ip4_address_and_mask_t, dst_addr) \
+  _fe(ip_port_and_mask_t, src_port) \
+  _fe(ip_port_and_mask_t, dst_port) \
+  _fe(ip_protocol_t, protocol)
+
+#define foreach_flow_entry_ip6_n_tuple \
+  _fe(ip6_address_and_mask_t, src_addr) \
+  _fe(ip6_address_and_mask_t, dst_addr) \
+  _fe(ip_port_and_mask_t, src_port) \
+  _fe(ip_port_and_mask_t, dst_port) \
+  _fe(ip_protocol_t, protocol)
+
+#define foreach_flow_entry_ip4_vxlan \
+  _fe(ip4_address_t, src_addr) \
+  _fe(ip4_address_t, dst_addr) \
+  _fe(u16, dst_port) \
+  _fe(u16, vni)
+
+#define foreach_flow_entry_ip6_vxlan \
+  _fe(ip6_address_t, src_addr) \
+  _fe(ip6_address_t, dst_addr) \
+  _fe(u16, dst_port) \
+  _fe(u16, vni)
+
+#define foreach_flow_action \
+  _(0, COUNT, "count") \
+  _(1, MARK, "mark") \
+  _(2, BUFFER_ADVANCE, "buffer-advance") \
+  _(3, REDIRECT_TO_NODE, "redirect-to-node") \
+  _(4, REDIRECT_TO_QUEUE, "redirect-to-queue") \
+  _(5, DROP, "drop")
+
+typedef enum
+{
+#define _(v,n,s)  VNET_FLOW_ACTION_##n = (1 << v),
+  foreach_flow_action
+#undef _
+} vnet_flow_action_t;
+
+
+#define foreach_flow_error \
+  _( -1, NOT_SUPPORTED, "not supported")			\
+  _( -2, ALREADY_DONE, "already done")				\
+  _( -3, ALREADY_EXISTS, "already exists")			\
+  _( -4, NO_SUCH_ENTRY, "no such entry")			\
+  _( -5, NO_SUCH_INTERFACE, "no such interface")		\
+  _( -6, INTERNAL, "internal error")
+
+typedef enum
+{
+  VNET_FLOW_NO_ERROR = 0,
+#define _(v,n,s)  VNET_FLOW_ERROR_##n = v,
+  foreach_flow_error
+#undef _
+} vnet_flow_error_t;
+
+typedef struct
+{
+  u16 port, mask;
+} ip_port_and_mask_t;
+
+typedef enum
+{
+  VNET_FLOW_TYPE_UNKNOWN,
+#define _(a,b,c) VNET_FLOW_TYPE_##a,
+  foreach_flow_type
+#undef _
+    VNET_FLOW_N_TYPES,
+} vnet_flow_type_t;
+
+
+/*
+ * Create typedef struct vnet_flow_XXX_t
+ */
+#define _fe(a, b) a b;
+#define _(a,b,c) \
+typedef struct { \
+int foo; \
+foreach_flow_entry_##b \
+} vnet_flow_##b##_t;
+foreach_flow_type;
+#undef _
+#undef _fe
+
+/* main flow struct */
+typedef struct
+{
+  /* flow type */
+  vnet_flow_type_t type;
+
+  /* flow index */
+  u32 index;
+
+  /* bitmap of flow actions (VNET_FLOW_ACTION_*) */
+  u32 actions;
+
+  /* flow id for VNET_FLOW_ACTION_MARK */
+  u32 mark_flow_id;
+
+  /* node index and next index for VNET_FLOW_ACTION_REDIRECT_TO_NODE */
+  u32 redirect_node_index;
+  u32 redirect_device_input_next_index;
+
+  /* queue for VNET_FLOW_ACTION_REDIRECT_TO_QUEUE */
+  u32 redirect_queue;
+
+  /* buffer offset for VNET_FLOW_ACTION_BUFFER_ADVANCE */
+  i32 buffer_advance;
+
+  union
+  {
+#define _(a,b,c) vnet_flow_##b##_t b;
+    foreach_flow_type
+#undef _
+  };
+
+  /* per-interface private data */
+  uword *private_data;
+} vnet_flow_t;
+
+int vnet_flow_get_range (vnet_main_t * vnm, char *owner, u32 count,
+			 u32 * start);
+int vnet_flow_add (vnet_main_t * vnm, vnet_flow_t * flow, u32 * flow_index);
+int vnet_flow_enable (vnet_main_t * vnm, u32 flow_index, u32 hw_if_index);
+int vnet_flow_disable (vnet_main_t * vnm, u32 flow_index, u32 hw_if_index);
+int vnet_flow_del (vnet_main_t * vnm, u32 flow_index);
+vnet_flow_t *vnet_get_flow (u32 flow_index);
+
+typedef struct
+{
+  u32 start;
+  u32 count;
+  u8 *owner;
+} vnet_flow_range_t;
+
+typedef struct
+{
+  /* pool of device flow entries */
+  vnet_flow_t *global_flow_pool;
+
+  /* flow ids allocated */
+  u32 flows_used;
+
+  /* vector of flow ranges */
+  vnet_flow_range_t *ranges;
+
+} vnet_flow_main_t;
+
+extern vnet_flow_main_t flow_main;
+
+format_function_t format_flow_actions;
+
+#endif /* included_vnet_flow_flow_h */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/flow/flow_cli.c b/src/vnet/flow/flow_cli.c
new file mode 100644
index 00000000000..11f3bcfad97
--- /dev/null
+++ b/src/vnet/flow/flow_cli.c
@@ -0,0 +1,467 @@
+/*
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/vnet.h>
+#include <vnet/devices/devices.h>
+#include <vnet/ip/ip.h>
+#include <vnet/ethernet/ethernet.h>
+#include <vnet/flow/flow.h>
+
+static format_function_t format_flow;
+
+uword
+unformat_ip_port_and_mask (unformat_input_t * input, va_list * args)
+{
+  ip_port_and_mask_t *pm = va_arg (*args, ip_port_and_mask_t *);
+  u32 port = 0, mask = 0;
+
+  if (unformat (input, "any"))
+    ;
+  else if (unformat (input, "%u/%u", &port, &mask))
+    ;
+  else if (unformat (input, "%u/0x%x", &port, &mask))
+    ;
+  else if (unformat (input, "%u", &port))
+    mask = 0xffff;
+  else
+    return 0;
+
+  if (port > 0xffff || mask > 0xffff)
+    return 0;
+
+  pm->port = port;
+  pm->mask = mask;
+  return 1;
+}
+
+u8 *
+format_ip_port_and_mask (u8 * s, va_list * args)
+{
+  ip_port_and_mask_t *pm = va_arg (*args, ip_port_and_mask_t *);
+
+  if (pm->port == 0 && pm->mask == 0)
+    return format (s, "any");
+
+  if (pm->mask == 0xffff)
+    return format (s, "%u", pm->port);
+
+  return format (s, "%u/0x%x", pm->port, pm->mask);
+}
+
+u8 *
+format_flow_error (u8 * s, va_list * args)
+{
+  int error = va_arg (*args, int);
+
+  if (error == 0)
+    return format (s, "no error");
+
+#define _(v,n,str) if (error == v) return format (s, #str);
+  foreach_flow_error;
+#undef _
+
+  return format (s, "unknown error (%d)", error);
+}
+
+u8 *
+format_flow_actions (u8 * s, va_list * args)
+{
+  u32 actions = va_arg (*args, u32);
+  u8 *t = 0;
+
+#define _(a, b, c) if (actions & (1 << a)) \
+  t = format (t, "%s%s", t ? " ":"", c);
+  foreach_flow_action
+#undef _
+    s = format (s, "%v", t);
+  vec_free (t);
+  return s;
+}
+
+static const char *flow_type_strings[] = { 0,
+#define _(a,b,c) c,
+  foreach_flow_type
+#undef _
+};
+
+static clib_error_t *
+show_flow_entry (vlib_main_t * vm, unformat_input_t * input,
+		 vlib_cli_command_t * cmd_arg)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  vnet_flow_main_t *fm = &flow_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  vnet_hw_interface_t *hi;
+  vnet_device_class_t *dev_class;
+  vnet_flow_t *f;
+  uword private_data;
+  u32 index = ~0, hw_if_index;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    goto no_args;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "index %u", &index))
+	;
+      else
+	return clib_error_return (0, "parse error: '%U'",
+				  format_unformat_error, line_input);
+    }
+
+  unformat_free (line_input);
+
+  if (index != ~0)
+    {
+      if ((f = vnet_get_flow (index)) == 0)
+	return clib_error_return (0, "no such flow");
+
+      vlib_cli_output (vm, "%-10s: %u", "index", f->index);
+      vlib_cli_output (vm, "%-10s: %s", "type", flow_type_strings[f->type]);
+      vlib_cli_output (vm, "%-10s: %U", "match", format_flow, f);
+      /* *INDENT-OFF* */
+      hash_foreach (hw_if_index, private_data, f->private_data,
+        ({
+	 hi = vnet_get_hw_interface (vnm, hw_if_index);
+	  dev_class = vnet_get_device_class (vnm, hi->dev_class_index);
+	  vlib_cli_output (vm,  "interface %U\n",
+			   format_vnet_hw_if_index_name, vnm, hw_if_index);
+	  if (dev_class->format_flow)
+	    vlib_cli_output (vm,  "  %U\n", dev_class->format_flow,
+			     hi->dev_instance, f->index, private_data);
+         }));
+      /* *INDENT-ON* */
+      return 0;
+    }
+
+no_args:
+  /* *INDENT-OFF* */
+  pool_foreach (f, fm->global_flow_pool,
+    {
+      vlib_cli_output (vm, "%U\n", format_flow, f);
+    });
+  /* *INDENT-ON* */
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_flow_entry_command, static) = {
+    .path = "show flow entry",
+    .short_help = "show flow entry [index <index>]",
+    .function = show_flow_entry,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+show_flow_ranges (vlib_main_t * vm, unformat_input_t * input,
+		  vlib_cli_command_t * cmd_arg)
+{
+  vnet_flow_main_t *fm = &flow_main;
+  vnet_flow_range_t *r = 0;
+
+  vlib_cli_output (vm, "%8s  %8s  %s", "Start", "Count", "Owner");
+
+  /* *INDENT-OFF* */
+  vec_foreach (r, fm->ranges)
+    {
+      vlib_cli_output (vm, "%8u  %8u  %s", r->start, r->count, r->owner);
+    };
+  /* *INDENT-ON* */
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_flow_ranges_command, static) = {
+    .path = "show flow ranges",
+    .short_help = "show flow ranges",
+    .function = show_flow_ranges,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+show_flow_interface (vlib_main_t * vm, unformat_input_t * input,
+		     vlib_cli_command_t * cmd_arg)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  vnet_hw_interface_t *hi;
+  vnet_device_class_t *dev_class;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  u32 hw_if_index = ~0;
+
+  if (unformat_user (input, unformat_line_input, line_input))
+    {
+      while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+	{
+	  if (unformat (line_input, "%U",
+			unformat_vnet_hw_interface, vnm, &hw_if_index))
+	    ;
+	  else
+	    return clib_error_return (0, "parse error: '%U'",
+				      format_unformat_error, line_input);
+	}
+      unformat_free (line_input);
+    }
+
+  if (hw_if_index == ~0)
+    return clib_error_return (0, "please specify interface");
+
+  hi = vnet_get_hw_interface (vnm, hw_if_index);
+  dev_class = vnet_get_device_class (vnm, hi->dev_class_index);
+  if (dev_class->format_flow == 0)
+    return clib_error_return (0, "not supported");
+
+  vlib_cli_output (vm, "%U", dev_class->format_flow, hi->dev_instance, ~0, 0);
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_flow_interface_command, static) = {
+    .path = "show flow interface",
+    .short_help = "show flow interface <interface name>",
+    .function = show_flow_interface,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+test_flow (vlib_main_t * vm, unformat_input_t * input,
+	   vlib_cli_command_t * cmd_arg)
+{
+  vnet_flow_t flow;
+  vnet_main_t *vnm = vnet_get_main ();
+  unformat_input_t _line_input, *line_input = &_line_input;
+  enum
+  {
+    FLOW_UNKNOWN_ACTION,
+    FLOW_ADD,
+    FLOW_DEL,
+    FLOW_ENABLE,
+    FLOW_DISABLE
+  } action = FLOW_UNKNOWN_ACTION;
+  u32 hw_if_index = ~0, tmp, flow_index = ~0;
+  int rv;
+  u8 prot;
+
+  memset (&flow, 0, sizeof (vnet_flow_t));
+  flow.index = ~0;
+  flow.actions = 0;
+  flow.ip4_n_tuple.protocol = ~0;
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "add"))
+	action = FLOW_ADD;
+      else if (unformat (line_input, "del"))
+	action = FLOW_DEL;
+      else if (unformat (line_input, "enable"))
+	action = FLOW_ENABLE;
+      else if (unformat (line_input, "disable"))
+	action = FLOW_DISABLE;
+      else if (unformat (line_input, "src-ip %U",
+			 unformat_ip4_address_and_mask,
+			 &flow.ip4_n_tuple.src_addr))
+	;
+      else if (unformat (line_input, "dst-ip %U",
+			 unformat_ip4_address_and_mask,
+			 &flow.ip4_n_tuple.dst_addr))
+	;
+      else if (unformat (line_input, "src-port %U", unformat_ip_port_and_mask,
+			 &flow.ip4_n_tuple.src_port))
+	;
+      else if (unformat (line_input, "dst-port %U", unformat_ip_port_and_mask,
+			 &flow.ip4_n_tuple.dst_port))
+	;
+      else if (unformat (line_input, "proto %U", unformat_ip_protocol, &prot))
+	flow.ip4_n_tuple.protocol = prot;
+      else if (unformat (line_input, "proto %u", &tmp))
+	flow.ip4_n_tuple.protocol = tmp;
+      else if (unformat (line_input, "index %u", &flow_index))
+	;
+      else if (unformat (line_input, "next-node %U", unformat_vlib_node, vm,
+			 &flow.redirect_node_index))
+	flow.actions |= VNET_FLOW_ACTION_REDIRECT_TO_NODE;
+      else if (unformat (line_input, "mark %d", &flow.mark_flow_id))
+	flow.actions |= VNET_FLOW_ACTION_MARK;
+      else if (unformat (line_input, "buffer-advance %d",
+			 &flow.buffer_advance))
+	flow.actions |= VNET_FLOW_ACTION_BUFFER_ADVANCE;
+      else if (unformat (line_input, "%U", unformat_vnet_hw_interface, vnm,
+			 &hw_if_index))
+	;
+      else
+	return clib_error_return (0, "parse error: '%U'",
+				  format_unformat_error, line_input);
+    }
+
+  unformat_free (line_input);
+
+  if (hw_if_index == ~0 && (action == FLOW_ENABLE || action == FLOW_DISABLE))
+    return clib_error_return (0, "Please specify interface name");
+
+  if (flow_index == ~0 && (action == FLOW_ENABLE || action == FLOW_DISABLE ||
+			   action == FLOW_DEL))
+    return clib_error_return (0, "Please specify flow index");
+
+  switch (action)
+    {
+    case FLOW_ADD:
+      if (flow.ip4_n_tuple.protocol == (ip_protocol_t) ~ 0)
+	return clib_error_return (0, "Please specify ip protocol");
+
+      if (flow.actions == 0)
+	return clib_error_return (0, "Please specify at least one action");
+      flow.type = VNET_FLOW_TYPE_IP4_N_TUPLE;
+      rv = vnet_flow_add (vnm, &flow, &flow_index);
+      break;
+    case FLOW_DEL:
+      rv = vnet_flow_del (vnm, flow_index);
+      break;
+    case FLOW_ENABLE:
+      rv = vnet_flow_enable (vnm, flow_index, hw_if_index);
+      break;
+    case FLOW_DISABLE:
+      rv = vnet_flow_disable (vnm, flow_index, hw_if_index);
+      break;
+    default:
+      return clib_error_return (0, "please specify action (add, del, enable,"
+				" disable)");
+    }
+
+  if (rv < 0)
+    return clib_error_return (0, "flow error: %U", format_flow_error, rv);
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (test_flow_command, static) = {
+    .path = "test flow",
+    .short_help = "test flow add [src-ip <ip-addr/mask>] [dst-ip "
+      "<ip-addr/mask>] [src-port <port/mask>] [dst-port <port/mask>] "
+      "[proto <ip-proto>",
+    .function = test_flow,
+};
+/* *INDENT-ON* */
+
+
+static u8 *
+format_flow_match_element (u8 * s, va_list * args)
+{
+  char *type = va_arg (*args, char *);
+  void *ptr = va_arg (*args, void *);
+
+  if (strncmp (type, "u8", 2) == 0)
+    return format (s, "%d", *(u8 *) ptr);
+
+  if (strncmp (type, "u16", 3) == 0)
+    return format (s, "%d", *(u16 *) ptr);
+
+  if (strncmp (type, "u32", 3) == 0)
+    return format (s, "%d", *(u32 *) ptr);
+
+  if (strncmp (type, "ip4_address_t", 13) == 0)
+    return format (s, "%U", format_ip4_address, ptr);
+
+  if (strncmp (type, "ip4_address_and_mask_t", 13) == 0)
+    return format (s, "%U", format_ip4_address_and_mask, ptr);
+
+  if (strncmp (type, "ip6_address_t", 13) == 0)
+    return format (s, "%U", format_ip6_address, ptr);
+
+  if (strncmp (type, "ip6_address_and_mask_t", 13) == 0)
+    return format (s, "%U", format_ip6_address_and_mask, ptr);
+
+  if (strncmp (type, "ip_protocol_t", 13) == 0)
+    return format (s, "%U", format_ip_protocol, *(ip_protocol_t *) ptr);
+
+  if (strncmp (type, "ip_port_and_mask_t", 18) == 0)
+    return format (s, "%U", format_ip_port_and_mask, ptr);
+
+  s = format (s, "unknown type '%s'", type);
+  return s;
+}
+
+#define _fe(a,b) s2 = format (s2, "%s%s %U", s2 ? ", ":"", #b, \
+			      format_flow_match_element, #a, &f->b);
+#define _(a,b,c) \
+u8 * format_flow_match_##b (u8 * s, va_list * args)			\
+{									\
+  vnet_flow_##b##_t *f = __builtin_va_arg (*args, vnet_flow_##b##_t *); \
+  u8 *s2 = 0; \
+foreach_flow_entry_##b \
+  s = format (s, "%v", s2);; \
+  vec_free (s2); \
+return s; \
+}
+foreach_flow_type
+#undef _
+#undef _fe
+static u8 *
+format_flow_match (u8 * s, va_list * args)
+{
+  vnet_flow_t *f = va_arg (*args, vnet_flow_t *);
+
+#define _(a,b,c) \
+  if (f->type == VNET_FLOW_TYPE_##a) \
+    return format (s, "%U", format_flow_match_##b, &f->b);
+  foreach_flow_type;
+#undef _
+
+  return s;
+}
+
+static u8 *
+format_flow (u8 * s, va_list * args)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  vnet_flow_t *f = va_arg (*args, vnet_flow_t *);
+  u32 indent = format_get_indent (s);
+  u8 *t = 0;
+
+  s = format (s, "flow-index %u type %s active %u",
+	      f->index, flow_type_strings[f->type],
+	      hash_elts (f->private_data)),
+    s = format (s, "\n%Umatch: %U", format_white_space, indent + 2,
+		format_flow_match, f);
+  s = format (s, "\n%Uaction: %U", format_white_space, indent + 2,
+	      format_flow_actions, f->actions);
+
+  if (f->actions & VNET_FLOW_ACTION_MARK)
+    t = format (t, "%smark %u", t ? ", " : "", f->mark_flow_id);
+
+  if (f->actions & VNET_FLOW_ACTION_REDIRECT_TO_NODE)
+    t = format (t, "%snext-node %U", t ? ", " : "",
+		format_vlib_node_name, vm, f->redirect_node_index);
+
+  if (f->actions & VNET_FLOW_ACTION_BUFFER_ADVANCE)
+    t = format (t, "%sbuffer-advance %d", t ? ", " : "", f->buffer_advance);
+
+  if (t)
+    {
+      s = format (s, "\n%U%v", format_white_space, indent + 4, t);
+      vec_free (t);
+    }
+
+  return s;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/interface.h b/src/vnet/interface.h
index b7b350f68fe..6eaf50166c0 100644
--- a/src/vnet/interface.h
+++ b/src/vnet/interface.h
@@ -81,6 +81,20 @@ typedef clib_error_t *(vnet_interface_set_l2_mode_function_t)
   (struct vnet_main_t * vnm, struct vnet_hw_interface_t * hi,
    i32 l2_if_adjust);
 
+typedef enum
+{
+  VNET_FLOW_DEV_OP_ADD_FLOW,
+  VNET_FLOW_DEV_OP_DEL_FLOW,
+  VNET_FLOW_DEV_OP_GET_COUNTER,
+  VNET_FLOW_DEV_OP_RESET_COUNTER,
+} vnet_flow_dev_op_t;
+
+/* Interface flow opeations callback. */
+typedef int (vnet_flow_dev_ops_function_t) (struct vnet_main_t * vnm,
+					    vnet_flow_dev_op_t op,
+					    u32 hw_if_index, u32 index,
+					    uword * private_data);
+
 typedef enum vnet_interface_function_priority_t_
 {
   VNET_ITF_FUNC_PRIORITY_LOW,
@@ -189,6 +203,9 @@ typedef struct _vnet_device_class
   int (*name_renumber) (struct vnet_hw_interface_t * hi,
 			u32 new_dev_instance);
 
+  /* Interface flow offload operations */
+  vnet_flow_dev_ops_function_t *flow_ops_function;
+
   /* Format device instance as name. */
   format_function_t *format_device_name;
 
@@ -201,6 +218,9 @@ typedef struct _vnet_device_class
   /* Trace buffer format for TX function. */
   format_function_t *format_tx_trace;
 
+  /* Format flow offload entry */
+  format_function_t *format_flow;
+
   /* Function to clear hardware counters for device. */
   void (*clear_counters) (u32 dev_class_instance);
 
diff --git a/src/vnet/interface_format.c b/src/vnet/interface_format.c
index e58369a1126..7e94092b52e 100644
--- a/src/vnet/interface_format.c
+++ b/src/vnet/interface_format.c
@@ -177,6 +177,21 @@ format_vnet_sw_if_index_name (u8 * s, va_list * args)
 }
 
 u8 *
+format_vnet_hw_if_index_name (u8 * s, va_list * args)
+{
+  vnet_main_t *vnm = va_arg (*args, vnet_main_t *);
+  u32 hw_if_index = va_arg (*args, u32);
+  vnet_hw_interface_t *hi;
+
+  hi = vnet_get_hw_interface (vnm, hw_if_index);
+
+  if (hi == 0)
+    return format (s, "DELETED");
+
+  return format (s, "%v", hi->name);
+}
+
+u8 *
 format_vnet_sw_interface_cntrs (u8 * s, vnet_interface_main_t * im,
 				vnet_sw_interface_t * si)
 {
diff --git a/src/vnet/interface_funcs.h b/src/vnet/interface_funcs.h
index 6e188f7f543..3f910f9d13f 100644
--- a/src/vnet/interface_funcs.h
+++ b/src/vnet/interface_funcs.h
@@ -46,6 +46,14 @@ vnet_get_hw_interface (vnet_main_t * vnm, u32 hw_if_index)
   return pool_elt_at_index (vnm->interface_main.hw_interfaces, hw_if_index);
 }
 
+always_inline vnet_hw_interface_t *
+vnet_get_hw_interface_safe (vnet_main_t * vnm, u32 hw_if_index)
+{
+  if (!pool_is_free_index (vnm->interface_main.hw_interfaces, hw_if_index))
+    return pool_elt_at_index (vnm->interface_main.hw_interfaces, hw_if_index);
+  return (NULL);
+}
+
 always_inline vnet_sw_interface_t *
 vnet_get_sw_interface (vnet_main_t * vnm, u32 sw_if_index)
 {
@@ -185,6 +193,19 @@ vnet_sw_interface_get_flags (vnet_main_t * vnm, u32 sw_if_index)
 }
 
 always_inline uword
+vnet_sw_interface_is_valid (vnet_main_t * vnm, u32 sw_if_index)
+{
+  return !pool_is_free_index (vnm->interface_main.sw_interfaces, sw_if_index);
+}
+
+always_inline uword
+vnet_hw_interface_is_valid (vnet_main_t * vnm, u32 hw_if_index)
+{
+  return !pool_is_free_index (vnm->interface_main.hw_interfaces, hw_if_index);
+}
+
+
+always_inline uword
 vnet_sw_interface_is_admin_up (vnet_main_t * vnm, u32 sw_if_index)
 {
   return (vnet_sw_interface_get_flags (vnm, sw_if_index) &
@@ -305,6 +326,7 @@ int vnet_sw_interface_stats_collect_enable_disable (u32 sw_if_index,
 /* Formats sw/hw interface. */
 format_function_t format_vnet_hw_interface;
 format_function_t format_vnet_hw_interface_rx_mode;
+format_function_t format_vnet_hw_if_index_name;
 format_function_t format_vnet_sw_interface;
 format_function_t format_vnet_sw_interface_name;
 format_function_t format_vnet_sw_interface_name_override;
diff --git a/src/vnet/ip/format.h b/src/vnet/ip/format.h
index d527e31a05e..9ebcf084be3 100644
--- a/src/vnet/ip/format.h
+++ b/src/vnet/ip/format.h
@@ -74,10 +74,13 @@ unformat_function_t unformat_ip46_address;
 
 /* Parse an IP4 address %d.%d.%d.%d. */
 unformat_function_t unformat_ip4_address;
+/* Parse an IP4 address and mask %d.%d.%d.%d/%d.%d.%d.%d */
+unformat_function_t unformat_ip4_address_and_mask;
 
 /* Format an IP4 address. */
 format_function_t format_ip4_address;
 format_function_t format_ip4_address_and_length;
+format_function_t format_ip4_address_and_mask;
 
 /* Parse an IP4 header. */
 unformat_function_t unformat_ip4_header;
@@ -92,8 +95,10 @@ unformat_function_t unformat_pg_ip4_header;
 
 /* IP6 */
 unformat_function_t unformat_ip6_address;
+unformat_function_t unformat_ip6_address_and_mask;
 format_function_t format_ip6_address;
 format_function_t format_ip6_address_and_length;
+format_function_t format_ip6_address_and_mask;
 unformat_function_t unformat_ip6_header;
 format_function_t format_ip6_header;
 unformat_function_t unformat_pg_ip6_header;
diff --git a/src/vnet/ip/ip4_format.c b/src/vnet/ip/ip4_format.c
index 3f2221022c7..0a2d7d293d5 100644
--- a/src/vnet/ip/ip4_format.c
+++ b/src/vnet/ip/ip4_format.c
@@ -56,6 +56,21 @@ format_ip4_address_and_length (u8 * s, va_list * args)
   return format (s, "%U/%d", format_ip4_address, a, l);
 }
 
+u8 *
+format_ip4_address_and_mask (u8 * s, va_list * args)
+{
+  ip4_address_and_mask_t *am = va_arg (*args, ip4_address_and_mask_t *);
+
+  if (am->addr.as_u32 == 0 && am->mask.as_u32 == 0)
+    return format (s, "any");
+
+  if (am->mask.as_u32 == ~0)
+    return format (s, "%U", format_ip4_address, &am->addr);
+
+  return format (s, "%U/%U", format_ip4_address, &am->addr,
+		 format_ip4_address, &am->mask);
+}
+
 /* Parse an IP4 address %d.%d.%d.%d. */
 uword
 unformat_ip4_address (unformat_input_t * input, va_list * args)
@@ -77,6 +92,27 @@ unformat_ip4_address (unformat_input_t * input, va_list * args)
   return 1;
 }
 
+uword
+unformat_ip4_address_and_mask (unformat_input_t * input, va_list * args)
+{
+  ip4_address_and_mask_t *am = va_arg (*args, ip4_address_and_mask_t *);
+  u32 addr = 0, mask = 0;
+
+  if (unformat (input, "any"))
+    ;
+  else if (unformat (input, "%U/%U", unformat_ip4_address, &addr,
+		     unformat_ip4_address, &mask))
+    ;
+  else if (unformat (input, "%U", unformat_ip4_address, &addr))
+    mask = ~0;
+  else
+    return 0;
+
+  am->addr.as_u32 = addr;
+  am->mask.as_u32 = mask;
+  return 1;
+}
+
 /* Format an IP4 header. */
 u8 *
 format_ip4_header (u8 * s, va_list * args)
diff --git a/src/vnet/ip/ip4_packet.h b/src/vnet/ip/ip4_packet.h
index 1ff9fbdba13..2f0c75e4924 100644
--- a/src/vnet/ip/ip4_packet.h
+++ b/src/vnet/ip/ip4_packet.h
@@ -77,6 +77,11 @@ typedef struct
   ip4_address_t src, dst;
 } ip4_address_pair_t;
 
+typedef struct
+{
+  ip4_address_t addr, mask;
+} ip4_address_and_mask_t;
+
 /* If address is a valid netmask, return length of mask. */
 always_inline uword
 ip4_address_netmask_length (ip4_address_t * a)
diff --git a/src/vnet/ip/ip6_format.c b/src/vnet/ip/ip6_format.c
index b7ae2ffae29..b1a8adf58de 100644
--- a/src/vnet/ip/ip6_format.c
+++ b/src/vnet/ip/ip6_format.c
@@ -117,6 +117,23 @@ format_ip6_address_and_length (u8 * s, va_list * args)
   return format (s, "%U/%d", format_ip6_address, a, l);
 }
 
+u8 *
+format_ip6_address_and_mask (u8 * s, va_list * args)
+{
+  ip6_address_and_mask_t *am = va_arg (*args, ip6_address_and_mask_t *);
+
+  if (am->addr.as_u64[0] == 0 && am->addr.as_u64[1] == 0 &&
+      am->mask.as_u64[0] == 0 && am->mask.as_u64[1] == 0)
+    return format (s, "any");
+
+  if (am->mask.as_u64[0] == ~0 && am->mask.as_u64[1] == ~0)
+    return format (s, "%U", format_ip4_address, &am->addr);
+
+  return format (s, "%U/%U", format_ip6_address, &am->addr,
+		 format_ip4_address, &am->mask);
+}
+
+
 /* Parse an IP6 address. */
 uword
 unformat_ip6_address (unformat_input_t * input, va_list * args)
@@ -212,6 +229,32 @@ unformat_ip6_address (unformat_input_t * input, va_list * args)
   }
 }
 
+uword
+unformat_ip6_address_and_mask (unformat_input_t * input, va_list * args)
+{
+  ip6_address_and_mask_t *am = va_arg (*args, ip6_address_and_mask_t *);
+  ip6_address_t addr, mask;
+
+  memset (&addr, 0, sizeof (ip6_address_t));
+  memset (&mask, 0, sizeof (ip6_address_t));
+
+  if (unformat (input, "any"))
+    ;
+  else if (unformat (input, "%U/%U", unformat_ip6_address, &addr,
+		     unformat_ip6_address, &mask))
+    ;
+  else if (unformat (input, "%U", unformat_ip6_address, &addr))
+    mask.as_u64[0] = mask.as_u64[1] = ~0;
+  else
+    return 0;
+
+  am->addr.as_u64[0] = addr.as_u64[0];
+  am->addr.as_u64[1] = addr.as_u64[1];
+  am->mask.as_u64[0] = mask.as_u64[0];
+  am->mask.as_u64[1] = mask.as_u64[1];
+  return 1;
+}
+
 /* Format an IP6 header. */
 u8 *
 format_ip6_header (u8 * s, va_list * args)
diff --git a/src/vnet/ip/ip6_packet.h b/src/vnet/ip/ip6_packet.h
index 70c3dab473d..b8f8d6e8657 100644
--- a/src/vnet/ip/ip6_packet.h
+++ b/src/vnet/ip/ip6_packet.h
@@ -53,6 +53,11 @@ typedef union
 }
 ip6_address_t;
 
+typedef struct
+{
+  ip6_address_t addr, mask;
+} ip6_address_and_mask_t;
+
 /* Packed so that the mhash key doesn't include uninitialized pad bytes */
 /* *INDENT-OFF* */
 typedef CLIB_PACKED (struct {