GBP: fixes for l3-out routing

Change-Id: I4d73b712da911588d511a8401b73cdc3c66346fe
Signed-off-by: Neale Ranns <nranns@cisco.com>

11 files changed, 395 insertions, 72 deletions

diff --git a/extras/vom/vom/vxlan_gbp_tunnel_cmds.cpp b/extras/vom/vom/vxlan_gbp_tunnel_cmds.cpp
index 24d1883ed75..db4e7fc796c 100644
--- a/extras/vom/vom/vxlan_gbp_tunnel_cmds.cpp
+++ b/extras/vom/vom/vxlan_gbp_tunnel_cmds.cpp
@@ -24,9 +24,11 @@ namespace vxlan_gbp_tunnel_cmds {
 create_cmd::create_cmd(HW::item<handle_t>& item,
                        const std::string& name,
                        const vxlan_tunnel::endpoint_t& ep,
+                       bool is_l2,
                        handle_t mcast_itf)
   : interface::create_cmd<vapi::Vxlan_gbp_tunnel_add_del>(item, name)
   , m_ep(ep)
+  , m_is_l2(is_l2)
   , m_mcast_itf(mcast_itf)
 {
 }
@@ -52,6 +54,8 @@ create_cmd::issue(connection& con)
   payload.tunnel.encap_table_id = 0;
   payload.tunnel.vni = m_ep.vni;
   payload.tunnel.instance = ~0;
+  payload.tunnel.mode =
+    (m_is_l2 ? VXLAN_GBP_API_TUNNEL_MODE_L2 : VXLAN_GBP_API_TUNNEL_MODE_L3);
 
   VAPI_CALL(req.execute());
 
diff --git a/extras/vom/vom/vxlan_gbp_tunnel_cmds.hpp b/extras/vom/vom/vxlan_gbp_tunnel_cmds.hpp
index c7ec87245df..87bca98d302 100644
--- a/extras/vom/vom/vxlan_gbp_tunnel_cmds.hpp
+++ b/extras/vom/vom/vxlan_gbp_tunnel_cmds.hpp
@@ -38,6 +38,7 @@ public:
   create_cmd(HW::item<handle_t>& item,
              const std::string& name,
              const vxlan_tunnel::endpoint_t& ep,
+             bool is_l2,
              handle_t mcast_itf);
 
   /**
@@ -59,6 +60,7 @@ private:
    * Enpoint values of the tunnel to be created
    */
   const vxlan_tunnel::endpoint_t m_ep;
+  bool m_is_l2;
   handle_t m_mcast_itf;
 };
 
diff --git a/extras/vom/vom/vxlan_tunnel.cpp b/extras/vom/vom/vxlan_tunnel.cpp
index 248aadfeade..db2886321c5 100644
--- a/extras/vom/vom/vxlan_tunnel.cpp
+++ b/extras/vom/vom/vxlan_tunnel.cpp
@@ -15,6 +15,7 @@
 
 #include "vom/vxlan_tunnel.hpp"
 #include "vom/api_types.hpp"
+#include "vom/interface_cmds.hpp"
 #include "vom/logger.hpp"
 #include "vom/singular_db_funcs.hpp"
 #include "vom/vxlan_gbp_tunnel_cmds.hpp"
@@ -26,8 +27,9 @@ const std::string VXLAN_TUNNEL_NAME = "vxlan-tunnel-itf";
 vxlan_tunnel::event_handler vxlan_tunnel::m_evh;
 
 const vxlan_tunnel::mode_t vxlan_tunnel::mode_t::STANDARD(0, "standard");
-const vxlan_tunnel::mode_t vxlan_tunnel::mode_t::GBP(1, "GBP");
-const vxlan_tunnel::mode_t vxlan_tunnel::mode_t::GPE(2, "GPE");
+const vxlan_tunnel::mode_t vxlan_tunnel::mode_t::GBP_L2(1, "GBP-L2");
+const vxlan_tunnel::mode_t vxlan_tunnel::mode_t::GBP_L3(2, "GBP-L3");
+const vxlan_tunnel::mode_t vxlan_tunnel::mode_t::GPE(3, "GPE");
 
 vxlan_tunnel::mode_t::mode_t(int v, const std::string s)
   : enum_base<vxlan_tunnel::mode_t>(v, s)
@@ -92,6 +94,8 @@ vxlan_tunnel::vxlan_tunnel(const boost::asio::ip::address& src,
   , m_tep(src, dst, vni)
   , m_mode(mode)
   , m_mcast_itf()
+  , m_rd()
+  , m_table_id(route::DEFAULT_TABLE)
 {
 }
 
@@ -106,6 +110,24 @@ vxlan_tunnel::vxlan_tunnel(const boost::asio::ip::address& src,
   , m_tep(src, dst, vni)
   , m_mode(mode)
   , m_mcast_itf(mcast_itf.singular())
+  , m_rd()
+  , m_table_id(route::DEFAULT_TABLE)
+{
+}
+
+vxlan_tunnel::vxlan_tunnel(const boost::asio::ip::address& src,
+                           const boost::asio::ip::address& dst,
+                           uint32_t vni,
+                           const route_domain& rd,
+                           const mode_t& mode)
+  : interface(mk_name(src, dst, mode, vni),
+              interface::type_t::VXLAN,
+              interface::admin_state_t::UP)
+  , m_tep(src, dst, vni)
+  , m_mode(mode)
+  , m_mcast_itf()
+  , m_rd(rd.singular())
+  , m_table_id(m_rd->table_id())
 {
 }
 
@@ -114,6 +136,8 @@ vxlan_tunnel::vxlan_tunnel(const vxlan_tunnel& o)
   , m_tep(o.m_tep)
   , m_mode(o.m_mode)
   , m_mcast_itf(o.m_mcast_itf)
+  , m_rd(o.m_rd)
+  , m_table_id(o.m_table_id)
 {
 }
 
@@ -142,7 +166,7 @@ vxlan_tunnel::sweep()
   if (m_hdl) {
     if (mode_t::STANDARD == m_mode)
       HW::enqueue(new vxlan_tunnel_cmds::delete_cmd(m_hdl, m_tep));
-    else if (mode_t::GBP == m_mode)
+    else if (mode_t::GBP_L2 == m_mode || mode_t::GBP_L3 == m_mode)
       HW::enqueue(new vxlan_gbp_tunnel_cmds::delete_cmd(m_hdl, m_tep));
   }
   HW::write();
@@ -156,11 +180,21 @@ vxlan_tunnel::replay()
       HW::enqueue(new vxlan_tunnel_cmds::create_cmd(
         m_hdl, name(), m_tep,
         (m_mcast_itf ? m_mcast_itf->handle() : handle_t::INVALID)));
-    else if (mode_t::GBP == m_mode)
+    else if (mode_t::GBP_L2 == m_mode)
       HW::enqueue(new vxlan_gbp_tunnel_cmds::create_cmd(
-        m_hdl, name(), m_tep,
+        m_hdl, name(), m_tep, true,
+        (m_mcast_itf ? m_mcast_itf->handle() : handle_t::INVALID)));
+    else if (mode_t::GBP_L3 == m_mode)
+      HW::enqueue(new vxlan_gbp_tunnel_cmds::create_cmd(
+        m_hdl, name(), m_tep, false,
         (m_mcast_itf ? m_mcast_itf->handle() : handle_t::INVALID)));
   }
+  if (m_rd && (m_rd->table_id() != route::DEFAULT_TABLE)) {
+    HW::enqueue(
+      new interface_cmds::set_table_cmd(m_table_id, l3_proto_t::IPV4, m_hdl));
+    HW::enqueue(
+      new interface_cmds::set_table_cmd(m_table_id, l3_proto_t::IPV6, m_hdl));
+  }
 }
 
 vxlan_tunnel::~vxlan_tunnel()
@@ -192,10 +226,20 @@ vxlan_tunnel::update(const vxlan_tunnel& desired)
       HW::enqueue(new vxlan_tunnel_cmds::create_cmd(
         m_hdl, name(), m_tep,
         (m_mcast_itf ? m_mcast_itf->handle() : handle_t::INVALID)));
-    else if (mode_t::GBP == m_mode)
+    else if (mode_t::GBP_L2 == m_mode)
       HW::enqueue(new vxlan_gbp_tunnel_cmds::create_cmd(
-        m_hdl, name(), m_tep,
+        m_hdl, name(), m_tep, true,
         (m_mcast_itf ? m_mcast_itf->handle() : handle_t::INVALID)));
+    else if (mode_t::GBP_L3 == m_mode)
+      HW::enqueue(new vxlan_gbp_tunnel_cmds::create_cmd(
+        m_hdl, name(), m_tep, false,
+        (m_mcast_itf ? m_mcast_itf->handle() : handle_t::INVALID)));
+  }
+  if (!m_table_id && m_rd) {
+    HW::enqueue(
+      new interface_cmds::set_table_cmd(m_table_id, l3_proto_t::IPV4, m_hdl));
+    HW::enqueue(
+      new interface_cmds::set_table_cmd(m_table_id, l3_proto_t::IPV6, m_hdl));
   }
 }
 
@@ -255,7 +299,11 @@ vxlan_tunnel::event_handler::handle_populate(const client_db::key_t& key)
       boost::asio::ip::address dst = from_api(payload.tunnel.dst);
 
       std::shared_ptr<vxlan_tunnel> vt =
-        vxlan_tunnel(src, dst, payload.tunnel.vni, mode_t::GBP).singular();
+        vxlan_tunnel(src, dst, payload.tunnel.vni,
+                     (payload.tunnel.mode == VXLAN_GBP_API_TUNNEL_MODE_L2
+                        ? mode_t::GBP_L2
+                        : mode_t::GBP_L3))
+          .singular();
       vt->set(hdl);
 
       VOM_LOG(log_level_t::DEBUG) << "dump: " << vt->to_string();
diff --git a/extras/vom/vom/vxlan_tunnel.hpp b/extras/vom/vom/vxlan_tunnel.hpp
index c085ba0b629..4c46e75a700 100644
--- a/extras/vom/vom/vxlan_tunnel.hpp
+++ b/extras/vom/vom/vxlan_tunnel.hpp
@@ -82,7 +82,8 @@ public:
   {
     ~mode_t() = default;
     const static mode_t STANDARD;
-    const static mode_t GBP;
+    const static mode_t GBP_L2;
+    const static mode_t GBP_L3;
     const static mode_t GPE;
 
   private:
@@ -102,6 +103,11 @@ public:
                uint32_t vni,
                const interface& mcast_itf,
                const mode_t& mode = mode_t::STANDARD);
+  vxlan_tunnel(const boost::asio::ip::address& src,
+               const boost::asio::ip::address& dst,
+               uint32_t vni,
+               const route_domain& rd,
+               const mode_t& mode = mode_t::STANDARD);
 
   /*
    * Destructor
@@ -227,6 +233,16 @@ private:
   std::shared_ptr<interface> m_mcast_itf;
 
   /**
+   * The RD an L3 interface is bound to
+   */
+  std::shared_ptr<const route_domain> m_rd;
+
+  /**
+   * HW state of the VPP table mapping
+   */
+  HW::item<route::table_id_t> m_table_id;
+
+  /**
    * Construct a unique name for the tunnel
    */
   static std::string mk_name(const boost::asio::ip::address& src,
diff --git a/src/plugins/gbp/gbp_classify_node.c b/src/plugins/gbp/gbp_classify_node.c
index 9d9e29303e6..1179076a4c7 100644
--- a/src/plugins/gbp/gbp_classify_node.c
+++ b/src/plugins/gbp/gbp_classify_node.c
@@ -516,6 +516,10 @@ gbp_lpm_classify_inline (vlib_main_t * vm,
 	  lb0 = load_balance_get (lbi0);
 	  dpo0 = load_balance_get_bucket_i (lb0, 0);
 
+	  /* all packets from an external network should not be learned by the
+	   * reciever. so set the Do-not-learn bit here */
+	  vnet_buffer2 (b0)->gbp.flags = VXLAN_GBP_GPFLAGS_D;
+
 	  if (gbp_policy_dpo_type == dpo0->dpoi_type)
 	    {
 	      gpd0 = gbp_policy_dpo_get (dpo0->dpoi_index);
diff --git a/src/plugins/gbp/gbp_policy_dpo.c b/src/plugins/gbp/gbp_policy_dpo.c
index c52dcc4dc48..5fb04ff4df5 100644
--- a/src/plugins/gbp/gbp_policy_dpo.c
+++ b/src/plugins/gbp/gbp_policy_dpo.c
@@ -211,10 +211,11 @@ VLIB_INIT_FUNCTION (gbp_policy_dpo_module_init);
 
 typedef struct gbp_policy_dpo_trace_t_
 {
-  u32 src_epg;
-  u32 dst_epg;
+  u32 sclass;
+  u32 dclass;
   u32 acl_index;
   u32 a_bit;
+  u32 action;
 } gbp_policy_dpo_trace_t;
 
 typedef enum
@@ -268,7 +269,9 @@ gbp_policy_dpo_inline (vlib_main_t * vm,
 	  gbp_contract_t *gc0;
 	  vlib_buffer_t *b0;
 	  index_t gci0;
+	  u8 action0;
 
+	  action0 = 0;
 	  bi0 = from[0];
 	  to_next[0] = bi0;
 	  from += 1;
@@ -302,6 +305,7 @@ gbp_policy_dpo_inline (vlib_main_t * vm,
 		   */
 		  next0 = gpd0->gpd_dpo.dpoi_next_node;
 		  vnet_buffer2 (b0)->gbp.flags |= VXLAN_GBP_GPFLAGS_A;
+		  action0 = 0;
 		}
 	      else
 		{
@@ -310,7 +314,6 @@ gbp_policy_dpo_inline (vlib_main_t * vm,
 		  if (INDEX_INVALID != gci0)
 		    {
 		      fa_5tuple_opaque_t pkt_5tuple0;
-		      u8 action0 = 0;
 		      u32 acl_pos_p0, acl_match_p0;
 		      u32 rule_match_p0, trace_bitmap0;
 		      /*
@@ -335,9 +338,9 @@ gbp_policy_dpo_inline (vlib_main_t * vm,
 
 		      if (action0 > 0)
 			{
-
 			  vnet_buffer2 (b0)->gbp.flags |= VXLAN_GBP_GPFLAGS_A;
 			  gu = gbp_rule_get (gc0->gc_rules[rule_match_p0]);
+			  action0 = gu->gu_action;
 
 			  switch (gu->gu_action)
 			    {
@@ -369,10 +372,11 @@ gbp_policy_dpo_inline (vlib_main_t * vm,
 	      gbp_policy_dpo_trace_t *tr;
 
 	      tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
-	      tr->src_epg = key0.gck_src;
-	      tr->dst_epg = key0.gck_dst;
+	      tr->sclass = key0.gck_src;
+	      tr->dclass = key0.gck_dst;
 	      tr->acl_index = (gc0 ? gc0->gc_acl_index : ~0);
 	      tr->a_bit = vnet_buffer2 (b0)->gbp.flags & VXLAN_GBP_GPFLAGS_A;
+	      tr->action = action0;
 	    }
 
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
@@ -390,8 +394,8 @@ format_gbp_policy_dpo_trace (u8 * s, va_list * args)
   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
   gbp_policy_dpo_trace_t *t = va_arg (*args, gbp_policy_dpo_trace_t *);
 
-  s = format (s, " src-epg:%d dst-epg:%d acl-index:%d a-bit:%d",
-	      t->src_epg, t->dst_epg, t->acl_index, t->a_bit);
+  s = format (s, " sclass:%d dclass:%d acl-index:%d a-bit:%d action:%d",
+	      t->sclass, t->dclass, t->acl_index, t->a_bit, t->action);
 
   return s;
 }
diff --git a/src/vnet/vxlan-gbp/vxlan_gbp.api b/src/vnet/vxlan-gbp/vxlan_gbp.api
index 3e213ddc563..13ec50395bc 100644
--- a/src/vnet/vxlan-gbp/vxlan_gbp.api
+++ b/src/vnet/vxlan-gbp/vxlan_gbp.api
@@ -17,6 +17,12 @@
 option version = "1.1.0";
 import "vnet/ip/ip_types.api";
 
+enum vxlan_gbp_api_tunnel_mode
+{
+  VXLAN_GBP_API_TUNNEL_MODE_L2,
+  VXLAN_GBP_API_TUNNEL_MODE_L3,
+};
+
 /** \brief Definition of a VXLAN GBP tunnel
     @param instance - optional unique custom device instance, else ~0.
     @param src - Source IP address
@@ -35,6 +41,7 @@ typedef vxlan_gbp_tunnel
   u32 encap_table_id;
   u32 vni;
   u32 sw_if_index;
+  vl_api_vxlan_gbp_api_tunnel_mode_t mode;
 };
 
 /** \brief Create or delete a VXLAN-GBP tunnel
diff --git a/src/vnet/vxlan-gbp/vxlan_gbp_api.c b/src/vnet/vxlan-gbp/vxlan_gbp_api.c
index f5e97e5a364..6a87d4dda52 100644
--- a/src/vnet/vxlan-gbp/vxlan_gbp_api.c
+++ b/src/vnet/vxlan-gbp/vxlan_gbp_api.c
@@ -66,10 +66,29 @@ static void
   REPLY_MACRO (VL_API_SW_INTERFACE_SET_VXLAN_GBP_BYPASS_REPLY);
 }
 
+static int
+vxlan_gbp_tunnel_mode_decode (vl_api_vxlan_gbp_api_tunnel_mode_t in,
+			      vxlan_gbp_tunnel_mode_t * out)
+{
+  in = clib_net_to_host_u32 (in);
+
+  switch (in)
+    {
+    case VXLAN_GBP_API_TUNNEL_MODE_L2:
+      *out = VXLAN_GBP_TUNNEL_MODE_L2;
+      return (0);
+    case VXLAN_GBP_API_TUNNEL_MODE_L3:
+      *out = VXLAN_GBP_TUNNEL_MODE_L3;
+      return (0);
+    }
+  return (1);
+}
+
 static void vl_api_vxlan_gbp_tunnel_add_del_t_handler
   (vl_api_vxlan_gbp_tunnel_add_del_t * mp)
 {
   vl_api_vxlan_gbp_tunnel_add_del_reply_t *rmp;
+  vxlan_gbp_tunnel_mode_t mode;
   ip46_address_t src, dst;
   ip46_type_t itype;
   int rv = 0;
@@ -86,6 +105,11 @@ static void vl_api_vxlan_gbp_tunnel_add_del_t_handler
       goto out;
     }
 
+  rv = vxlan_gbp_tunnel_mode_decode (mp->tunnel.mode, &mode);
+
+  if (rv)
+    goto out;
+
   vnet_vxlan_gbp_tunnel_add_del_args_t a = {
     .is_add = mp->is_add,
     .is_ip6 = (itype == IP46_TYPE_IP6),
@@ -95,7 +119,7 @@ static void vl_api_vxlan_gbp_tunnel_add_del_t_handler
     .vni = ntohl (mp->tunnel.vni),
     .dst = dst,
     .src = src,
-    .mode = VXLAN_GBP_TUNNEL_MODE_L2,
+    .mode = mode,
   };
 
   /* Check src & dst are different */
diff --git a/test/test_gbp.py b/test/test_gbp.py
index 3fddde01fe6..1eedf54f433 100644
--- a/test/test_gbp.py
+++ b/test/test_gbp.py
@@ -26,6 +26,7 @@ from vpp_papi import VppEnum, MACAddress
 from vpp_papi_provider import L2_VTR_OP
 from vpp_vxlan_gbp_tunnel import find_vxlan_gbp_tunnel, INDEX_INVALID, \
     VppVxlanGbpTunnel
+from vpp_neighbor import VppNeighbor
 
 
 def find_gbp_endpoint(test, sw_if_index=None, ip=None, mac=None):
@@ -230,13 +231,13 @@ class VppGbpSubnet(VppObject):
     GBP Subnet
     """
     def __init__(self, test, rd, address, address_len,
-                 type, sw_if_index=None, epg=None):
+                 type, sw_if_index=None, sclass=None):
         self._test = test
         self.rd_id = rd.rd_id
         self.prefix = VppIpPrefix(address, address_len)
         self.type = type
         self.sw_if_index = sw_if_index
-        self.epg = epg
+        self.sclass = sclass
 
     def add_vpp_config(self):
         self._test.vapi.gbp_subnet_add_del(
@@ -245,7 +246,7 @@ class VppGbpSubnet(VppObject):
             self.prefix.encode(),
             self.type,
             sw_if_index=self.sw_if_index if self.sw_if_index else 0xffffffff,
-            sclass=self.epg.sclass if self.epg else 0xffff)
+            sclass=self.sclass if self.sclass else 0xffff)
         self._test.registry.register(self, self._test.logger)
 
     def remove_vpp_config(self):
@@ -449,12 +450,12 @@ class VppGbpContract(VppObject):
     GBP Contract
     """
 
-    def __init__(self, test, src_epg, dst_epg, acl_index,
+    def __init__(self, test, sclass, dclass, acl_index,
                  rules, allowed_ethertypes):
         self._test = test
         self.acl_index = acl_index
-        self.src_epg = src_epg
-        self.dst_epg = dst_epg
+        self.sclass = sclass
+        self.dclass = dclass
         self.rules = rules
         self.allowed_ethertypes = allowed_ethertypes
         while (len(self.allowed_ethertypes) < 16):
@@ -466,8 +467,8 @@ class VppGbpContract(VppObject):
             rules.append(r.encode())
         self._test.vapi.gbp_contract_add_del(
             1,
-            self.src_epg.sclass,
-            self.dst_epg.sclass,
+            self.sclass,
+            self.dclass,
             self.acl_index,
             rules,
             self.allowed_ethertypes)
@@ -476,8 +477,8 @@ class VppGbpContract(VppObject):
     def remove_vpp_config(self):
         self._test.vapi.gbp_contract_add_del(
             0,
-            self.src_epg.sclass,
-            self.dst_epg.sclass,
+            self.sclass,
+            self.dclass,
             self.acl_index,
             [],
             self.allowed_ethertypes)
@@ -486,15 +487,15 @@ class VppGbpContract(VppObject):
         return self.object_id()
 
     def object_id(self):
-        return "gbp-contract:[%d:%s:%d]" % (self.src_epg.sclass,
-                                            self.dst_epg.sclass,
+        return "gbp-contract:[%d:%s:%d]" % (self.sclass,
+                                            self.dclass,
                                             self.acl_index)
 
     def query_vpp_config(self):
         cs = self._test.vapi.gbp_contract_dump()
         for c in cs:
-            if c.contract.sclass == self.src_epg.sclass \
-               and c.contract.dclass == self.dst_epg.sclass:
+            if c.contract.sclass == self.sclass \
+               and c.contract.dclass == self.dclass:
                 return True
         return False
 
@@ -1143,7 +1144,7 @@ class TestGBP(VppTestCase):
         rule2 = acl.create_rule(is_ipv6=1, permit_deny=1, proto=17)
         acl_index = acl.add_vpp_config([rule, rule2])
         c1 = VppGbpContract(
-            self, epgs[0], epgs[1], acl_index,
+            self, epgs[0].sclass, epgs[1].sclass, acl_index,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
                 []),
@@ -1163,7 +1164,7 @@ class TestGBP(VppTestCase):
         # contract for the return direction
         #
         c2 = VppGbpContract(
-            self, epgs[1], epgs[0], acl_index,
+            self, epgs[1].sclass, epgs[0].sclass, acl_index,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
                 []),
@@ -1200,7 +1201,7 @@ class TestGBP(VppTestCase):
         # A uni-directional contract from EPG 220 -> 222 'L3 routed'
         #
         c3 = VppGbpContract(
-            self, epgs[0], epgs[2], acl_index,
+            self, epgs[0].sclass, epgs[2].sclass, acl_index,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
                 []),
@@ -1242,33 +1243,33 @@ class TestGBP(VppTestCase):
             self, rd0, "0.0.0.0", 0,
             VppEnum.vl_api_gbp_subnet_type_t.GBP_API_SUBNET_STITCHED_EXTERNAL,
             sw_if_index=recirc_nat.recirc.sw_if_index,
-            epg=epg_nat)
+            sclass=epg_nat.sclass)
         se2 = VppGbpSubnet(
             self, rd0, "11.0.0.0", 8,
             VppEnum.vl_api_gbp_subnet_type_t.GBP_API_SUBNET_STITCHED_EXTERNAL,
             sw_if_index=recirc_nat.recirc.sw_if_index,
-            epg=epg_nat)
+            sclass=epg_nat.sclass)
         se16 = VppGbpSubnet(
             self, rd0, "::", 0,
             VppEnum.vl_api_gbp_subnet_type_t.GBP_API_SUBNET_STITCHED_EXTERNAL,
             sw_if_index=recirc_nat.recirc.sw_if_index,
-            epg=epg_nat)
+            sclass=epg_nat.sclass)
         # in the NAT RD an external subnet via the NAT EPG's uplink
         se3 = VppGbpSubnet(
             self, rd20, "0.0.0.0", 0,
             VppEnum.vl_api_gbp_subnet_type_t.GBP_API_SUBNET_STITCHED_EXTERNAL,
             sw_if_index=epg_nat.uplink.sw_if_index,
-            epg=epg_nat)
+            sclass=epg_nat.sclass)
         se36 = VppGbpSubnet(
             self, rd20, "::", 0,
             VppEnum.vl_api_gbp_subnet_type_t.GBP_API_SUBNET_STITCHED_EXTERNAL,
             sw_if_index=epg_nat.uplink.sw_if_index,
-            epg=epg_nat)
+            sclass=epg_nat.sclass)
         se4 = VppGbpSubnet(
             self, rd20, "11.0.0.0", 8,
             VppEnum.vl_api_gbp_subnet_type_t.GBP_API_SUBNET_STITCHED_EXTERNAL,
             sw_if_index=epg_nat.uplink.sw_if_index,
-            epg=epg_nat)
+            sclass=epg_nat.sclass)
         se1.add_vpp_config()
         se2.add_vpp_config()
         se16.add_vpp_config()
@@ -1305,7 +1306,7 @@ class TestGBP(VppTestCase):
 
         acl_index2 = acl2.add_vpp_config([rule, rule2])
         c4 = VppGbpContract(
-            self, epgs[0], epgs[3], acl_index2,
+            self, epgs[0].sclass, epgs[3].sclass, acl_index2,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
                 []),
@@ -1346,7 +1347,7 @@ class TestGBP(VppTestCase):
                                         pkt_inter_epg_220_from_global * 65)
 
         c5 = VppGbpContract(
-            self, epgs[3], epgs[0], acl_index2,
+            self, epgs[3].sclass, epgs[0].sclass, acl_index2,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
                 []),
@@ -1849,7 +1850,7 @@ class TestGBP(VppTestCase):
         rule2 = acl.create_rule(is_ipv6=1, permit_deny=1, proto=17)
         acl_index = acl.add_vpp_config([rule, rule2])
         c1 = VppGbpContract(
-            self, epg_220, epg_330, acl_index,
+            self, epg_220.sclass, epg_330.sclass, acl_index,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
                 []),
@@ -2744,7 +2745,7 @@ class TestGBP(VppTestCase):
         # test the src-ip hash mode
         #
         c1 = VppGbpContract(
-            self, epg_220, epg_222, acl_index,
+            self, epg_220.sclass, epg_222.sclass, acl_index,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_REDIRECT,
                 VppEnum.vl_api_gbp_hash_mode_t.GBP_API_HASH_MODE_SRC_IP,
@@ -2763,7 +2764,7 @@ class TestGBP(VppTestCase):
         c1.add_vpp_config()
 
         c2 = VppGbpContract(
-            self, epg_222, epg_220, acl_index,
+            self, epg_222.sclass, epg_220.sclass, acl_index,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_REDIRECT,
                 VppEnum.vl_api_gbp_hash_mode_t.GBP_API_HASH_MODE_SRC_IP,
@@ -2876,7 +2877,7 @@ class TestGBP(VppTestCase):
         # test the symmetric hash mode
         #
         c1 = VppGbpContract(
-            self, epg_220, epg_222, acl_index,
+            self, epg_220.sclass, epg_222.sclass, acl_index,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_REDIRECT,
                 VppEnum.vl_api_gbp_hash_mode_t.GBP_API_HASH_MODE_SYMMETRIC,
@@ -2895,7 +2896,7 @@ class TestGBP(VppTestCase):
         c1.add_vpp_config()
 
         c2 = VppGbpContract(
-            self, epg_222, epg_220, acl_index,
+            self, epg_222.sclass, epg_220.sclass, acl_index,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_REDIRECT,
                 VppEnum.vl_api_gbp_hash_mode_t.GBP_API_HASH_MODE_SYMMETRIC,
@@ -2960,7 +2961,7 @@ class TestGBP(VppTestCase):
                Raw('\xa5' * 100))]
 
         c3 = VppGbpContract(
-            self, epg_220, epg_221, acl_index,
+            self, epg_220.sclass, epg_221.sclass, acl_index,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_REDIRECT,
                 VppEnum.vl_api_gbp_hash_mode_t.GBP_API_HASH_MODE_SYMMETRIC,
@@ -2996,7 +2997,7 @@ class TestGBP(VppTestCase):
         vx_tun_l3.add_vpp_config()
 
         c4 = VppGbpContract(
-            self, epg_221, epg_220, acl_index,
+            self, epg_221.sclass, epg_220.sclass, acl_index,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
                 []),
@@ -3074,7 +3075,7 @@ class TestGBP(VppTestCase):
         # test the dst-ip hash mode
         #
         c5 = VppGbpContract(
-            self, epg_220, epg_221, acl_index,
+            self, epg_220.sclass, epg_221.sclass, acl_index,
             [VppGbpContractRule(
                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_REDIRECT,
                 VppEnum.vl_api_gbp_hash_mode_t.GBP_API_HASH_MODE_DST_IP,
@@ -3184,7 +3185,7 @@ class TestGBP(VppTestCase):
         l3o_1 = VppGbpSubnet(
             self, rd1, "10.0.0.0", 24,
             VppEnum.vl_api_gbp_subnet_type_t.GBP_API_SUBNET_L3_OUT,
-            epg=epg_220)
+            sclass=113)
         l3o_1.add_vpp_config()
 
         #
@@ -3227,7 +3228,7 @@ class TestGBP(VppTestCase):
         eep2.add_vpp_config()
 
         #
-        # A remote endpoint
+        # A remote external endpoint
         #
         rep = VppGbpEndpoint(self, vx_tun_l3,
                              epg_220, None,
@@ -3330,7 +3331,7 @@ class TestGBP(VppTestCase):
             self.assertEqual(rx[Dot1Q].vlan, 101)
 
         #
-        # A subnet reachable through the external EP
+        # A subnet reachable through the external EP1
         #
         ip_220 = VppIpRoute(self, "10.220.0.0", 24,
                             [VppRoutePath(eep1.ip4.address,
@@ -3341,24 +3342,11 @@ class TestGBP(VppTestCase):
         l3o_220 = VppGbpSubnet(
             self, rd1, "10.220.0.0", 24,
             VppEnum.vl_api_gbp_subnet_type_t.GBP_API_SUBNET_L3_OUT,
-            epg=epg_220)
+            sclass=4220)
         l3o_220.add_vpp_config()
 
-        p = (Ether(src=self.pg7.remote_mac,
-                   dst=self.pg7.local_mac) /
-             IP(src=self.pg7.remote_ip4,
-                dst=self.pg7.local_ip4) /
-             UDP(sport=1234, dport=48879) /
-             VXLAN(vni=444, gpid=113, flags=0x88) /
-             Ether(src=self.pg0.remote_mac, dst=str(self.router_mac)) /
-             IP(src="10.0.0.101", dst="10.220.0.1") /
-             UDP(sport=1234, dport=1234) /
-             Raw('\xa5' * 100))
-
-        rxs = self.send_and_expect(self.pg7, p * 1, self.pg0)
-
         #
-        # A subnet reachable through the external EP
+        # A subnet reachable through the external EP2
         #
         ip_221 = VppIpRoute(self, "10.221.0.0", 24,
                             [VppRoutePath(eep2.ip4.address,
@@ -3369,17 +3357,72 @@ class TestGBP(VppTestCase):
         l3o_221 = VppGbpSubnet(
             self, rd1, "10.221.0.0", 24,
             VppEnum.vl_api_gbp_subnet_type_t.GBP_API_SUBNET_L3_OUT,
-            epg=epg_220)
+            sclass=4221)
         l3o_221.add_vpp_config()
 
         #
         # ping between hosts in remote subnets
+        #  dropped without a contract
         #
         p = (Ether(src=eep1.mac, dst=str(self.router_mac)) /
              Dot1Q(vlan=100) /
              IP(src="10.220.0.1", dst="10.221.0.1") /
              ICMP(type='echo-request'))
 
+        rxs = self.send_and_assert_no_replies(self.pg0, p * 1)
+
+        #
+        # contract for the external nets to communicate
+        #
+        acl = VppGbpAcl(self)
+        rule4 = acl.create_rule(permit_deny=1, proto=17)
+        rule6 = acl.create_rule(is_ipv6=1, permit_deny=1, proto=17)
+        acl_index = acl.add_vpp_config([rule4, rule6])
+
+        c1 = VppGbpContract(
+            self, 4220, 4221, acl_index,
+            [VppGbpContractRule(
+                VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
+                []),
+             VppGbpContractRule(
+                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
+                 [])],
+            [ETH_P_IP, ETH_P_IPV6])
+        c1.add_vpp_config()
+
+        #
+        # Contracts allowing ext-net 200 to talk with external EPs
+        #
+        c2 = VppGbpContract(
+            self, 4220, 113, acl_index,
+            [VppGbpContractRule(
+                VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
+                []),
+             VppGbpContractRule(
+                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
+                 [])],
+            [ETH_P_IP, ETH_P_IPV6])
+        c2.add_vpp_config()
+        c3 = VppGbpContract(
+            self, 113, 4220, acl_index,
+            [VppGbpContractRule(
+                VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
+                []),
+             VppGbpContractRule(
+                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
+                 [])],
+            [ETH_P_IP, ETH_P_IPV6])
+        c3.add_vpp_config()
+
+        #
+        # ping between hosts in remote subnets
+        #
+        p = (Ether(src=eep1.mac, dst=str(self.router_mac)) /
+             Dot1Q(vlan=100) /
+             IP(src="10.220.0.1", dst="10.221.0.1") /
+             UDP(sport=1234, dport=1234) /
+             Raw('\xa5' * 100))
+
         rxs = self.send_and_expect(self.pg0, p * 1, self.pg0)
 
         for rx in rxs:
@@ -3387,6 +3430,168 @@ class TestGBP(VppTestCase):
             self.assertEqual(rx[Ether].dst, eep2.mac)
             self.assertEqual(rx[Dot1Q].vlan, 101)
 
+        # we did not learn these external hosts
+        self.assertFalse(find_gbp_endpoint(self, ip="10.220.0.1"))
+        self.assertFalse(find_gbp_endpoint(self, ip="10.221.0.1"))
+
+        #
+        # from remote external EP to local external EP
+        #
+        p = (Ether(src=self.pg7.remote_mac,
+                   dst=self.pg7.local_mac) /
+             IP(src=self.pg7.remote_ip4,
+                dst=self.pg7.local_ip4) /
+             UDP(sport=1234, dport=48879) /
+             VXLAN(vni=444, gpid=113, flags=0x88) /
+             Ether(src=self.pg0.remote_mac, dst=str(self.router_mac)) /
+             IP(src="10.0.0.101", dst="10.220.0.1") /
+             UDP(sport=1234, dport=1234) /
+             Raw('\xa5' * 100))
+
+        rxs = self.send_and_expect(self.pg7, p * 1, self.pg0)
+
+        #
+        # ping from an external host to the remote external EP
+        #
+        p = (Ether(src=eep1.mac, dst=str(self.router_mac)) /
+             Dot1Q(vlan=100) /
+             IP(src="10.220.0.1", dst=rep.ip4.address) /
+             UDP(sport=1234, dport=1234) /
+             Raw('\xa5' * 100))
+
+        rxs = self.send_and_expect(self.pg0, p * 1, self.pg7)
+
+        for rx in rxs:
+            self.assertEqual(rx[Ether].src, self.pg7.local_mac)
+            # self.assertEqual(rx[Ether].dst, self.pg7.remote_mac)
+            self.assertEqual(rx[IP].src, self.pg7.local_ip4)
+            self.assertEqual(rx[IP].dst, self.pg7.remote_ip4)
+            self.assertEqual(rx[VXLAN].vni, 444)
+            self.assertTrue(rx[VXLAN].flags.G)
+            self.assertTrue(rx[VXLAN].flags.Instance)
+            # the sclass of the ext-net the packet came from
+            self.assertEqual(rx[VXLAN].gpid, 4220)
+            # policy was applied to the original IP packet
+            self.assertTrue(rx[VXLAN].gpflags.A)
+            # since it's an external host the reciever should not learn it
+            self.assertTrue(rx[VXLAN].gpflags.D)
+            inner = rx[VXLAN].payload
+            self.assertEqual(inner[IP].src, "10.220.0.1")
+            self.assertEqual(inner[IP].dst, rep.ip4.address)
+
+        #
+        # An external subnet reachable via the remote external EP
+        #
+
+        #
+        # first the VXLAN-GBP tunnel over which it is reached
+        #
+        vx_tun_r = VppVxlanGbpTunnel(
+            self, self.pg7.local_ip4,
+            self.pg7.remote_ip4, 445,
+            mode=(VppEnum.vl_api_vxlan_gbp_api_tunnel_mode_t.
+                  VXLAN_GBP_API_TUNNEL_MODE_L3))
+        vx_tun_r.add_vpp_config()
+        VppIpInterfaceBind(self, vx_tun_r, t4).add_vpp_config()
+
+        self.logger.info(self.vapi.cli("sh vxlan-gbp tunnel"))
+
+        #
+        # then the special adj to resolve through on that tunnel
+        #
+        n1 = VppNeighbor(self,
+                         vx_tun_r.sw_if_index,
+                         "00:0c:0c:0c:0c:0c",
+                         self.pg7.remote_ip4)
+        n1.add_vpp_config()
+
+        #
+        # the route via the adj above
+        #
+        ip_222 = VppIpRoute(self, "10.222.0.0", 24,
+                            [VppRoutePath(self.pg7.remote_ip4,
+                                          vx_tun_r.sw_if_index)],
+                            table_id=t4.table_id)
+        ip_222.add_vpp_config()
+
+        l3o_222 = VppGbpSubnet(
+            self, rd1, "10.222.0.0", 24,
+            VppEnum.vl_api_gbp_subnet_type_t.GBP_API_SUBNET_L3_OUT,
+            sclass=4222)
+        l3o_222.add_vpp_config()
+
+        #
+        # ping between hosts in local and remote external subnets
+        #  dropped without a contract
+        #
+        p = (Ether(src=eep1.mac, dst=str(self.router_mac)) /
+             Dot1Q(vlan=100) /
+             IP(src="10.220.0.1", dst="10.222.0.1") /
+             UDP(sport=1234, dport=1234) /
+             Raw('\xa5' * 100))
+
+        rxs = self.send_and_assert_no_replies(self.pg0, p * 1)
+
+        #
+        # Add contracts ext-nets for 220 -> 222
+        #
+        c4 = VppGbpContract(
+            self, 4220, 4222, acl_index,
+            [VppGbpContractRule(
+                VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
+                []),
+             VppGbpContractRule(
+                 VppEnum.vl_api_gbp_rule_action_t.GBP_API_RULE_PERMIT,
+                 [])],
+            [ETH_P_IP, ETH_P_IPV6])
+        c4.add_vpp_config()
+
+        #
+        # ping from host in local to remote external subnets
+        #
+        p = (Ether(src=eep1.mac, dst=str(self.router_mac)) /
+             Dot1Q(vlan=100) /
+             IP(src="10.220.0.1", dst="10.222.0.1") /
+             UDP(sport=1234, dport=1234) /
+             Raw('\xa5' * 100))
+
+        rxs = self.send_and_expect(self.pg0, p * 3, self.pg7)
+
+        for rx in rxs:
+            self.assertEqual(rx[Ether].src, self.pg7.local_mac)
+            self.assertEqual(rx[Ether].dst, self.pg7.remote_mac)
+            self.assertEqual(rx[IP].src, self.pg7.local_ip4)
+            self.assertEqual(rx[IP].dst, self.pg7.remote_ip4)
+            self.assertEqual(rx[VXLAN].vni, 445)
+            self.assertTrue(rx[VXLAN].flags.G)
+            self.assertTrue(rx[VXLAN].flags.Instance)
+            # the sclass of the ext-net the packet came from
+            self.assertEqual(rx[VXLAN].gpid, 4220)
+            # policy was applied to the original IP packet
+            self.assertTrue(rx[VXLAN].gpflags.A)
+            # since it's an external host the reciever should not learn it
+            self.assertTrue(rx[VXLAN].gpflags.D)
+            inner = rx[VXLAN].payload
+            self.assertEqual(inner[Ether].dst, "00:0c:0c:0c:0c:0c")
+            self.assertEqual(inner[IP].src, "10.220.0.1")
+            self.assertEqual(inner[IP].dst, "10.222.0.1")
+
+        #
+        # ping from host in remote to local external subnets
+        # there's no contract for this, but the A bit is set.
+        #
+        p = (Ether(src=self.pg7.remote_mac, dst=self.pg7.local_mac) /
+             IP(src=self.pg7.remote_ip4, dst=self.pg7.local_ip4) /
+             UDP(sport=1234, dport=48879) /
+             VXLAN(vni=445, gpid=4222, flags=0x88, gpflags='A') /
+             Ether(src=self.pg0.remote_mac, dst=str(self.router_mac)) /
+             IP(src="10.222.0.1", dst="10.220.0.1") /
+             UDP(sport=1234, dport=1234) /
+             Raw('\xa5' * 100))
+
+        rxs = self.send_and_expect(self.pg7, p * 3, self.pg0)
+        self.assertFalse(find_gbp_endpoint(self, ip="10.222.0.1"))
+
         #
         # cleanup
         #
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index 5e7d6df42b7..1240647582f 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -1837,6 +1837,7 @@ class VppPapiProvider(object):
             is_ipv6=0,
             encap_table_id=0,
             vni=0,
+            mode=1,
             instance=0xFFFFFFFF):
         """
 
@@ -1859,7 +1860,8 @@ class VppPapiProvider(object):
                              'mcast_sw_if_index': mcast_sw_if_index,
                              'encap_table_id': encap_table_id,
                              'vni': vni,
-                             'instance': instance}})
+                             'instance': instance,
+                             "mode": mode}})
 
     def vxlan_gbp_tunnel_dump(self, sw_if_index=0xffffffff):
         return self.api(self.papi.vxlan_gbp_tunnel_dump,
diff --git a/test/vpp_vxlan_gbp_tunnel.py b/test/vpp_vxlan_gbp_tunnel.py
index 73f951718a4..efeb4f960db 100644
--- a/test/vpp_vxlan_gbp_tunnel.py
+++ b/test/vpp_vxlan_gbp_tunnel.py
@@ -1,6 +1,7 @@
 
 from vpp_interface import VppInterface
 from vpp_ip import VppIpAddress
+from vpp_papi import VppEnum
 
 
 INDEX_INVALID = 0xffffffff
@@ -24,13 +25,18 @@ class VppVxlanGbpTunnel(VppInterface):
     VPP VXLAN GBP interface
     """
 
-    def __init__(self, test, src, dst, vni, mcast_itf=None):
+    def __init__(self, test, src, dst, vni, mcast_itf=None, mode=None):
         """ Create VXLAN-GBP Tunnel interface """
         super(VppVxlanGbpTunnel, self).__init__(test)
         self.src = VppIpAddress(src)
         self.dst = VppIpAddress(dst)
         self.vni = vni
         self.mcast_itf = mcast_itf
+        if not mode:
+            self.mode = (VppEnum.vl_api_vxlan_gbp_api_tunnel_mode_t.
+                         VXLAN_GBP_API_TUNNEL_MODE_L2)
+        else:
+            self.mode = mode
 
     def add_vpp_config(self):
         mcast_sw_if_index = INDEX_INVALID
@@ -39,6 +45,7 @@ class VppVxlanGbpTunnel(VppInterface):
         reply = self.test.vapi.vxlan_gbp_tunnel_add_del(
             self.src.encode(),
             self.dst.encode(),
+            mode=self.mode,
             vni=self.vni,
             mcast_sw_if_index=mcast_sw_if_index)
         self.set_sw_if_index(reply.sw_if_index)