QoS: marking and recording for MPLS and VLAN

Change-Id: Icec79aa9039d5d7835d311fde0b7c1a0c76c9eb1
Signed-off-by: Neale Ranns <nranns@cisco.com>
(cherry picked from commit 0809f6c0300f85cf5cf5d49df1aa8f1e2d080f6c)

4 files changed, 477 insertions, 22 deletions

diff --git a/src/vnet/ethernet/packet.h b/src/vnet/ethernet/packet.h
index 1a0506dc4d6..d70960b0f7b 100644
--- a/src/vnet/ethernet/packet.h
+++ b/src/vnet/ethernet/packet.h
@@ -146,6 +146,14 @@ ethernet_vlan_header_set_priority_net_order (ethernet_vlan_header_t * h,
   bytes[0] |= (prio & 0x7) << 5;
 }
 
+always_inline u8
+ethernet_vlan_header_get_priority_net_order (ethernet_vlan_header_t * h)
+{
+  u8 *bytes = (u8 *) (&h->priority_cfi_and_id);
+
+  return (bytes[0] >> 5);
+}
+
 /* VLAN with ethertype first and vlan id second */
 typedef struct
 {
diff --git a/src/vnet/qos/qos_mark.c b/src/vnet/qos/qos_mark.c
index 5dc87e8ef81..0f569b766a6 100644
--- a/src/vnet/qos/qos_mark.c
+++ b/src/vnet/qos/qos_mark.c
@@ -36,7 +36,15 @@ qos_mark_ip_enable_disable (u32 sw_if_index, u8 enable)
 void
 qos_mark_vlan_enable_disable (u32 sw_if_index, u8 enable)
 {
-  vnet_feature_enable_disable ("interface-output", "vlan-qos-mark",
+  /*
+   * one cannot run a feature on a sub-interface, so we need
+   * to enable a feature on all the L3 output paths
+   */
+  vnet_feature_enable_disable ("ip6-output", "vlan-ip6-qos-mark",
+			       sw_if_index, enable, NULL, 0);
+  vnet_feature_enable_disable ("ip4-output", "vlan-ip4-qos-mark",
+			       sw_if_index, enable, NULL, 0);
+  vnet_feature_enable_disable ("mpls-output", "vlan-mpls-qos-mark",
 			       sw_if_index, enable, NULL, 0);
 }
 
@@ -239,8 +247,22 @@ mpls_qos_mark (vlib_main_t * vm, vlib_node_runtime_t * node,
 }
 
 static inline uword
-vlan_qos_mark (vlib_main_t * vm, vlib_node_runtime_t * node,
-	       vlib_frame_t * frame)
+vlan_mpls_qos_mark (vlib_main_t * vm, vlib_node_runtime_t * node,
+		    vlib_frame_t * frame)
+{
+  return (qos_mark_inline (vm, node, frame, QOS_SOURCE_VLAN, 0));
+}
+
+static inline uword
+vlan_ip4_qos_mark (vlib_main_t * vm, vlib_node_runtime_t * node,
+		   vlib_frame_t * frame)
+{
+  return (qos_mark_inline (vm, node, frame, QOS_SOURCE_VLAN, 0));
+}
+
+static inline uword
+vlan_ip6_qos_mark (vlib_main_t * vm, vlib_node_runtime_t * node,
+		   vlib_frame_t * frame)
 {
   return (qos_mark_inline (vm, node, frame, QOS_SOURCE_VLAN, 0));
 }
@@ -311,9 +333,10 @@ VNET_FEATURE_INIT (mpls_qos_mark_node, static) = {
     .arc_name = "mpls-output",
     .node_name = "mpls-qos-mark",
 };
-VLIB_REGISTER_NODE (vlan_qos_mark_node) = {
-  .function = vlan_qos_mark,
-  .name = "vlan-qos-mark",
+
+VLIB_REGISTER_NODE (vlan_ip4_qos_mark_node) = {
+  .function = vlan_ip4_qos_mark,
+  .name = "vlan-ip4-qos-mark",
   .vector_size = sizeof (u32),
   .format_trace = format_qos_mark_trace,
   .type = VLIB_NODE_TYPE_INTERNAL,
@@ -326,11 +349,58 @@ VLIB_REGISTER_NODE (vlan_qos_mark_node) = {
   },
 };
 
-VLIB_NODE_FUNCTION_MULTIARCH (vlan_qos_mark_node, vlan_qos_mark);
+VLIB_NODE_FUNCTION_MULTIARCH (vlan_ip4_qos_mark_node, vlan_ip4_qos_mark);
+
+VNET_FEATURE_INIT (vlan_ip4_qos_mark_node, static) = {
+    .arc_name = "ip4-output",
+    .node_name = "vlan-ip4-qos-mark",
+    .runs_after = VNET_FEATURES ("ip4-qos-mark"),
+};
+
+VLIB_REGISTER_NODE (vlan_ip6_qos_mark_node) = {
+  .function = vlan_ip6_qos_mark,
+  .name = "vlan-ip6-qos-mark",
+  .vector_size = sizeof (u32),
+  .format_trace = format_qos_mark_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = 0,
+  .n_next_nodes = 1,
 
-VNET_FEATURE_INIT (vlan_qos_mark_node, static) = {
-    .arc_name = "interface-output",
-    .node_name = "vlan-qos-mark",
+  .next_nodes = {
+    [0] = "error-drop",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (vlan_ip6_qos_mark_node, vlan_ip6_qos_mark);
+
+VNET_FEATURE_INIT (vlan_ip6_qos_mark_node, static) = {
+    .arc_name = "ip6-output",
+    .node_name = "vlan-ip6-qos-mark",
+    .runs_after = VNET_FEATURES ("ip6-qos-mark"),
+};
+
+VLIB_REGISTER_NODE (vlan_mpls_qos_mark_node) = {
+  .function = vlan_mpls_qos_mark,
+  .name = "vlan-mpls-qos-mark",
+  .vector_size = sizeof (u32),
+  .format_trace = format_qos_mark_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = 0,
+  .n_next_nodes = 1,
+
+  .next_nodes = {
+    [0] = "error-drop",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (vlan_mpls_qos_mark_node, vlan_mpls_qos_mark);
+
+VNET_FEATURE_INIT (vlan_mpls_qos_mark_node, static) = {
+    .arc_name = "mpls-output",
+    .node_name = "vlan-mpls-qos-mark",
+    .runs_after = VNET_FEATURES ("mpls-qos-mark"),
 };
 /* *INDENT-ON* */
 
diff --git a/src/vnet/qos/qos_record.c b/src/vnet/qos/qos_record.c
index 7181e3bf599..92620458f08 100644
--- a/src/vnet/qos/qos_record.c
+++ b/src/vnet/qos/qos_record.c
@@ -44,9 +44,23 @@ qos_record_feature_config (u32 sw_if_index,
 				  enable);
       break;
     case QOS_SOURCE_MPLS:
+      vnet_feature_enable_disable ("mpls-input", "mpls-qos-record",
+				   sw_if_index, enable, NULL, 0);
+      break;
     case QOS_SOURCE_VLAN:
+      vnet_feature_enable_disable ("ip6-unicast", "vlan-ip6-qos-record",
+				   sw_if_index, enable, NULL, 0);
+      vnet_feature_enable_disable ("ip6-multicast", "vlan-ip6-qos-record",
+				   sw_if_index, enable, NULL, 0);
+      vnet_feature_enable_disable ("ip4-unicast", "vlan-ip4-qos-record",
+				   sw_if_index, enable, NULL, 0);
+      vnet_feature_enable_disable ("ip4-multicast", "vlan-ip4-qos-record",
+				   sw_if_index, enable, NULL, 0);
+      vnet_feature_enable_disable ("mpls-input", "vlan-mpls-qos-record",
+				   sw_if_index, enable, NULL, 0);
+      break;
     case QOS_SOURCE_EXT:
-      // not implemented yet
+      /* not a valid option for recording */
       break;
     }
 }
@@ -119,7 +133,7 @@ typedef struct qos_record_trace_t_
 static inline uword
 qos_record_inline (vlib_main_t * vm,
 		   vlib_node_runtime_t * node,
-		   vlib_frame_t * frame, int is_ip6, int is_l2)
+		   vlib_frame_t * frame, dpo_proto_t dproto, int is_l2)
 {
   u32 n_left_from, *from, *to_next, next_index;
 
@@ -164,23 +178,42 @@ qos_record_inline (vlib_main_t * vm,
 	      ethertype = clib_net_to_host_u16 (*(u16 *) (l3h - 2));
 
 	      if (ethertype == ETHERNET_TYPE_IP4)
-		is_ip6 = 0;
+		dproto = DPO_PROTO_IP4;
 	      else if (ethertype == ETHERNET_TYPE_IP6)
-		is_ip6 = 1;
+		dproto = DPO_PROTO_IP6;
+	      else if (ethertype == ETHERNET_TYPE_MPLS)
+		dproto = DPO_PROTO_MPLS;
 	      else
 		goto non_ip;
 	    }
 
-	  if (is_ip6)
+	  if (DPO_PROTO_IP6 == dproto)
 	    {
 	      ip6_0 = vlib_buffer_get_current (b0);
 	      qos0 = ip6_traffic_class_network_order (ip6_0);
 	    }
-	  else
+	  else if (DPO_PROTO_IP4 == dproto)
 	    {
 	      ip4_0 = vlib_buffer_get_current (b0);
 	      qos0 = ip4_0->tos;
 	    }
+	  else if (DPO_PROTO_ETHERNET == dproto)
+	    {
+	      ethernet_vlan_header_t *vlan0;
+
+	      vlan0 = (vlib_buffer_get_current (b0) -
+		       sizeof (ethernet_vlan_header_t));
+
+	      qos0 = ethernet_vlan_header_get_priority_net_order (vlan0);
+	    }
+	  else if (DPO_PROTO_MPLS)
+	    {
+	      mpls_unicast_header_t *mh;
+
+	      mh = vlib_buffer_get_current (b0);
+	      qos0 = vnet_mpls_uc_get_exp (mh->label_exp_s_ttl);
+	    }
+
 	  vnet_buffer2 (b0)->qos.bits = qos0;
 	  vnet_buffer2 (b0)->qos.source = QOS_SOURCE_IP;
 	  b0->flags |= VNET_BUFFER_F_QOS_DATA_VALID;
@@ -234,14 +267,42 @@ static inline uword
 ip4_qos_record (vlib_main_t * vm, vlib_node_runtime_t * node,
 		vlib_frame_t * frame)
 {
-  return (qos_record_inline (vm, node, frame, 0, 0));
+  return (qos_record_inline (vm, node, frame, DPO_PROTO_IP4, 0));
 }
 
 static inline uword
 ip6_qos_record (vlib_main_t * vm, vlib_node_runtime_t * node,
 		vlib_frame_t * frame)
 {
-  return (qos_record_inline (vm, node, frame, 1, 0));
+  return (qos_record_inline (vm, node, frame, DPO_PROTO_IP6, 0));
+}
+
+static inline uword
+mpls_qos_record (vlib_main_t * vm, vlib_node_runtime_t * node,
+		 vlib_frame_t * frame)
+{
+  return (qos_record_inline (vm, node, frame, DPO_PROTO_MPLS, 0));
+}
+
+static inline uword
+vlan_ip4_qos_record (vlib_main_t * vm, vlib_node_runtime_t * node,
+		     vlib_frame_t * frame)
+{
+  return (qos_record_inline (vm, node, frame, DPO_PROTO_ETHERNET, 0));
+}
+
+static inline uword
+vlan_ip6_qos_record (vlib_main_t * vm, vlib_node_runtime_t * node,
+		     vlib_frame_t * frame)
+{
+  return (qos_record_inline (vm, node, frame, DPO_PROTO_ETHERNET, 0));
+}
+
+static inline uword
+vlan_mpls_qos_record (vlib_main_t * vm, vlib_node_runtime_t * node,
+		      vlib_frame_t * frame)
+{
+  return (qos_record_inline (vm, node, frame, DPO_PROTO_ETHERNET, 0));
 }
 
 static inline uword
@@ -273,6 +334,10 @@ VNET_FEATURE_INIT (ip4_qos_record_node, static) = {
     .arc_name = "ip4-unicast",
     .node_name = "ip4-qos-record",
 };
+VNET_FEATURE_INIT (ip4m_qos_record_node, static) = {
+    .arc_name = "ip4-multicast",
+    .node_name = "ip4-qos-record",
+};
 
 VLIB_REGISTER_NODE (ip6_qos_record_node) = {
   .function = ip6_qos_record,
@@ -295,6 +360,111 @@ VNET_FEATURE_INIT (ip6_qos_record_node, static) = {
     .arc_name = "ip6-unicast",
     .node_name = "ip6-qos-record",
 };
+VNET_FEATURE_INIT (ip6m_qos_record_node, static) = {
+    .arc_name = "ip6-multicast",
+    .node_name = "ip6-qos-record",
+};
+
+VLIB_REGISTER_NODE (mpls_qos_record_node) = {
+  .function = mpls_qos_record,
+  .name = "mpls-qos-record",
+  .vector_size = sizeof (u32),
+  .format_trace = format_qos_record_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = 0,
+  .n_next_nodes = 1,
+
+  .next_nodes = {
+    [0] = "mpls-drop",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (mpls_qos_record_node, mpls_qos_record);
+
+VNET_FEATURE_INIT (mpls_qos_record_node, static) = {
+    .arc_name = "mpls-input",
+    .node_name = "mpls-qos-record",
+};
+
+VLIB_REGISTER_NODE (vlan_mpls_qos_record_node) = {
+  .function = vlan_mpls_qos_record,
+  .name = "vlan-mpls-qos-record",
+  .vector_size = sizeof (u32),
+  .format_trace = format_qos_record_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = 0,
+  .n_next_nodes = 1,
+
+  .next_nodes = {
+    [0] = "mpls-drop",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (vlan_mpls_qos_record_node, vlan_mpls_qos_record);
+
+VNET_FEATURE_INIT (vlan_mpls_qos_record_node, static) = {
+    .arc_name = "mpls-input",
+    .node_name = "vlan-mpls-qos-record",
+    .runs_before = VNET_FEATURES ("mpls-qos-mark"),
+};
+
+VLIB_REGISTER_NODE (vlan_ip4_qos_record_node) = {
+  .function = vlan_ip4_qos_record,
+  .name = "vlan-ip4-qos-record",
+  .vector_size = sizeof (u32),
+  .format_trace = format_qos_record_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = 0,
+  .n_next_nodes = 1,
+
+  .next_nodes = {
+    [0] = "ip4-drop",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (vlan_ip4_qos_record_node, vlan_ip4_qos_record);
+
+VNET_FEATURE_INIT (vlan_ip4_qos_record_node, static) = {
+    .arc_name = "ip4-unicast",
+    .node_name = "vlan-ip4-qos-record",
+    .runs_before = VNET_FEATURES ("ip4-qos-mark"),
+};
+VNET_FEATURE_INIT (vlan_ip4m_qos_record_node, static) = {
+    .arc_name = "ip4-multicast",
+    .node_name = "vlan-ip4-qos-record",
+    .runs_before = VNET_FEATURES ("ip4-qos-mark"),
+};
+
+VLIB_REGISTER_NODE (vlan_ip6_qos_record_node) = {
+  .function = vlan_ip6_qos_record,
+  .name = "vlan-ip6-qos-record",
+  .vector_size = sizeof (u32),
+  .format_trace = format_qos_record_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = 0,
+  .n_next_nodes = 1,
+
+  .next_nodes = {
+    [0] = "ip6-drop",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (vlan_ip6_qos_record_node, vlan_ip6_qos_record);
+
+VNET_FEATURE_INIT (vlan_ip6_qos_record_node, static) = {
+    .arc_name = "ip6-unicast",
+    .node_name = "vlan-ip6-qos-record",
+    .runs_before = VNET_FEATURES ("ip6-qos-mark"),
+};
+VNET_FEATURE_INIT (vlan_ip6m_qos_record_node, static) = {
+    .arc_name = "ip6-multicast",
+    .node_name = "vlan-ip6-qos-record",
+    .runs_before = VNET_FEATURES ("ip6-qos-mark"),
+};
 
 VLIB_REGISTER_NODE (l2_ip_qos_record_node, static) = {
   .function = l2_ip_qos_record,
@@ -313,6 +483,7 @@ VLIB_REGISTER_NODE (l2_ip_qos_record_node, static) = {
 };
 
 VLIB_NODE_FUNCTION_MULTIARCH (l2_ip_qos_record_node, l2_ip_qos_record);
+
 /* *INDENT-ON* */
 
 clib_error_t *
diff --git a/test/test_qos.py b/test/test_qos.py
index 939cca5d8d1..76fa8a11b6e 100644
--- a/test/test_qos.py
+++ b/test/test_qos.py
@@ -4,10 +4,12 @@ import unittest
 
 from framework import VppTestCase, VppTestRunner
 from vpp_papi_provider import QOS_SOURCE
-from vpp_ip_route import VppIpRoute, VppRoutePath
+from vpp_sub_interface import VppDot1QSubint
+from vpp_ip_route import VppIpRoute, VppRoutePath, VppMplsRoute, \
+    VppMplsLabel, VppMplsTable, DpoProto
 
 from scapy.packet import Raw
-from scapy.layers.l2 import Ether
+from scapy.layers.l2 import Ether, Dot1Q
 from scapy.layers.inet import IP, UDP
 from scapy.layers.inet6 import IPv6
 from scapy.contrib.mpls import MPLS
@@ -21,22 +23,27 @@ class TestQOS(VppTestCase):
 
         self.create_pg_interfaces(range(5))
 
+        tbl = VppMplsTable(self, 0)
+        tbl.add_vpp_config()
+
         for i in self.pg_interfaces:
             i.admin_up()
             i.config_ip4()
             i.resolve_arp()
             i.config_ip6()
             i.resolve_ndp()
+            i.enable_mpls()
 
     def tearDown(self):
         for i in self.pg_interfaces:
             i.unconfig_ip4()
             i.unconfig_ip6()
+            i.disable_mpls()
 
         super(TestQOS, self).tearDown()
 
     def test_qos_ip(self):
-        """ QoS Mark IP """
+        """ QoS Mark/Record IP """
 
         #
         # for table 1 map the n=0xff possible values of input QoS mark,
@@ -265,7 +272,7 @@ class TestQOS(VppTestCase):
         self.vapi.qos_egress_map_delete(7)
 
     def test_qos_mpls(self):
-        """ QoS Mark MPLS """
+        """ QoS Mark/Record MPLS """
 
         #
         # 255 QoS for all input values
@@ -346,6 +353,64 @@ class TestQOS(VppTestCase):
             self.assertEqual(h[MPLS].s, 1)
 
         #
+        # enable MPLS QoS recording on the input Pg0 and IP egress marking
+        # on Pg1
+        #
+        self.vapi.qos_record_enable_disable(self.pg0.sw_if_index,
+                                            QOS_SOURCE.MPLS,
+                                            1)
+        self.vapi.qos_mark_enable_disable(self.pg1.sw_if_index,
+                                          QOS_SOURCE.IP,
+                                          1,
+                                          1)
+
+        #
+        # MPLS x-connect - COS is preserved
+        #
+        route_32_eos = VppMplsRoute(self, 32, 1,
+                                    [VppRoutePath(self.pg1.remote_ip4,
+                                                  self.pg1.sw_if_index,
+                                                  labels=[VppMplsLabel(33)])])
+        route_32_eos.add_vpp_config()
+
+        p_m1 = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+                MPLS(label=32, cos=3, ttl=2) /
+                IP(src=self.pg0.remote_ip4, dst="10.0.0.1", tos=1) /
+                UDP(sport=1234, dport=1234) /
+                Raw(chr(100) * 65))
+
+        rx = self.send_and_expect(self.pg0, p_m1 * 65, self.pg1)
+        for p in rx:
+            self.assertEqual(p[MPLS].cos, 7)
+            self.assertEqual(p[MPLS].label, 33)
+            self.assertEqual(p[MPLS].s, 1)
+
+        #
+        # MPLS deag - COS is copied from MPLS to IP
+        #
+        route_33_eos = VppMplsRoute(self, 33, 1,
+                                    [VppRoutePath("0.0.0.0",
+                                                  0xffffffff,
+                                                  nh_table_id=0)])
+        route_33_eos.add_vpp_config()
+
+        route_10_0_0_4 = VppIpRoute(self, "10.0.0.4", 32,
+                                    [VppRoutePath(self.pg1.remote_ip4,
+                                                  self.pg1.sw_if_index)])
+        route_10_0_0_4.add_vpp_config()
+
+        p_m2 = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+                MPLS(label=33, ttl=2, cos=3) /
+                IP(src=self.pg0.remote_ip4, dst="10.0.0.4", tos=1) /
+                UDP(sport=1234, dport=1234) /
+                Raw(chr(100) * 65))
+
+        rx = self.send_and_expect(self.pg0, p_m2 * 65, self.pg1)
+
+        for p in rx:
+            self.assertEqual(p[IP].tos, 255)
+
+        #
         # cleanup
         #
         self.vapi.qos_record_enable_disable(self.pg0.sw_if_index,
@@ -355,8 +420,149 @@ class TestQOS(VppTestCase):
                                           QOS_SOURCE.MPLS,
                                           1,
                                           0)
+        self.vapi.qos_record_enable_disable(self.pg0.sw_if_index,
+                                            QOS_SOURCE.MPLS,
+                                            0)
+        self.vapi.qos_mark_enable_disable(self.pg1.sw_if_index,
+                                          QOS_SOURCE.IP,
+                                          1,
+                                          0)
         self.vapi.qos_egress_map_delete(1)
 
+    def test_qos_vlan(self):
+        """QoS mark/record VLAN """
+
+        #
+        # QoS for all input values
+        #
+        output = [chr(0)] * 256
+        for i in range(0, 255):
+            output[i] = chr(255 - i)
+        os = ''.join(output)
+        rows = [{'outputs': os},
+                {'outputs': os},
+                {'outputs': os},
+                {'outputs': os}]
+
+        self.vapi.qos_egress_map_update(1, rows)
+
+        sub_if = VppDot1QSubint(self, self.pg0, 11)
+
+        sub_if.admin_up()
+        sub_if.config_ip4()
+        sub_if.resolve_arp()
+        sub_if.config_ip6()
+        sub_if.resolve_ndp()
+
+        #
+        # enable VLAN QoS recording/marking on the input Pg0 subinterface and
+        #
+        self.vapi.qos_record_enable_disable(sub_if.sw_if_index,
+                                            QOS_SOURCE.VLAN,
+                                            1)
+        self.vapi.qos_mark_enable_disable(sub_if.sw_if_index,
+                                          QOS_SOURCE.VLAN,
+                                          1,
+                                          1)
+
+        #
+        # IP marking/recording on pg1
+        #
+        self.vapi.qos_record_enable_disable(self.pg1.sw_if_index,
+                                            QOS_SOURCE.IP,
+                                            1)
+        self.vapi.qos_mark_enable_disable(self.pg1.sw_if_index,
+                                          QOS_SOURCE.IP,
+                                          1,
+                                          1)
+
+        #
+        # a routes to/from sub-interface
+        #
+        route_10_0_0_1 = VppIpRoute(self, "10.0.0.1", 32,
+                                    [VppRoutePath(sub_if.remote_ip4,
+                                                  sub_if.sw_if_index)])
+        route_10_0_0_1.add_vpp_config()
+        route_10_0_0_2 = VppIpRoute(self, "10.0.0.2", 32,
+                                    [VppRoutePath(self.pg1.remote_ip4,
+                                                  self.pg1.sw_if_index)])
+        route_10_0_0_2.add_vpp_config()
+        route_2001_1 = VppIpRoute(self, "2001::1", 128,
+                                  [VppRoutePath(sub_if.remote_ip6,
+                                                sub_if.sw_if_index,
+                                                proto=DpoProto.DPO_PROTO_IP6)],
+                                  is_ip6=1)
+        route_2001_1.add_vpp_config()
+        route_2001_2 = VppIpRoute(self, "2001::2", 128,
+                                  [VppRoutePath(self.pg1.remote_ip6,
+                                                self.pg1.sw_if_index,
+                                                proto=DpoProto.DPO_PROTO_IP6)],
+                                  is_ip6=1)
+        route_2001_2.add_vpp_config()
+
+        p_v1 = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+                Dot1Q(vlan=11, prio=1) /
+                IP(src="1.1.1.1", dst="10.0.0.2", tos=1) /
+                UDP(sport=1234, dport=1234) /
+                Raw(chr(100) * 65))
+
+        p_v2 = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
+                IP(src="1.1.1.1", dst="10.0.0.1", tos=1) /
+                UDP(sport=1234, dport=1234) /
+                Raw(chr(100) * 65))
+
+        rx = self.send_and_expect(self.pg1, p_v2 * 65, self.pg0)
+
+        for p in rx:
+            self.assertEqual(p[Dot1Q].prio, 6)
+
+        rx = self.send_and_expect(self.pg0, p_v1 * 65, self.pg1)
+
+        for p in rx:
+            self.assertEqual(p[IP].tos, 254)
+
+        p_v1 = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+                Dot1Q(vlan=11, prio=2) /
+                IPv6(src="2001::1", dst="2001::2", tc=1) /
+                UDP(sport=1234, dport=1234) /
+                Raw(chr(100) * 65))
+
+        p_v2 = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
+                IPv6(src="3001::1", dst="2001::1", tc=1) /
+                UDP(sport=1234, dport=1234) /
+                Raw(chr(100) * 65))
+
+        rx = self.send_and_expect(self.pg1, p_v2 * 65, self.pg0)
+
+        for p in rx:
+            self.assertEqual(p[Dot1Q].prio, 6)
+
+        rx = self.send_and_expect(self.pg0, p_v1 * 65, self.pg1)
+
+        for p in rx:
+            self.assertEqual(p[IPv6].tc, 253)
+
+        #
+        # cleanup
+        #
+        sub_if.unconfig_ip4()
+        sub_if.unconfig_ip6()
+
+        self.vapi.qos_record_enable_disable(sub_if.sw_if_index,
+                                            QOS_SOURCE.VLAN,
+                                            0)
+        self.vapi.qos_mark_enable_disable(sub_if.sw_if_index,
+                                          QOS_SOURCE.VLAN,
+                                          1,
+                                          0)
+        self.vapi.qos_record_enable_disable(self.pg1.sw_if_index,
+                                            QOS_SOURCE.IP,
+                                            0)
+        self.vapi.qos_mark_enable_disable(self.pg1.sw_if_index,
+                                          QOS_SOURCE.IP,
+                                          1,
+                                          0)
+
 
 if __name__ == '__main__':
     unittest.main(testRunner=VppTestRunner)