flowprobe: fix sending L4 fields in L2 template and flows

Currently, when L2 and L4 recording is enabled on the L2 datapath, the
L2 template will contain L4 fields and L2 flows will be exported with
those fields always set to zero.

With this fix, when L4 recording is enabled, add L4 fields to templates
other than the L2 template (i.e. to the IP4, IP6, L2_IP4, and L2_IP6
templates). And export L2 flows without L4 fields. Also, cover that case
in the tests.

Type: fix
Change-Id: Id5ed8b99af5634fb9d5c6e695203344782fdac01
Signed-off-by: Alexander Chernavin <achernavin@netgate.com>

3 files changed, 53 insertions, 10 deletions

diff --git a/src/plugins/flowprobe/flowprobe.c b/src/plugins/flowprobe/flowprobe.c
index fb05158576e..af8b8ce5218 100644
--- a/src/plugins/flowprobe/flowprobe.c
+++ b/src/plugins/flowprobe/flowprobe.c
@@ -245,6 +245,7 @@ flowprobe_template_rewrite_inline (ipfix_exporter_t *exp, flow_report_t *fr,
   flowprobe_main_t *fm = &flowprobe_main;
   flowprobe_record_t flags = fr->opaque.as_uword;
   bool collect_ip4 = false, collect_ip6 = false;
+  bool collect_l4 = false;
 
   stream = &exp->streams[fr->stream_index];
 
@@ -257,6 +258,10 @@ flowprobe_template_rewrite_inline (ipfix_exporter_t *exp, flow_report_t *fr,
       if (which == FLOW_VARIANT_L2_IP6)
 	flags |= FLOW_RECORD_L2_IP6;
     }
+  if (flags & FLOW_RECORD_L4)
+    {
+      collect_l4 = (which != FLOW_VARIANT_L2);
+    }
 
   field_count += flowprobe_template_common_field_count ();
   if (flags & FLOW_RECORD_L2)
@@ -265,7 +270,7 @@ flowprobe_template_rewrite_inline (ipfix_exporter_t *exp, flow_report_t *fr,
     field_count += flowprobe_template_ip4_field_count ();
   if (collect_ip6)
     field_count += flowprobe_template_ip6_field_count ();
-  if (flags & FLOW_RECORD_L4)
+  if (collect_l4)
     field_count += flowprobe_template_l4_field_count ();
 
   /* allocate rewrite space */
@@ -304,7 +309,7 @@ flowprobe_template_rewrite_inline (ipfix_exporter_t *exp, flow_report_t *fr,
     f = flowprobe_template_ip4_fields (f);
   if (collect_ip6)
     f = flowprobe_template_ip6_fields (f);
-  if (flags & FLOW_RECORD_L4)
+  if (collect_l4)
     f = flowprobe_template_l4_fields (f);
 
   /* Back to the template packet... */
diff --git a/src/plugins/flowprobe/node.c b/src/plugins/flowprobe/node.c
index 44096d6245f..8466eda3792 100644
--- a/src/plugins/flowprobe/node.c
+++ b/src/plugins/flowprobe/node.c
@@ -701,6 +701,7 @@ flowprobe_export_entry (vlib_main_t * vm, flowprobe_entry_t * e)
   ipfix_exporter_t *exp = pool_elt_at_index (flow_report_main.exporters, 0);
   vlib_buffer_t *b0;
   bool collect_ip4 = false, collect_ip6 = false;
+  bool collect_l4 = false;
   flowprobe_variant_t which = e->key.which;
   flowprobe_record_t flags = fm->context[which].flags;
   u16 offset =
@@ -719,6 +720,10 @@ flowprobe_export_entry (vlib_main_t * vm, flowprobe_entry_t * e)
       collect_ip4 = which == FLOW_VARIANT_L2_IP4 || which == FLOW_VARIANT_IP4;
       collect_ip6 = which == FLOW_VARIANT_L2_IP6 || which == FLOW_VARIANT_IP6;
     }
+  if (flags & FLOW_RECORD_L4)
+    {
+      collect_l4 = (which != FLOW_VARIANT_L2);
+    }
 
   offset += flowprobe_common_add (b0, e, offset);
 
@@ -728,7 +733,7 @@ flowprobe_export_entry (vlib_main_t * vm, flowprobe_entry_t * e)
     offset += flowprobe_l3_ip6_add (b0, e, offset);
   if (collect_ip4)
     offset += flowprobe_l3_ip4_add (b0, e, offset);
-  if (flags & FLOW_RECORD_L4)
+  if (collect_l4)
     offset += flowprobe_l4_add (b0, e, offset);
 
   /* Reset per flow-export counters */
diff --git a/test/test_flowprobe.py b/test/test_flowprobe.py
index 0bdbd3d6789..5da0f74944b 100644
--- a/test/test_flowprobe.py
+++ b/test/test_flowprobe.py
@@ -28,6 +28,12 @@ from socket import inet_ntop
 from vpp_papi import VppEnum
 
 
+TMPL_COMMON_FIELD_COUNT = 6
+TMPL_L2_FIELD_COUNT = 3
+TMPL_L3_FIELD_COUNT = 4
+TMPL_L4_FIELD_COUNT = 3
+
+
 class VppCFLOW(VppObject):
     """CFLOW object for IPFIX exporter and Flowprobe feature"""
 
@@ -125,15 +131,18 @@ class VppCFLOW(VppObject):
     def query_vpp_config(self):
         return self._configured
 
-    def verify_templates(self, decoder=None, timeout=1, count=3):
+    def verify_templates(self, decoder=None, timeout=1, count=3, field_count_in=None):
         templates = []
         self._test.assertIn(count, (1, 2, 3))
         for _ in range(count):
             p = self._test.wait_for_cflow_packet(self._test.collector, 2, timeout)
             self._test.assertTrue(p.haslayer(IPFIX))
-            if decoder is not None and p.haslayer(Template):
+            self._test.assertTrue(p.haslayer(Template))
+            if decoder is not None:
                 templates.append(p[Template].templateID)
                 decoder.add_template(p.getlayer(Template))
+            if field_count_in is not None:
+                self._test.assertIn(p[Template].fieldCount, field_count_in)
         return templates
 
 
@@ -265,7 +274,13 @@ class MethodHolder(VppTestCase):
         return dst_if.get_capture(len(self.pkts))
 
     def verify_cflow_data_detail(
-        self, decoder, capture, cflow, data_set={1: "octets", 2: "packets"}, ip_ver="v4"
+        self,
+        decoder,
+        capture,
+        cflow,
+        data_set={1: "octets", 2: "packets"},
+        ip_ver="v4",
+        field_count=None,
     ):
         if self.debug_print:
             print(capture[0].show())
@@ -312,6 +327,9 @@ class MethodHolder(VppTestCase):
                         self.assertEqual(
                             int(binascii.hexlify(record[field]), 16), value
                         )
+            if field_count is not None:
+                for record in data:
+                    self.assertEqual(len(record), field_count)
 
     def verify_cflow_data_notimer(self, decoder, capture, cflows):
         idx = 0
@@ -683,19 +701,30 @@ class DatapathTestsHolder(object):
         ipfix.remove_vpp_config()
         self.logger.info("FFP_TEST_FINISH_0002")
 
-    def test_L23onL2(self):
-        """L2/3 data on L2 datapath"""
+    def test_L234onL2(self):
+        """L2/3/4 data on L2 datapath"""
         self.pg_enable_capture(self.pg_interfaces)
         self.pkts = []
 
         ipfix = VppCFLOW(
-            test=self, intf=self.intf1, layer="l2 l3", direction=self.direction
+            test=self, intf=self.intf1, layer="l2 l3 l4", direction=self.direction
         )
         ipfix.add_vpp_config()
 
         ipfix_decoder = IPFIXDecoder()
         # template packet should arrive immediately
-        templates = ipfix.verify_templates(ipfix_decoder, count=3)
+        tmpl_l2_field_count = TMPL_COMMON_FIELD_COUNT + TMPL_L2_FIELD_COUNT
+        tmpl_ip_field_count = (
+            TMPL_COMMON_FIELD_COUNT
+            + TMPL_L2_FIELD_COUNT
+            + TMPL_L3_FIELD_COUNT
+            + TMPL_L4_FIELD_COUNT
+        )
+        templates = ipfix.verify_templates(
+            ipfix_decoder,
+            count=3,
+            field_count_in=(tmpl_l2_field_count, tmpl_ip_field_count),
+        )
 
         # verify IPv4 and IPv6 flows
         for ip_ver in ("v4", "v6"):
@@ -717,11 +746,14 @@ class DatapathTestsHolder(object):
                     2: "packets",
                     256: 8 if ip_ver == "v4" else 56710,
                     4: 17,
+                    7: "sport",
+                    11: "dport",
                     src_ip_id: "src_ip",
                     dst_ip_id: "dst_ip",
                     61: (self.direction == "tx"),
                 },
                 ip_ver=ip_ver,
+                field_count=tmpl_ip_field_count,
             )
 
         # verify non-IP flow
@@ -742,6 +774,7 @@ class DatapathTestsHolder(object):
             capture,
             cflow,
             {2: "packets", 256: 2440, 61: (self.direction == "tx")},
+            field_count=tmpl_l2_field_count,
         )
 
         self.collector.get_capture(6)