1 files changed, 180 insertions, 195 deletions

diff --git a/src/vnet/ipfix-export/ipfix_doc.md b/src/vnet/ipfix-export/ipfix_doc.md
index fba420ecf12..95db15cc341 100644
--- a/src/vnet/ipfix-export/ipfix_doc.md
+++ b/src/vnet/ipfix-export/ipfix_doc.md
@@ -13,37 +13,114 @@ vnet_flow_report_add_del_args_t structure, and call vnet_flow_report_add_del.
 
 ```{.c}
    #include <vnet/ipfix-export/flow_report.h>
+   /* Defined in flow_report.h, of interest when constructing reports */
 
+   /* ipfix field definitions for a particular report */
    typedef struct
    {
+     u32 info_element;
+     u32 size;
+   } ipfix_report_element_t;
+
+   /* Report add/del argument structure */
+   typedef struct
+   {
+     /* Callback to flush current ipfix packet / frame */
      vnet_flow_data_callback_t *flow_data_callback;
+
+     /* Callback to build the template packet rewrite string */
      vnet_flow_rewrite_callback_t *rewrite_callback;
+
+     /* List of ipfix elements in the report */
+     ipfix_report_element_t *report_elements;
+     u32 n_report_elements;
+     /* Kept in flow report, used e.g. by flow classifier */
      opaque_t opaque;
+     /* Add / delete a report */
      int is_add;
+     /* Ipfix "domain-ID", see RFC, set as desired */
      u32 domain_id;
+     /* ipfix packet source port, often set to UDP_DST_PORT_ipfix */
      u16 src_port;
+     /* Set by ipfix infra, needed to send data packets */
+     u32 *stream_indexp;
    } vnet_flow_report_add_del_args_t;
 
+   /* Private header file contents */
+
+   /* Report ipfix element definition */
+   #define foreach_simple_report_ipfix_element     \
+   _(sourceIPv4Address, 4)                         \
+   _(destinationIPv4Address, 4)                    \
+   _(sourceTransportPort, 2)                       \
+   _(destinationTransportPort, 2)                  \
+   _(protocolIdentifier, 1)                        \
+   _(flowStartMicroseconds, 8)                     \
+   _(flowEndMicroseconds, 8)
+
+   static ipfix_report_element_t simple_report_elements[] = {
+   #define _(a,b) {a,b},
+     foreach_simple_report_ipfix_element
+   #undef _
+   };
+
+   typedef struct
+   {
+     /** Buffers and frames, per thread */
+     vlib_buffer_t **buffers_by_thread;
+     vlib_frame_t **frames_by_thread;
+     u32 *next_record_offset_by_thread;
+
+     /** Template ID's */
+     u16 *template_ids;
+
+     /** Time reference pair */
+     u64 usec_time_0;
+     f64 vlib_time_0;
+
+     /** Stream index */
+     u32 stream_index;
+
+     /* Convenience */
+     flow_report_main_t *flow_report_main;
+     vlib_main_t *vlib_main;
+     vnet_main_t *vnet_main;
+   } my_logging_main_t;
+   
+   extern my_logging_main_t my_logging_main;
+
    ...
 
+   /* Recitations */
    flow_report_main_t *frm = &flow_report_main;
+   my_logging_main_t *mlm = &my_logging_main;
    vnet_flow_report_add_del_args_t a;
    int rv;
    u16 template_id;
 
    ... 
 
-   /* Set up time reference pair */
+   /* Init function: set up time reference pair */
    mlm->vlib_time_0 = vlib_time_now (vm);
    mlm->milisecond_time_0 = unix_time_now_nsec () * 1e-6;
 
    ...
 
+   /* Create a report */
    memset (&a, 0, sizeof (a));
    a.is_add = 1 /* to enable the report */;
    a.domain_id = 1 /* pick a domain ID */;
    a.src_port = UDP_DST_PORT_ipfix /* src port for reports */;
-   a.rewrite_callback = my_template_packet_rewrite_callback;
+
+   /* Use the generic template packet rewrite string generator */
+   a.rewrite_callback = vnet_flow_rewrite_generic_callback;
+
+   /* Supply a list of ipfix report elements */
+   a.report_elements = simple_report_elements;
+   a.n_report_elements = ARRAY_LEN (simple_report_elements);
+
+   /* Pointer to the ipfix stream index, set by the report infra */
+   a.stream_indexp = &mlm->stream_index;
    a.flow_data_callback = my_flow_data_callback;
 
    /* Create the report */
@@ -56,100 +133,13 @@ vnet_flow_report_add_del_args_t structure, and call vnet_flow_report_add_del.
 
 ```
 
-Several functions are worth describing in detail.
+Several things are worth describing in more detail.
 
-### template packet rewrite callback function
+### vnet_flow_rewrite_generic_callback programming
 
-This callback helps build ipfix template packets when required. We
-should reduce the amount of cut-'n-paste coding, since only a fraction
-of the code has anything to do with the specific ipfix template we're
-trying to build.
-
-```{.c}
-   u8 *
-   my_template_packet_rewrite_callback (flow_report_main_t * frm,
-                                        flow_report_t * fr,
-				        ip4_address_t * collector_address,
-				        ip4_address_t * src_address,
-				        u16 collector_port)
-   {
-       my_logging_main_t *mlm = &my_logging_main; /* typical */
-       ip4_header_t *ip;
-       udp_header_t *udp;
-       ipfix_message_header_t *h;
-       ipfix_set_header_t *s;
-       ipfix_template_header_t *t;
-       ipfix_field_specifier_t *f;
-       ipfix_field_specifier_t *first_field;
-       u8 *rewrite = 0;
-       ip4_ipfix_template_packet_t *tp;
-       u32 field_count = 0;
-       flow_report_stream_t *stream;
-
-       stream = &frm->streams[fr->stream_index];
-
-       field_count = number_of_fields_to_export;
-
-       /* allocate rewrite space */
-       vec_validate_aligned (rewrite,
-  			sizeof (ip4_ipfix_template_packet_t)
-			+ field_count * sizeof (ipfix_field_specifier_t) - 1,
-			CLIB_CACHE_LINE_BYTES);
-
-       /* create the packet rewrite string */
-       tp = (ip4_ipfix_template_packet_t *) rewrite;
-       ip = (ip4_header_t *) & tp->ip4;
-       udp = (udp_header_t *) (ip + 1);
-       h = (ipfix_message_header_t *) (udp + 1);
-       s = (ipfix_set_header_t *) (h + 1);
-       t = (ipfix_template_header_t *) (s + 1);
-       first_field = f = (ipfix_field_specifier_t *) (t + 1);
-
-       ip->ip_version_and_header_length = 0x45;
-       ip->ttl = 254;
-       ip->protocol = IP_PROTOCOL_UDP;
-       ip->src_address.as_u32 = src_address->as_u32;
-       ip->dst_address.as_u32 = collector_address->as_u32;
-       udp->src_port = clib_host_to_net_u16 (stream->src_port);
-       udp->dst_port = clib_host_to_net_u16 (collector_port);
-       udp->length = clib_host_to_net_u16 (vec_len (rewrite) - sizeof (*ip));
-
-       /* FIXUP LATER: message header export_time */
-       h->domain_id = clib_host_to_net_u32 (stream->domain_id);
-
-       /* 
-        * Add your favorite info elements to the template. See
-        * .../src/vnet/ipfix-export/ipfix_info_elements.h
-        *
-        * Highly advisable to make sure field count is correct!
-        */
-
-       f->e_id_length = ipfix_e_id_length (0, sourceIPv6Address, 16);
-       f++;
-       f->e_id_length = ipfix_e_id_length (0, postNATSourceIPv4Address, 4);
-       f++;
-
-       /* Back to the template packet... */
-       ip = (ip4_header_t *) & tp->ip4;
-       udp = (udp_header_t *) (ip + 1);
-
-       ASSERT (f - first_field);
-       /* Field count in this template */
-       t->id_count = ipfix_id_count (fr->template_id, f - first_field);
-
-       /* set length in octets */
-       s->set_id_length =
-         ipfix_set_id_length (2 /* set_id */ , (u8 *) f - (u8 *) s);
-
-       /* message length in octets */
-       h->version_length = version_length ((u8 *) f - (u8 *) h);
-
-       ip->length = clib_host_to_net_u16 ((u8 *) f - (u8 *) ip);
-       ip->checksum = ip4_header_checksum (ip);
-
-       return rewrite;
-   }      
-```
+This generic callback helps build ipfix template packets.  When
+registering an ipfix report, pass an (array, count)
+of ipfix elements as shown above. 
 
 ### my_flow_data_callback
 
@@ -185,10 +175,12 @@ This function creates the packet header for an ipfix data packet
    my_flow_report_header (flow_report_main_t * frm,
 			  vlib_buffer_t * b0, u32 * offset)
    {
-      snat_ipfix_logging_main_t *mlm = &my_logging_main;
+      my_logging_main_t *mlm = &my_logging_main;
       flow_report_stream_t *stream;
       ip4_ipfix_template_packet_t *tp;
       ipfix_message_header_t *h = 0;
+
+
       ipfix_set_header_t *s = 0;
       ip4_header_t *ip;
       udp_header_t *udp;
@@ -219,97 +211,82 @@ This function creates the packet header for an ipfix data packet
 
       h->export_time = clib_host_to_net_u32 ((u32)
             				 (((f64) frm->unix_time_0) +
-            				  (vlib_time_now (frm->vlib_main) -
-            				   frm->vlib_time_0)));
-      h->sequence_number = clib_host_to_net_u32 (stream->sequence_number++);
-      h->domain_id = clib_host_to_net_u32 (stream->domain_id);
+               				  (vlib_time_now (frm->vlib_main) -
+               				   frm->vlib_time_0)));
+         h->sequence_number = clib_host_to_net_u32 (stream->sequence_number++);
+         h->domain_id = clib_host_to_net_u32 (stream->domain_id);
 
-      *offset = (u32) (((u8 *) (s + 1)) - (u8 *) tp);
-}
-```
-
-### fixup and transmit a flow record
-
-```{.c}
-   
+         *offset = (u32) (((u8 *) (s + 1)) - (u8 *) tp);
+   }
+   ```
+
+   ### fixup and transmit a flow record
+
+   ```{.c}
+      
+      static inline void
+      my_send_ipfix_pkt (flow_report_main_t * frm,
+           		 vlib_frame_t * f, vlib_buffer_t * b0, u16 template_id)
+      {
+        ip4_ipfix_template_packet_t *tp;
+        ipfix_message_header_t *h = 0;
+        ipfix_set_header_t *s = 0;
+        ip4_header_t *ip;
+        udp_header_t *udp;
+        vlib_main_t *vm = frm->vlib_main;
+
+        tp = vlib_buffer_get_current (b0);
+        ip = (ip4_header_t *) & tp->ip4;
+        udp = (udp_header_t *) (ip + 1);
+        h = (ipfix_message_header_t *) (udp + 1);
+        s = (ipfix_set_header_t *) (h + 1);
+
+        s->set_id_length = ipfix_set_id_length (template_id,
+      					  b0->current_length -
+      					  (sizeof (*ip) + sizeof (*udp) +
+      					   sizeof (*h)));
+        h->version_length = version_length (b0->current_length -
+      				      (sizeof (*ip) + sizeof (*udp)));
+
+        ip->length = clib_host_to_net_u16 (b0->current_length);
+        ip->checksum = ip4_header_checksum (ip);
+        udp->length = clib_host_to_net_u16 (b0->current_length - sizeof (*ip));
+
+        if (frm->udp_checksum)
+          {
+            udp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip);
+            if (udp->checksum == 0)
+      	udp->checksum = 0xffff;
+          }
+
+        ASSERT (ip->checksum == ip4_header_checksum (ip));
+
+        vlib_put_frame_to_node (vm, ip4_lookup_node.index, f);
+      }  
+   ```
+
+   ### my_buffer_flow_record
+
+   This is the key routine which paints individual flow records into
+   an ipfix packet under construction. It's pretty straightforward
+   (albeit stateful) vpp data-plane code. The code shown below is
+   thread-safe by construction.
+
+   ```{.c}
    static inline void
-   my_send_ipfix_pkt (flow_report_main_t * frm,
-        		 vlib_frame_t * f, vlib_buffer_t * b0, u16 template_id)
+   my_buffer_flow_record_internal (my_flow_record_t * rp, int do_flush,
+                                       u32 thread_index)
    {
-     ip4_ipfix_template_packet_t *tp;
-     ipfix_message_header_t *h = 0;
-     ipfix_set_header_t *s = 0;
-     ip4_header_t *ip;
-     udp_header_t *udp;
-     vlib_main_t *vm = frm->vlib_main;
-
-     tp = vlib_buffer_get_current (b0);
-     ip = (ip4_header_t *) & tp->ip4;
-     udp = (udp_header_t *) (ip + 1);
-     h = (ipfix_message_header_t *) (udp + 1);
-     s = (ipfix_set_header_t *) (h + 1);
-
-     s->set_id_length = ipfix_set_id_length (template_id,
-   					  b0->current_length -
-   					  (sizeof (*ip) + sizeof (*udp) +
-   					   sizeof (*h)));
-     h->version_length = version_length (b0->current_length -
-   				      (sizeof (*ip) + sizeof (*udp)));
-
-     ip->length = clib_host_to_net_u16 (b0->current_length);
-     ip->checksum = ip4_header_checksum (ip);
-     udp->length = clib_host_to_net_u16 (b0->current_length - sizeof (*ip));
-
-     if (frm->udp_checksum)
-       {
-         udp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip);
-         if (udp->checksum == 0)
-   	udp->checksum = 0xffff;
-       }
-
-     ASSERT (ip->checksum == ip4_header_checksum (ip));
-
-     vlib_put_frame_to_node (vm, ip4_lookup_node.index, f);
-   }  
-```
-
-### my_buffer_flow_record
-
-This is the key routine which paints individual flow records into
-an ipfix packet under construction. It's pretty straightforward
-(albeit stateful) vpp data-plane code.
-
-
-```{.c}
-   static void
-   my_buffer_flow_record (u32 datum0, u32 datum1, ..., int do_flush)
-   {
-     my_logging_main_t *mlm = &my_logging_main;
+     vlib_main_t *vm = vlib_mains[thread_index];
+     my_logging_main_t *mlm = &jvp_ipfix_main;
      flow_report_main_t *frm = &flow_report_main;
      vlib_frame_t *f;
      vlib_buffer_t *b0 = 0;
      u32 bi0 = ~0;
      u32 offset;
-     vlib_main_t *vm = frm->vlib_main;
-     u64 now;
      vlib_buffer_free_list_t *fl;
-     my_flow_record_t my_flow_record;
-
-     if (!mlm->enabled)
-       return;
 
-     now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
-     now += mlm->milisecond_time_0;
-
-     /* 
-      * (maybe) set up a packed structure from datum0...datumN 
-      * Otherwise, paint directly into the buffer below...
-      */
-     my_flow_record.xxx = datum0;
-     my_flow_record.yyy = datum1;
-
-
-     b0 = mlm->my_data_buffer;
+     b0 = mlm->buffers_by_thread[thread_index];
 
      if (PREDICT_FALSE (b0 == 0))
        {
@@ -322,28 +299,30 @@ an ipfix packet under construction. It's pretty straightforward
    	  return;
    	}
 
-         b0 = mlm->my_data_buffer = vlib_get_buffer (vm, bi0);
+         b0 = vlib_get_buffer (vm, bi0);
          fl =
    	vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
          vlib_buffer_init_for_free_list (b0, fl);
          VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
          offset = 0;
+         mlm->buffers_by_thread[thread_index] = b0;
        }
      else
        {
          bi0 = vlib_get_buffer_index (vm, b0);
-         offset = mlm->my_next_record_offset;
+         offset = mlm->next_record_offset_by_thread[thread_index];
        }
 
-     f = mlm->my_ipfix_frame;
+     f = mlm->frames_by_thread[thread_index];
      if (PREDICT_FALSE (f == 0))
        {
          u32 *to_next;
          f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-         mlm->my_ipfix_frame = f;
+         mlm->frames_by_thread[thread_index] = f;
          to_next = vlib_frame_vector_args (f);
          to_next[0] = bi0;
          f->n_vectors = 1;
+         mlm->frames_by_thread[thread_index] = f;
        }
 
      if (PREDICT_FALSE (offset == 0))
@@ -351,25 +330,31 @@ an ipfix packet under construction. It's pretty straightforward
 
      if (PREDICT_TRUE (do_flush == 0))
        {
-         /* paint time stamp into buffer */
-         clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp));
-         offset += sizeof (time_stamp);
-
          /* Paint the new ipfix data record into the buffer */
-         clib_memcpy (b0->data + offset, &my_flow_record, 
-                     sizeof (my_flow_record));
-         offset += sizeof (my_flow_record);
-         b0->current_length += sizeof(my_flow_record);
+         clib_memcpy (b0->data + offset, rp, sizeof (*rp));
+         offset += sizeof (*rp);
+         b0->current_length += sizeof (*rp);
        }
 
-     if (PREDICT_FALSE
-         (do_flush || (offset + sizeof (my_flow_record)) > frm->path_mtu))
+     if (PREDICT_FALSE (do_flush || (offset + sizeof (*rp)) > frm->path_mtu))
        {
-         my_send_ipfix_pkt (frm, f, b0, mlm->template_id);
-         mlm->my_ipfix_frame = 0;
-         mlm->my_data_buffer = 0;
+         /* Nothing to send? */
+         if (offset == 0)
+   	return;
+
+         send_ipfix_pkt (frm, f, b0, mlm->template_ids[0]);
+         mlm->buffers_by_thread[thread_index] = 0;
+         mlm->frames_by_thread[thread_index] = 0;
          offset = 0;
        }
-     mlm->next_record_offset = offset;
-   }
+     mlm->next_record_offset_by_thread[thread_index] = offset;
+   }  
+
+   static void
+   my_buffer_flow_record (my_flow_record_t * rp, int do_flush)
+   {
+     u32 thread_index = vlib_get_thread_index();
+     my_buffer_flow_record_internal (rp, do_flush, thread_index);
+   }  
+
 ```