Move iOAM Trace as a plugin

Change-Id: I78322e91485e5429871bd356f668b8ed90f4265a
Signed-off-by: Vengada Govindan <venggovi@cisco.com>

16 files changed, 2047 insertions, 1003 deletions

diff --git a/plugins/ioam-plugin/Makefile.am b/plugins/ioam-plugin/Makefile.am
index 68e792482a9..92b2b37bb6a 100644
--- a/plugins/ioam-plugin/Makefile.am
+++ b/plugins/ioam-plugin/Makefile.am
@@ -87,6 +87,35 @@ ioam_export_test_plugin_la_SOURCES =  \
 vppapitestplugins_LTLIBRARIES += ioam_export_test_plugin.la
 vppplugins_LTLIBRARIES += ioam_export_plugin.la
 
+########################################
+# iOAM Trace
+########################################
+ioam_trace_plugin_la_SOURCES =			\
+	ioam/lib-trace/trace_util.c			\
+	ioam/encap/ip6_ioam_trace.c		\
+	ioam/lib-trace/trace_util.h			\
+	ioam/lib-trace/trace_api.c
+
+BUILT_SOURCES +=					\
+	ioam/lib-trace/trace.api.h ioam/lib-trace/trace.py
+
+pyapi_DATA += ioam/lib-trace/trace.py
+
+noinst_HEADERS +=                       \
+  ioam/export/ioam_export_all_api_h.h   \
+  ioam/lib-trace/trace_all_api_h.h			\
+  ioam/lib-trace/trace_msg_enum.h			\
+  ioam/lib-trace/trace.api.h			\
+  ioam/lib-trace/trace_util.h
+
+ioam_trace_test_plugin_la_SOURCES =		\
+	ioam/lib-trace/trace_test.c			\
+	ioam/lib-trace/trace_plugin.api.h
+
+vppapitestplugins_LTLIBRARIES += ioam_trace_test_plugin.la
+vppplugins_LTLIBRARIES += ioam_trace_plugin.la
+
+
 
 # Remove *.la files
 install-data-hook:
diff --git a/plugins/ioam-plugin/ioam/encap/ip6_ioam_pot.c b/plugins/ioam-plugin/ioam/encap/ip6_ioam_pot.c
index ec85ea72e5f..9daea9c058f 100644
--- a/plugins/ioam-plugin/ioam/encap/ip6_ioam_pot.c
+++ b/plugins/ioam-plugin/ioam/encap/ip6_ioam_pot.c
@@ -199,10 +199,10 @@ ip6_hbh_ioam_proof_of_transit_pop_handler (ip6_header_t *ip,
   return (rv);
 }
 
-int ip6_hop_by_hop_ioam_pot_rewrite_handler (u8 *rewrite_string, u8 rewrite_size)
+int ip6_hop_by_hop_ioam_pot_rewrite_handler (u8 *rewrite_string, u8 *rewrite_size)
 {
   ioam_pot_option_t * pot_option;
-  if (rewrite_string && rewrite_size == sizeof(ioam_pot_option_t))
+  if (rewrite_string && *rewrite_size == sizeof(ioam_pot_option_t))
     {
       pot_option = (ioam_pot_option_t *)rewrite_string;
       pot_option->hdr.type = HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT
diff --git a/plugins/ioam-plugin/ioam/encap/ip6_ioam_trace.c b/plugins/ioam-plugin/ioam/encap/ip6_ioam_trace.c
new file mode 100644
index 00000000000..e151a96c010
--- /dev/null
+++ b/plugins/ioam-plugin/ioam/encap/ip6_ioam_trace.c
@@ -0,0 +1,391 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <vlib/vlib.h>
+#include <vnet/vnet.h>
+#include <vnet/pg/pg.h>
+#include <vppinfra/error.h>
+
+#include <vnet/ip/ip6.h>
+#include <vnet/ip/ip6_hop_by_hop.h>
+#include <vnet/ip/ip6_hop_by_hop_packet.h>
+
+#include <vppinfra/hash.h>
+#include <vppinfra/error.h>
+#include <vppinfra/elog.h>
+
+#include <ioam/lib-trace/trace_util.h>
+
+/* Timestamp precision multipliers for seconds, milliseconds, microseconds
+ * and nanoseconds respectively.
+ */
+static f64 trace_tsp_mul[4] = { 1, 1e3, 1e6, 1e9 };
+
+typedef union
+{
+  u64 as_u64;
+  u32 as_u32[2];
+} time_u64_t;
+
+
+extern ip6_hop_by_hop_ioam_main_t ip6_hop_by_hop_ioam_main;
+extern ip6_main_t ip6_main;
+
+#define foreach_ip6_hop_by_hop_ioam_trace_stats                                \
+  _(PROCESSED, "Pkts with ip6 hop-by-hop trace options")                        \
+  _(PROFILE_MISS, "Pkts with ip6 hop-by-hop trace options but no profile set") \
+  _(PASSED, "Pkts with TRACE in Policy")                                        \
+  _(FAILED, "Pkts with TRACE out of Policy")
+
+static char *ip6_hop_by_hop_ioam_trace_stats_strings[] = {
+#define _(sym,string) string,
+  foreach_ip6_hop_by_hop_ioam_trace_stats
+#undef _
+};
+
+typedef enum
+{
+#define _(sym,str) IP6_IOAM_TRACE_##sym,
+  foreach_ip6_hop_by_hop_ioam_trace_stats
+#undef _
+    IP6_IOAM_TRACE_N_STATS,
+} ip6_ioam_trace_stats_t;
+
+
+typedef struct
+{
+  /* stats */
+  u64 counters[ARRAY_LEN (ip6_hop_by_hop_ioam_trace_stats_strings)];
+
+  /* convenience */
+  vlib_main_t *vlib_main;
+  vnet_main_t *vnet_main;
+} ip6_hop_by_hop_ioam_trace_main_t;
+
+ip6_hop_by_hop_ioam_trace_main_t ip6_hop_by_hop_ioam_trace_main;
+
+always_inline void
+ip6_ioam_trace_stats_increment_counter (u32 counter_index, u64 increment)
+{
+  ip6_hop_by_hop_ioam_trace_main_t *hm = &ip6_hop_by_hop_ioam_trace_main;
+
+  hm->counters[counter_index] += increment;
+}
+
+
+static u8 *
+format_ioam_data_list_element (u8 * s, va_list * args)
+{
+  u32 *elt = va_arg (*args, u32 *);
+  u8 *trace_type_p = va_arg (*args, u8 *);
+  u8 trace_type = *trace_type_p;
+
+
+  if (trace_type & BIT_TTL_NODEID)
+    {
+      u32 ttl_node_id_host_byte_order = clib_net_to_host_u32 (*elt);
+      s = format (s, "ttl 0x%x node id 0x%x ",
+		  ttl_node_id_host_byte_order >> 24,
+		  ttl_node_id_host_byte_order & 0x00FFFFFF);
+
+      elt++;
+    }
+
+  if (trace_type & BIT_ING_INTERFACE && trace_type & BIT_ING_INTERFACE)
+    {
+      u32 ingress_host_byte_order = clib_net_to_host_u32 (*elt);
+      s = format (s, "ingress 0x%x egress 0x%x ",
+		  ingress_host_byte_order >> 16,
+		  ingress_host_byte_order & 0xFFFF);
+      elt++;
+    }
+
+  if (trace_type & BIT_TIMESTAMP)
+    {
+      u32 ts_in_host_byte_order = clib_net_to_host_u32 (*elt);
+      s = format (s, "ts 0x%x \n", ts_in_host_byte_order);
+      elt++;
+    }
+
+  if (trace_type & BIT_APPDATA)
+    {
+      u32 appdata_in_host_byte_order = clib_net_to_host_u32 (*elt);
+      s = format (s, "app 0x%x ", appdata_in_host_byte_order);
+      elt++;
+    }
+
+  return s;
+}
+
+
+int
+ioam_trace_get_sizeof_handler (u32 * result)
+{
+  u16 size = 0;
+  u8 trace_data_size = 0;
+  trace_profile *profile = NULL;
+
+  *result = 0;
+
+  profile = trace_profile_find ();
+
+  if (PREDICT_FALSE (!profile))
+    {
+      ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_PROFILE_MISS, 1);
+      return (-1);
+    }
+
+  trace_data_size = fetch_trace_data_size (profile->trace_type);
+  if (PREDICT_FALSE (trace_data_size == 0))
+    return VNET_API_ERROR_INVALID_VALUE;
+
+  if (PREDICT_FALSE (profile->num_elts * trace_data_size > 254))
+    return VNET_API_ERROR_INVALID_VALUE;
+
+  size += profile->num_elts * trace_data_size;
+  *result = size;
+
+  return 0;
+}
+
+
+
+int
+ip6_hop_by_hop_ioam_trace_rewrite_handler (u8 * rewrite_string,
+					   u8 * rewrite_size)
+{
+  ioam_trace_option_t *trace_option = NULL;
+  u8 trace_data_size = 0;
+  u8 trace_option_elts = 0;
+  trace_profile *profile = NULL;
+
+
+  profile = trace_profile_find ();
+
+  if (PREDICT_FALSE (!profile))
+    {
+      ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_PROFILE_MISS, 1);
+      return (-1);
+    }
+
+  if (PREDICT_FALSE (!rewrite_string))
+    return -1;
+
+  trace_option_elts = profile->num_elts;
+  trace_data_size = fetch_trace_data_size (profile->trace_type);
+  trace_option = (ioam_trace_option_t *) rewrite_string;
+  trace_option->hdr.type = HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST |
+    HBH_OPTION_TYPE_DATA_CHANGE_ENROUTE;
+  trace_option->hdr.length = 2 /*ioam_trace_type,data_list_elts_left */  +
+    trace_option_elts * trace_data_size;
+  trace_option->ioam_trace_type = profile->trace_type & TRACE_TYPE_MASK;
+  trace_option->data_list_elts_left = trace_option_elts;
+  *rewrite_size =
+    sizeof (ioam_trace_option_t) + (trace_option_elts * trace_data_size);
+
+  return 0;
+}
+
+
+int
+ip6_hbh_ioam_trace_data_list_handler (vlib_buffer_t * b, ip6_header_t * ip,
+				      ip6_hop_by_hop_option_t * opt)
+{
+  ip6_main_t *im = &ip6_main;
+  ip_lookup_main_t *lm = &im->lookup_main;
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
+  u8 elt_index = 0;
+  ioam_trace_option_t *trace = (ioam_trace_option_t *) opt;
+  u32 adj_index = vnet_buffer (b)->ip.adj_index[VLIB_TX];
+  ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index);
+  time_u64_t time_u64;
+  u32 *elt;
+  int rv = 0;
+  trace_profile *profile = NULL;
+
+
+  profile = trace_profile_find ();
+
+  if (PREDICT_FALSE (!profile))
+    {
+      ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_PROFILE_MISS, 1);
+      return (-1);
+    }
+
+
+  time_u64.as_u64 = 0;
+
+  if (PREDICT_TRUE (trace->data_list_elts_left))
+    {
+      trace->data_list_elts_left--;
+      /* fetch_trace_data_size returns in bytes. Convert it to 4-bytes
+       * to skip to this node's location.
+       */
+      elt_index =
+	trace->data_list_elts_left *
+	fetch_trace_data_size (trace->ioam_trace_type) / 4;
+      elt = &trace->elts[elt_index];
+      if (trace->ioam_trace_type & BIT_TTL_NODEID)
+	{
+	  *elt =
+	    clib_host_to_net_u32 ((ip->hop_limit << 24) | profile->node_id);
+	  elt++;
+	}
+
+      if (trace->ioam_trace_type & BIT_ING_INTERFACE)
+	{
+	  *elt =
+	    (vnet_buffer (b)->sw_if_index[VLIB_RX] & 0xFFFF) << 16 |
+	    (adj->rewrite_header.sw_if_index & 0xFFFF);
+	  *elt = clib_host_to_net_u32 (*elt);
+	  elt++;
+	}
+
+      if (trace->ioam_trace_type & BIT_TIMESTAMP)
+	{
+	  /* Send least significant 32 bits */
+	  f64 time_f64 =
+	    (f64) (((f64) hm->unix_time_0) +
+		   (vlib_time_now (hm->vlib_main) - hm->vlib_time_0));
+
+	  time_u64.as_u64 = time_f64 * trace_tsp_mul[profile->trace_tsp];
+	  *elt = clib_host_to_net_u32 (time_u64.as_u32[0]);
+	  elt++;
+	}
+
+      if (trace->ioam_trace_type & BIT_APPDATA)
+	{
+	  /* $$$ set elt0->app_data */
+	  *elt = clib_host_to_net_u32 (profile->app_data);
+	  elt++;
+	}
+      ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_PASSED, 1);
+    }
+  return (rv);
+}
+
+u8 *
+ip6_hbh_ioam_trace_data_list_trace_handler (u8 * s,
+					    ip6_hop_by_hop_option_t * opt)
+{
+  ioam_trace_option_t *trace;
+  u8 trace_data_size_in_words = 0;
+  u32 *elt;
+  int elt_index = 0;
+
+  trace = (ioam_trace_option_t *) opt;
+#if 0
+  s =
+    format (s, "  Trace Type 0x%x , %d elts left ts msb(s) 0x%x\n",
+	    trace->ioam_trace_type, trace->data_list_elts_left,
+	    t->timestamp_msbs);
+#endif
+  s =
+    format (s, "  Trace Type 0x%x , %d elts left\n", trace->ioam_trace_type,
+	    trace->data_list_elts_left);
+  trace_data_size_in_words =
+    fetch_trace_data_size (trace->ioam_trace_type) / 4;
+  elt = &trace->elts[0];
+  while ((u8 *) elt < ((u8 *) (&trace->elts[0]) + trace->hdr.length - 2
+		       /* -2 accounts for ioam_trace_type,elts_left */ ))
+    {
+      s = format (s, "    [%d] %U\n", elt_index,
+		  format_ioam_data_list_element,
+		  elt, &trace->ioam_trace_type);
+      elt_index++;
+      elt += trace_data_size_in_words;
+    }
+  return (s);
+}
+
+
+static clib_error_t *
+ip6_show_ioam_trace_cmd_fn (vlib_main_t * vm,
+			    unformat_input_t * input,
+			    vlib_cli_command_t * cmd)
+{
+  ip6_hop_by_hop_ioam_trace_main_t *hm = &ip6_hop_by_hop_ioam_trace_main;
+  u8 *s = 0;
+  int i = 0;
+
+  for (i = 0; i < IP6_IOAM_TRACE_N_STATS; i++)
+    {
+      s =
+	format (s, " %s - %lu\n", ip6_hop_by_hop_ioam_trace_stats_strings[i],
+		hm->counters[i]);
+    }
+
+  vlib_cli_output (vm, "%v", s);
+  vec_free (s);
+  return 0;
+}
+
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip6_show_ioam_trace_cmd, static) = {
+  .path = "show ioam trace",
+  .short_help = "iOAM trace statistics",
+  .function = ip6_show_ioam_trace_cmd_fn,
+};
+/* *INDENT-ON* */
+
+
+static clib_error_t *
+ip6_hop_by_hop_ioam_trace_init (vlib_main_t * vm)
+{
+  ip6_hop_by_hop_ioam_trace_main_t *hm = &ip6_hop_by_hop_ioam_trace_main;
+  clib_error_t *error;
+
+  if ((error = vlib_call_init_function (vm, ip_main_init)))
+    return (error);
+
+  if ((error = vlib_call_init_function (vm, ip6_lookup_init)))
+    return error;
+
+  if ((error = vlib_call_init_function (vm, ip6_hop_by_hop_ioam_init)))
+    return (error);
+
+  hm->vlib_main = vm;
+  hm->vnet_main = vnet_get_main ();
+  memset (hm->counters, 0, sizeof (hm->counters));
+
+
+  if (ip6_hbh_register_option
+      (HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST,
+       ip6_hbh_ioam_trace_data_list_handler,
+       ip6_hbh_ioam_trace_data_list_trace_handler) < 0)
+    return (clib_error_create
+	    ("registration of HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST failed"));
+
+
+  if (ip6_hbh_add_register_option (HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST,
+				   sizeof (ioam_trace_option_t),
+				   ip6_hop_by_hop_ioam_trace_rewrite_handler)
+      < 0)
+    return (clib_error_create
+	    ("registration of HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST for rewrite failed"));
+
+
+  return (0);
+}
+
+VLIB_INIT_FUNCTION (ip6_hop_by_hop_ioam_trace_init);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace.api b/plugins/ioam-plugin/ioam/lib-trace/trace.api
new file mode 100644
index 00000000000..b5886b2a275
--- /dev/null
+++ b/plugins/ioam-plugin/ioam/lib-trace/trace.api
@@ -0,0 +1,65 @@
+/* Hey Emacs use -*- mode: C -*- */
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/** \brief iOAM6 Trace - Set the iOAM6 trace profile
+    @param id - id of the profile
+    @param trace_type - Type of trace requested
+    @param num_elts - Number of trace elements to be inserted
+    @param node_id - Trace Node ID
+    @param trace_tsp- Timestamp resolution
+    @param app_data - Application specific opaque
+*/
+define trace_profile_add {
+  u32 client_index;
+  u32 context;
+  u8 trace_type;
+  u8 num_elts;
+  u32 node_id;
+  u32 trace_tsp;
+  u32 app_data;
+};
+
+/** \brief Trace profile add / del response
+    @param context - sender context, to match reply w/ request
+    @param retval - return value for request
+*/
+define trace_profile_add_reply {
+    u32 context;
+    i32 retval;
+};
+
+
+
+/** \brief Delete trace Profile
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param list_name_len - length of the name of the profile list
+    @param list_name - name of profile list to delete
+*/
+define trace_profile_del {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief Trace profile add / del response
+    @param context - sender context, to match reply w/ request
+    @param retval - return value for request
+*/
+define trace_profile_del_reply {
+    u32 context;
+    i32 retval;
+};
diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace_all_api_h.h b/plugins/ioam-plugin/ioam/lib-trace/trace_all_api_h.h
new file mode 100644
index 00000000000..223f95450aa
--- /dev/null
+++ b/plugins/ioam-plugin/ioam/lib-trace/trace_all_api_h.h
@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Include the generated file, see BUILT_SOURCES in Makefile.am */
+#include <ioam/lib-trace/trace.api.h>
diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace_api.c b/plugins/ioam-plugin/ioam/lib-trace/trace_api.c
new file mode 100644
index 00000000000..1b8ad3cc0d6
--- /dev/null
+++ b/plugins/ioam-plugin/ioam/lib-trace/trace_api.c
@@ -0,0 +1,210 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ *------------------------------------------------------------------
+ * trace_api.c - iOAM Trace related APIs to create
+ *               and maintain profiles
+ *------------------------------------------------------------------
+ */
+
+#include <vnet/vnet.h>
+#include <vnet/plugin/plugin.h>
+#include <ioam/lib-trace/trace_util.h>
+
+#include <vlibapi/api.h>
+#include <vlibmemory/api.h>
+#include <vlibsocket/api.h>
+
+/* define message IDs */
+#include <ioam/lib-trace/trace_msg_enum.h>
+
+/* define message structures */
+#define vl_typedefs
+#include <ioam/lib-trace/trace_all_api_h.h>
+#undef vl_typedefs
+
+/* define generated endian-swappers */
+#define vl_endianfun
+#include <ioam/lib-trace/trace_all_api_h.h>
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
+#define vl_printfun
+#include <ioam/lib-trace/trace_all_api_h.h>
+#undef vl_printfun
+
+/* Get the API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include <ioam/lib-trace/trace_all_api_h.h>
+#undef vl_api_version
+
+/*
+ * A handy macro to set up a message reply.
+ * Assumes that the following variables are available:
+ * mp - pointer to request message
+ * rmp - pointer to reply message type
+ * rv - return value
+ */
+
+#define TRACE_REPLY_MACRO(t)                                          \
+do {                                                            \
+    unix_shared_memory_queue_t * q =                            \
+    vl_api_client_index_to_input_queue (mp->client_index);      \
+    if (!q)                                                     \
+        return;                                                 \
+                                                                \
+    rmp = vl_msg_api_alloc (sizeof (*rmp));                     \
+    rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base);               \
+    rmp->context = mp->context;                                 \
+    rmp->retval = ntohl(rv);                                    \
+                                                                \
+    vl_msg_api_send_shmem (q, (u8 *)&rmp);                      \
+} while(0);
+
+/* *INDENT-OFF* */
+#define TRACE_REPLY_MACRO2(t, body)                                   \
+do {                                                            \
+    unix_shared_memory_queue_t * q;                             \
+    rv = vl_msg_api_pd_handler (mp, rv);                        \
+    q = vl_api_client_index_to_input_queue (mp->client_index);  \
+    if (!q)                                                     \
+        return;                                                 \
+                                                                \
+    rmp = vl_msg_api_alloc (sizeof (*rmp));                     \
+    rmp->_vl_msg_id = ntohs((t));                               \
+    rmp->context = mp->context;                                 \
+    rmp->retval = ntohl(rv);                                    \
+    do {body;} while (0);                                       \
+    vl_msg_api_send_shmem (q, (u8 *)&rmp);                      \
+} while(0);
+/* *INDENT-ON* */
+
+/* List of message types that this plugin understands */
+
+#define foreach_trace_plugin_api_msg                                      \
+_(TRACE_PROFILE_ADD, trace_profile_add)                                     \
+_(TRACE_PROFILE_DEL, trace_profile_del)                                     \
+
+static void vl_api_trace_profile_add_t_handler
+  (vl_api_trace_profile_add_t * mp)
+{
+  trace_main_t *sm = &trace_main;
+  int rv = 0;
+  vl_api_trace_profile_add_reply_t *rmp;
+  trace_profile *profile = NULL;
+  u8 *name = 0;
+
+  profile = trace_profile_find ();
+  if (profile)
+    {
+      rv =
+	trace_profile_create (profile, mp->trace_type, mp->num_elts,
+			      mp->trace_tsp, mp->node_id, mp->app_data);
+      if (rv != 0)
+	goto ERROROUT;
+    }
+  else
+    {
+      rv = -3;
+    }
+ERROROUT:
+  vec_free (name);
+  TRACE_REPLY_MACRO (VL_API_TRACE_PROFILE_ADD_REPLY);
+}
+
+
+static void vl_api_trace_profile_del_t_handler
+  (vl_api_trace_profile_del_t * mp)
+{
+  trace_main_t *sm = &trace_main;
+  int rv = 0;
+  vl_api_trace_profile_del_reply_t *rmp;
+
+  clear_trace_profiles ();
+
+  TRACE_REPLY_MACRO (VL_API_TRACE_PROFILE_DEL_REPLY);
+}
+
+
+/*
+ * This routine exists to convince the vlib plugin framework that
+ * we haven't accidentally copied a random .dll into the plugin directory.
+ *
+ * Also collects global variable pointers passed from the vpp engine
+ */
+
+clib_error_t *
+vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h,
+		      int from_early_init)
+{
+  trace_main_t *sm = &trace_main;
+  clib_error_t *error = 0;
+
+  sm->vlib_main = vm;
+  sm->vnet_main = h->vnet_main;
+  return error;
+}
+
+/* Set up the API message handling tables */
+static clib_error_t *
+trace_plugin_api_hookup (vlib_main_t * vm)
+{
+  trace_main_t *sm = &trace_main;
+#define _(N,n)                                                  \
+    vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base),     \
+                           #n,					\
+                           vl_api_##n##_t_handler,              \
+                           vl_noop_handler,                     \
+                           vl_api_##n##_t_endian,               \
+                           vl_api_##n##_t_print,                \
+                           sizeof(vl_api_##n##_t), 1);
+  foreach_trace_plugin_api_msg;
+#undef _
+
+  return 0;
+}
+
+static clib_error_t *
+trace_init (vlib_main_t * vm)
+{
+  trace_main_t *sm = &trace_main;
+  clib_error_t *error = 0;
+  u8 *name;
+
+  bzero (sm, sizeof (trace_main));
+  (void) trace_util_init ();
+  name = format (0, "trace_%08x%c", api_version, 0);
+
+  /* Ask for a correctly-sized block of API message decode slots */
+  sm->msg_id_base = vl_msg_api_get_msg_ids
+    ((char *) name, VL_MSG_FIRST_AVAILABLE);
+
+  error = trace_plugin_api_hookup (vm);
+
+  vec_free (name);
+
+  return error;
+}
+
+VLIB_INIT_FUNCTION (trace_init);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace_msg_enum.h b/plugins/ioam-plugin/ioam/lib-trace/trace_msg_enum.h
new file mode 100644
index 00000000000..78c35665f3c
--- /dev/null
+++ b/plugins/ioam-plugin/ioam/lib-trace/trace_msg_enum.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef included_trace_msg_enum_h
+#define included_trace_msg_enum_h
+
+#include <vppinfra/byte_order.h>
+
+#define vl_msg_id(n,h) n,
+typedef enum {
+#include <ioam/lib-trace/trace_all_api_h.h>
+    /* We'll want to know how many messages IDs we need... */
+    VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+#endif /* included_trace_msg_enum_h */
diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace_test.c b/plugins/ioam-plugin/ioam/lib-trace/trace_test.c
new file mode 100644
index 00000000000..dc3851bec16
--- /dev/null
+++ b/plugins/ioam-plugin/ioam/lib-trace/trace_test.c
@@ -0,0 +1,251 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ *------------------------------------------------------------------
+ * trace_test.c - test harness for trace plugin
+ *------------------------------------------------------------------
+ */
+
+#include <vat/vat.h>
+#include <vlibapi/api.h>
+#include <vlibmemory/api.h>
+#include <vlibsocket/api.h>
+#include <vppinfra/error.h>
+
+/* Declare message IDs */
+#include <ioam/lib-trace/trace_msg_enum.h>
+
+/* define message structures */
+#define vl_typedefs
+#include <ioam/lib-trace/trace_all_api_h.h>
+#undef vl_typedefs
+
+/* declare message handlers for each api */
+
+#define vl_endianfun		/* define message structures */
+#include <ioam/lib-trace/trace_all_api_h.h>
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...)
+#define vl_printfun
+#include <ioam/lib-trace/trace_all_api_h.h>
+#undef vl_printfun
+
+/* Get the API version number. */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include <ioam/lib-trace/trace_all_api_h.h>
+#undef vl_api_version
+
+
+typedef struct
+{
+  /* API message ID base */
+  u16 msg_id_base;
+  vat_main_t *vat_main;
+} trace_test_main_t;
+
+trace_test_main_t trace_test_main;
+
+#define foreach_standard_reply_retval_handler     \
+_(trace_profile_add_reply)                          \
+_(trace_profile_del_reply)
+
+#define _(n)                                            \
+    static void vl_api_##n##_t_handler                  \
+    (vl_api_##n##_t * mp)                               \
+    {                                                   \
+        vat_main_t * vam = trace_test_main.vat_main;   \
+        i32 retval = ntohl(mp->retval);                 \
+        if (vam->async_mode) {                          \
+            vam->async_errors += (retval < 0);          \
+        } else {                                        \
+            vam->retval = retval;                       \
+            vam->result_ready = 1;                      \
+        }                                               \
+    }
+foreach_standard_reply_retval_handler;
+#undef _
+
+/*
+ * Table of message reply handlers, must include boilerplate handlers
+ * we just generated
+ */
+#define foreach_vpe_api_reply_msg                                       \
+_(TRACE_PROFILE_ADD_REPLY, trace_profile_add_reply)                         \
+_(TRACE_PROFILE_DEL_REPLY, trace_profile_del_reply)                         \
+
+
+/* M: construct, but don't yet send a message */
+
+#define M(T,t)                                                  \
+do {                                                            \
+    vam->result_ready = 0;                                      \
+    mp = vl_msg_api_alloc(sizeof(*mp));                         \
+    memset (mp, 0, sizeof (*mp));                               \
+    mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base);      \
+    mp->client_index = vam->my_client_index;                    \
+} while(0);
+
+#define M2(T,t,n)                                               \
+do {                                                            \
+    vam->result_ready = 0;                                      \
+    mp = vl_msg_api_alloc(sizeof(*mp)+(n));                     \
+    memset (mp, 0, sizeof (*mp));                               \
+    mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base);      \
+    mp->client_index = vam->my_client_index;                    \
+} while(0);
+
+/* S: send a message */
+#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp))
+
+/* W: wait for results, with timeout */
+#define W                                       \
+do {                                            \
+    timeout = vat_time_now (vam) + 1.0;         \
+                                                \
+    while (vat_time_now (vam) < timeout) {      \
+        if (vam->result_ready == 1) {           \
+            return (vam->retval);               \
+        }                                       \
+    }                                           \
+    return -99;                                 \
+} while(0);
+
+
+static int
+api_trace_profile_add (vat_main_t * vam)
+{
+  trace_test_main_t *sm = &trace_test_main;
+  unformat_input_t *input = vam->input;
+  vl_api_trace_profile_add_t *mp;
+  u8 trace_type = 0;
+  u8 num_elts = 0;
+  int rv = 0;
+  u32 node_id = 0;
+  u32 app_data = 0;
+  u32 trace_tsp = 0;
+  f64 timeout;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "trace-type 0x%x", &trace_type))
+	;
+      else if (unformat (input, "trace-elts %d", &num_elts))
+	;
+      else if (unformat (input, "trace-tsp %d", &trace_tsp))
+	;
+      else if (unformat (input, "node-id 0x%x", &node_id))
+	;
+      else if (unformat (input, "app-data 0x%x", &app_data))
+	;
+
+      else
+	break;
+    }
+
+
+  M (TRACE_PROFILE_ADD, trace_profile_add);
+
+  mp->trace_type = trace_type;
+  mp->trace_tsp = ntohl (trace_tsp);
+  mp->node_id = ntohl (node_id);
+  mp->app_data = ntohl (app_data);
+  mp->num_elts = num_elts;
+
+  S;
+  W;
+
+  return (rv);
+}
+
+
+
+static int
+api_trace_profile_del (vat_main_t * vam)
+{
+  trace_test_main_t *sm = &trace_test_main;
+  vl_api_trace_profile_del_t *mp;
+  f64 timeout;
+
+  M (TRACE_PROFILE_DEL, trace_profile_del);
+  S;
+  W;
+  return 0;
+}
+
+/*
+ * List of messages that the api test plugin sends,
+ * and that the data plane plugin processes
+ */
+#define foreach_vpe_api_msg \
+_(trace_profile_add, ""\
+  "trace-type <0x1f|0x3|0x9|0x11|0x19> trace-elts <nn> trace-tsp <0|1|2|3> node-id <node id in hex> app-data <app_data in hex>")  \
+_(trace_profile_del, "[id <nn>]")                    \
+
+
+void
+vat_api_hookup (vat_main_t * vam)
+{
+  trace_test_main_t *sm = &trace_test_main;
+  /* Hook up handlers for replies from the data plane plug-in */
+#define _(N,n)                                                  \
+    vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base),     \
+                           #n,                                  \
+                           vl_api_##n##_t_handler,              \
+                           vl_noop_handler,                     \
+                           vl_api_##n##_t_endian,               \
+                           vl_api_##n##_t_print,                \
+                           sizeof(vl_api_##n##_t), 1);
+  foreach_vpe_api_reply_msg;
+#undef _
+
+  /* API messages we can send */
+#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n);
+  foreach_vpe_api_msg;
+#undef _
+
+  /* Help strings */
+#define _(n,h) hash_set_mem (vam->help_by_name, #n, h);
+  foreach_vpe_api_msg;
+#undef _
+}
+
+clib_error_t *
+vat_plugin_register (vat_main_t * vam)
+{
+  trace_test_main_t *sm = &trace_test_main;
+  u8 *name;
+
+  sm->vat_main = vam;
+
+  name = format (0, "trace_%08x%c", api_version, 0);
+  sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
+
+  if (sm->msg_id_base != (u16) ~ 0)
+    vat_api_hookup (vam);
+
+  vec_free (name);
+
+  return 0;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace_util.c b/plugins/ioam-plugin/ioam/lib-trace/trace_util.c
new file mode 100644
index 00000000000..4bc9f84deec
--- /dev/null
+++ b/plugins/ioam-plugin/ioam/lib-trace/trace_util.c
@@ -0,0 +1,221 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <vnet/vnet.h>
+#include <stdint.h>
+#include <time.h>
+#include <string.h>
+#include <vppinfra/mem.h>
+#include "trace_util.h"
+
+trace_main_t trace_main;
+
+extern ip6_hop_by_hop_ioam_main_t ip6_hop_by_hop_ioam_main;
+
+static int
+trace_profile_cleanup (trace_profile * profile)
+{
+  int rv;
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
+
+  memset (profile, 0, sizeof (trace_profile));
+  profile->trace_tsp = TSP_MICROSECONDS;	/* Micro seconds */
+  hm->options_size[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST] = 0;
+  if (0 !=
+      (rv =
+       ip6_ioam_set_rewrite (&hm->rewrite, hm->has_trace_option,
+			     hm->has_pot_option, hm->has_ppc_option)))
+    return (-1);
+  return 0;
+
+}
+
+static int
+trace_main_profiles_reset (void)
+{
+  int rv;
+
+  trace_main_t *sm = &trace_main;
+  rv = trace_profile_cleanup (&(sm->profile));
+  return (rv);
+}
+
+int
+trace_util_init (void)
+{
+  int rv;
+
+  rv = trace_main_profiles_reset ();
+  return (rv);
+}
+
+
+int
+trace_profile_create (trace_profile * profile, u8 trace_type, u8 num_elts,
+		      u32 trace_tsp, u32 node_id, u32 app_data)
+{
+  u32 trace_size = 0;
+  int rv;
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
+
+  if (profile && !profile->valid)
+    {
+      //rv = trace_profile_cleanup (profile);
+      profile->trace_type = trace_type;
+      profile->num_elts = num_elts;
+      profile->trace_tsp = trace_tsp;
+      profile->node_id = node_id;
+      profile->app_data = app_data;
+      profile->valid = 1;
+
+      if (ioam_trace_get_sizeof_handler (&trace_size) < 0)
+	return (-1);
+
+      hm->options_size[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST] = trace_size;
+
+      if (hm->has_trace_option)
+	{
+	  if (0 !=
+	      (rv =
+	       ip6_ioam_set_rewrite (&hm->rewrite, hm->has_trace_option,
+				     hm->has_pot_option, hm->has_ppc_option)))
+	    return (-1);
+
+	}
+      return (0);
+    }
+
+  return (-1);
+}
+
+
+
+clib_error_t *
+clear_trace_profile_command_fn (vlib_main_t * vm,
+				unformat_input_t * input,
+				vlib_cli_command_t * cmd)
+{
+
+  trace_main_profiles_reset ();
+  return 0;
+}
+
+void
+clear_trace_profiles (void)
+{
+  clear_trace_profile_command_fn (0, 0, 0);
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND(clear_trace_profile_command) =
+{
+.path = "clear ioam-trace profile",
+.short_help = "clear ioam-trace profile [<index>|all]",
+.function = clear_trace_profile_command_fn,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+set_trace_profile_command_fn (vlib_main_t * vm,
+			      unformat_input_t * input,
+			      vlib_cli_command_t * cmd)
+{
+  u8 trace_type = 0;
+  u8 num_elts = 0;
+  u32 node_id = 0;
+  u32 app_data = 0;
+  u32 trace_tsp = 0;
+  trace_profile *profile = NULL;
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "trace-type 0x%x", &trace_type));
+      else if (unformat (input, "trace-elts %d", &num_elts));
+      else if (unformat (input, "trace-tsp %d", &trace_tsp));
+      else if (unformat (input, "node-id 0x%x", &node_id));
+      else if (unformat (input, "app-data 0x%x", &app_data));
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+  profile = trace_profile_find ();
+  if (profile)
+    {
+      trace_profile_create (profile, trace_type, num_elts, trace_tsp,
+			    node_id, app_data);
+    }
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (set_trace_profile_command) =
+{
+.path = "set ioam-trace profile",.short_help = "set ioam-trace \
+                  trace-type <0x1f|0x3|0x9|0x11|0x19> trace-elts <nn> trace-tsp <0|1|2|3> node-id <node id in hex> app-data <app_data in hex>",.function = set_trace_profile_command_fn,};
+
+/* *INDENT-ON* */
+
+static clib_error_t *
+show_trace_profile_command_fn (vlib_main_t * vm,
+			       unformat_input_t * input,
+			       vlib_cli_command_t * cmd)
+{
+  trace_profile *p = NULL;
+  u8 *s = 0;
+  p = trace_profile_find ();
+  if (p->valid == 0)
+    return 0;
+  s = format (s, " HOP BY HOP OPTIONS - TRACE CONFIG - \n");
+  s = format (s, "                        Trace Type : 0x%x (%d)\n",
+	      p->trace_type, p->trace_type);
+  s =
+    format (s, "         Trace timestamp precision : %d (%s)\n",
+	    p->trace_tsp,
+	    (p->trace_tsp ==
+	     TSP_SECONDS) ? "Seconds" : ((p->trace_tsp ==
+					  TSP_MILLISECONDS) ?
+					 "Milliseconds"
+					 : (((p->trace_tsp ==
+					      TSP_MICROSECONDS) ?
+					     "Microseconds" :
+					     "Nanoseconds"))));
+  s = format (s, "                Num of trace nodes : %d\n", p->num_elts);
+  s =
+    format (s, "                           Node-id : 0x%x (%d)\n",
+	    p->node_id, p->node_id);
+  s =
+    format (s, "                          App Data : 0x%x (%d)\n",
+	    p->app_data, p->app_data);
+  if (!(p && p->valid))
+    {
+      s = format (s, "\nTrace configuration not valid\n");
+    }
+  vlib_cli_output (vm, "%v", s);
+  vec_free (s);
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_trace_profile_command) =
+{
+.path = "show ioam-trace profile",.short_help =
+    "show ioam-trace profile",.function = show_trace_profile_command_fn,};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace_util.h b/plugins/ioam-plugin/ioam/lib-trace/trace_util.h
new file mode 100644
index 00000000000..4528b18a8b0
--- /dev/null
+++ b/plugins/ioam-plugin/ioam/lib-trace/trace_util.h
@@ -0,0 +1,250 @@
+/*
+ * trace_util.h -- Trace Profile Utility header
+ *
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef include_vnet_trace_util_h
+#define include_vnet_trace_util_h
+
+#include <vnet/ip/ip6_hop_by_hop.h>
+#define debug_ioam debug_ioam_fn
+
+
+/**
+ * Usage:
+ *
+ * On any node that participates in iOAM Trace.
+ *
+ * Step 1: Initialize this library by calling trace_init()
+ * Step 2: Setup a trace  profile that contains all the parameters needed to compute cumulative:
+ *         Call these functions:
+ *         trace_profile_find
+ *         trace_profile_create
+ * Step 2a: On initial node enable the profile to be used:
+ *          trace_profile_set_active / trace_profile_get_active will return the profile
+ * Step 4: TBD
+ *         trace_validate
+ *
+ */
+
+typedef struct trace_profile_
+{
+  u8 valid:1;
+  u8 trace_type;
+  u8 num_elts;
+  /* Configured node-id */
+  u32 node_id;
+  u32 app_data;
+  u32 trace_tsp;
+} trace_profile;
+
+typedef struct
+{
+  /* Name of the default profile list in use */
+  trace_profile profile;
+
+  /* API message ID base */
+  u16 msg_id_base;
+
+  /* convenience */
+  vlib_main_t *vlib_main;
+  vnet_main_t *vnet_main;
+} trace_main_t;
+
+extern trace_main_t trace_main;
+
+/*
+ * Initialize Trace profile
+ */
+int trace_util_init (void);
+
+
+/*
+ * Find a trace profile by ID
+ */
+always_inline trace_profile *trace_profile_find (void);
+
+always_inline trace_profile *
+trace_profile_find (void)
+{
+  trace_main_t *sm = &trace_main;
+
+  return (&(sm->profile));
+}
+
+
+/* setup and clean up profile */
+int trace_profile_create (trace_profile * profile, u8 trace_type, u8 num_elts,
+			  u32 trace_tsp, u32 node_id, u32 app_data);
+
+void clear_trace_profiles (void);
+
+
+
+#define    BIT_TTL_NODEID       (1<<0)
+#define    BIT_ING_INTERFACE    (1<<1)
+#define    BIT_EGR_INTERFACE    (1<<2)
+#define    BIT_TIMESTAMP        (1<<3)
+#define    BIT_APPDATA          (1<<4)
+#define    TRACE_TYPE_MASK      0x1F	/* Mask of all above bits */
+
+/*
+     0x00011111  iOAM-trace-type is 0x00011111 then the format of node
+        data is:
+
+          0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |   Hop_Lim     |              node_id                          |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |     ingress_if_id             |         egress_if_id          |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         +                           timestamp                           +
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |                            app_data                           |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+*/
+#define   TRACE_TYPE_IF_TS_APP   0x1f
+typedef struct
+{
+  u32 ttl_node_id;
+  u16 ingress_if;
+  u16 egress_if;
+  u32 timestamp;
+  u32 app_data;
+} ioam_trace_if_ts_app_t;
+
+/*
+     0x00000111  iOAM-trace-type is 0x00000111 then the format is:
+
+          0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |   Hop_Lim     |              node_id                          |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |     ingress_if_id             |         egress_if_id          |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+*/
+
+#define   TRACE_TYPE_IF   0x03
+typedef struct
+{
+  u32 ttl_node_id;
+  u16 ingress_if;
+  u16 egress_if;
+} ioam_trace_if_t;
+
+/*
+     0x00001001  iOAM-trace-type is 0x00001001 then the format is:
+
+          0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |   Hop_Lim     |              node_id                          |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         +                           timestamp                           +
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+*/
+
+#define   TRACE_TYPE_TS   0x09
+typedef struct
+{
+  u32 ttl_node_id;
+  u32 timestamp;
+} ioam_trace_ts_t;
+
+/*
+     0x00010001  iOAM-trace-type is 0x00010001 then the format is:
+
+
+          0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |   Hop_Lim     |              node_id                          |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |                            app_data                           |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+*/
+
+
+#define   TRACE_TYPE_APP   0x11
+typedef struct
+{
+  u32 ttl_node_id;
+  u32 app_data;
+} ioam_trace_app_t;
+
+/*
+
+     0x00011001  iOAM-trace-type is 0x00011001 then the format is:
+
+          0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |   Hop_Lim     |              node_id                          |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         +                           timestamp                           +
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |                            app_data                           |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*/
+
+#define   TRACE_TYPE_TS_APP   0x19
+typedef struct
+{
+  u32 ttl_node_id;
+  u32 timestamp;
+  u32 app_data;
+} ioam_trace_ts_app_t;
+
+
+/* *INDENT-OFF* */
+typedef CLIB_PACKED(struct {
+  ip6_hop_by_hop_option_t hdr;
+  u8 ioam_trace_type;
+  u8 data_list_elts_left;
+  u32 elts[0]; /* Variable type. So keep it generic */
+}) ioam_trace_option_t;
+/* *INDENT-ON* */
+
+
+static inline u8
+fetch_trace_data_size (u8 trace_type)
+{
+  u8 trace_data_size = 0;
+
+  if (trace_type == TRACE_TYPE_IF_TS_APP)
+    trace_data_size = sizeof (ioam_trace_if_ts_app_t);
+  else if (trace_type == TRACE_TYPE_IF)
+    trace_data_size = sizeof (ioam_trace_if_t);
+  else if (trace_type == TRACE_TYPE_TS)
+    trace_data_size = sizeof (ioam_trace_ts_t);
+  else if (trace_type == TRACE_TYPE_APP)
+    trace_data_size = sizeof (ioam_trace_app_t);
+  else if (trace_type == TRACE_TYPE_TS_APP)
+    trace_data_size = sizeof (ioam_trace_ts_app_t);
+
+  return trace_data_size;
+}
+
+int ioam_trace_get_sizeof_handler (u32 * result);
+#endif
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/vnet/vnet/ip/ip6_hop_by_hop.c b/vnet/vnet/ip/ip6_hop_by_hop.c
index 7038556848c..2a037033d13 100644
--- a/vnet/vnet/ip/ip6_hop_by_hop.c
+++ b/vnet/vnet/ip/ip6_hop_by_hop.c
@@ -25,180 +25,33 @@
 
 #include <vnet/ip/ip6_hop_by_hop.h>
 
-/* Timestamp precision multipliers for seconds, milliseconds, microseconds
- * and nanoseconds respectively.
- */
-static f64 trace_tsp_mul[4] = {1, 1e3, 1e6, 1e9};
-
-char *ppc_state[] = {"None", "Encap", "Decap"};
+char *ppc_state[] = { "None", "Encap", "Decap" };
 
 ip6_hop_by_hop_ioam_main_t ip6_hop_by_hop_ioam_main;
 
 #define foreach_ip6_hbyh_ioam_input_next	\
   _(IP6_REWRITE, "ip6-rewrite")			\
   _(IP6_LOOKUP, "ip6-lookup")			\
-  _(DROP, "error-drop")                        
+  _(DROP, "error-drop")
 
-typedef enum {
+typedef enum
+{
 #define _(s,n) IP6_HBYH_IOAM_INPUT_NEXT_##s,
   foreach_ip6_hbyh_ioam_input_next
 #undef _
-  IP6_HBYH_IOAM_INPUT_N_NEXT,
+    IP6_HBYH_IOAM_INPUT_N_NEXT,
 } ip6_hbyh_ioam_input_next_t;
 
-typedef union {
-    u64 as_u64;
-    u32 as_u32[2];
-} time_u64_t;
-
-static inline u8
-fetch_trace_data_size(u8 trace_type)
-{
-  u8 trace_data_size = 0;
-
-  if (trace_type == TRACE_TYPE_IF_TS_APP)   
-      trace_data_size = sizeof(ioam_trace_if_ts_app_t);
-  else if(trace_type == TRACE_TYPE_IF)      
-      trace_data_size = sizeof(ioam_trace_if_t);
-  else if(trace_type == TRACE_TYPE_TS)      
-      trace_data_size = sizeof(ioam_trace_ts_t);
-  else if(trace_type == TRACE_TYPE_APP)     
-      trace_data_size = sizeof(ioam_trace_app_t);
-  else if(trace_type == TRACE_TYPE_TS_APP)  
-      trace_data_size = sizeof(ioam_trace_ts_app_t);
-
-  return trace_data_size;
-}
-
-static u8 * format_ioam_data_list_element (u8 * s, va_list * args)
-{ 
-  u32 *elt = va_arg (*args, u32 *);
-  u8  *trace_type_p = va_arg (*args, u8 *);
-  u8  trace_type = *trace_type_p;
 
 
-  if (trace_type & BIT_TTL_NODEID)
-    {
-      u32 ttl_node_id_host_byte_order = clib_net_to_host_u32 (*elt);
-      s = format (s, "ttl 0x%x node id 0x%x ",
-              ttl_node_id_host_byte_order>>24,
-              ttl_node_id_host_byte_order & 0x00FFFFFF);
-
-      elt++;
-    }
- 
-  if (trace_type & BIT_ING_INTERFACE && trace_type & BIT_ING_INTERFACE)
-    {
-        u32 ingress_host_byte_order = clib_net_to_host_u32(*elt);
-        s = format (s, "ingress 0x%x egress 0x%x ", 
-                   ingress_host_byte_order >> 16, 
-                   ingress_host_byte_order &0xFFFF);
-        elt++;
-    }
- 
-  if (trace_type & BIT_TIMESTAMP)
-    {
-        u32 ts_in_host_byte_order = clib_net_to_host_u32 (*elt);
-        s = format (s, "ts 0x%x \n", ts_in_host_byte_order);
-        elt++;
-    }
- 
-  if (trace_type & BIT_APPDATA)
-    {
-        u32 appdata_in_host_byte_order = clib_net_to_host_u32 (*elt);
-        s = format (s, "app 0x%x ", appdata_in_host_byte_order);
-        elt++;
-    }
- 
-  return s;
-}
-
-u8 *
-ip6_hbh_ioam_trace_data_list_trace_handler (u8 *s, ip6_hop_by_hop_option_t *opt)
-{
-  ioam_trace_option_t *trace;
-  u8 trace_data_size_in_words = 0;
-  u32 *elt;
-  int elt_index = 0;
-
-  trace = (ioam_trace_option_t *)opt;
-#if 0
-  s = format (s, "  Trace Type 0x%x , %d elts left ts msb(s) 0x%x\n", trace->ioam_trace_type, trace->data_list_elts_left,
-	      t->timestamp_msbs);
-#endif
-  s = format (s, "  Trace Type 0x%x , %d elts left\n", trace->ioam_trace_type, trace->data_list_elts_left);
-  trace_data_size_in_words = fetch_trace_data_size(trace->ioam_trace_type)/4;
-  elt = &trace->elts[0];
-  while ((u8 *) elt < ((u8 *)(&trace->elts[0]) + trace->hdr.length - 2
-			/* -2 accounts for ioam_trace_type,elts_left */)) {
-    s = format (s, "    [%d] %U\n",elt_index,
-		format_ioam_data_list_element,
-		elt, &trace->ioam_trace_type);
-    elt_index++;
-    elt += trace_data_size_in_words;
-  }
-  return (s);
-}
-
-int
-ip6_hbh_ioam_trace_data_list_handler (vlib_buffer_t *b, ip6_header_t *ip, ip6_hop_by_hop_option_t *opt)
-{
-  ip6_main_t * im = &ip6_main;
-  ip_lookup_main_t * lm = &im->lookup_main;
-  ip6_hop_by_hop_ioam_main_t * hm = &ip6_hop_by_hop_ioam_main;
-  u8 elt_index = 0;
-  ioam_trace_option_t *trace = (ioam_trace_option_t *)opt;
-  u32 adj_index = vnet_buffer (b)->ip.adj_index[VLIB_TX];
-  ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index);
-  time_u64_t time_u64;
-  u32 *elt;
-  int rv = 0;
-
-  time_u64.as_u64 = 0;
-
-  if (PREDICT_TRUE (trace->data_list_elts_left)) {
-    trace->data_list_elts_left--;
-    /* fetch_trace_data_size returns in bytes. Convert it to 4-bytes
-     * to skip to this node's location.
-     */
-    elt_index = trace->data_list_elts_left * fetch_trace_data_size(trace->ioam_trace_type) / 4;
-    elt = &trace->elts[elt_index];
-    if (trace->ioam_trace_type & BIT_TTL_NODEID) {
-      *elt = clib_host_to_net_u32 ((ip->hop_limit<<24) | hm->node_id);
-      elt++;
-    }
-
-    if (trace->ioam_trace_type & BIT_ING_INTERFACE) {
-      *elt = (vnet_buffer(b)->sw_if_index[VLIB_RX]&0xFFFF) << 16 | (adj->rewrite_header.sw_if_index & 0xFFFF);
-      *elt = clib_host_to_net_u32(*elt);
-      elt++;
-    }
-                 
-    if (trace->ioam_trace_type & BIT_TIMESTAMP) {
-      /* Send least significant 32 bits */
-      f64 time_f64 = (f64)(((f64)hm->unix_time_0) + (vlib_time_now(hm->vlib_main) - hm->vlib_time_0));
-
-      time_u64.as_u64 = time_f64 * trace_tsp_mul[hm->trace_tsp];
-      *elt = clib_host_to_net_u32(time_u64.as_u32[0]);
-      elt++;
-    }
-
-    if (trace->ioam_trace_type & BIT_APPDATA) {
-      /* $$$ set elt0->app_data */
-      *elt = clib_host_to_net_u32(hm->app_data);
-      elt++;
-    }
-  }
-  return (rv);
-}
-
 /* The main h-b-h tracer will be invoked, no need to do much here */
 int
 ip6_hbh_add_register_option (u8 option,
 			     u8 size,
-			     int rewrite_options(u8 *rewrite_string, u8 rewrite_size))
+			     int rewrite_options (u8 * rewrite_string,
+						  u8 * rewrite_size))
 {
-  ip6_hop_by_hop_ioam_main_t * hm = &ip6_hop_by_hop_ioam_main;
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
 
   ASSERT (option < ARRAY_LEN (hm->add_options));
 
@@ -208,14 +61,14 @@ ip6_hbh_add_register_option (u8 option,
 
   hm->add_options[option] = rewrite_options;
   hm->options_size[option] = size;
-  
+
   return (0);
 }
 
 int
 ip6_hbh_add_unregister_option (u8 option)
 {
-  ip6_hop_by_hop_ioam_main_t * hm = &ip6_hop_by_hop_ioam_main;
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
 
   ASSERT (option < ARRAY_LEN (hm->add_options));
 
@@ -228,20 +81,21 @@ ip6_hbh_add_unregister_option (u8 option)
   return (0);
 }
 
-typedef struct {
+typedef struct
+{
   u32 next_index;
 } ip6_add_hop_by_hop_trace_t;
 
 /* packet trace format function */
-static u8 * format_ip6_add_hop_by_hop_trace (u8 * s, va_list * args)
+static u8 *
+format_ip6_add_hop_by_hop_trace (u8 * s, va_list * args)
 {
   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  ip6_add_hop_by_hop_trace_t * t = va_arg (*args, 
-                                            ip6_add_hop_by_hop_trace_t *);
-  
-  s = format (s, "IP6_ADD_HOP_BY_HOP: next index %d",
-              t->next_index);
+  ip6_add_hop_by_hop_trace_t *t = va_arg (*args,
+					  ip6_add_hop_by_hop_trace_t *);
+
+  s = format (s, "IP6_ADD_HOP_BY_HOP: next index %d", t->next_index);
   return s;
 }
 
@@ -250,14 +104,15 @@ vlib_node_registration_t ip6_add_hop_by_hop_node;
 #define foreach_ip6_add_hop_by_hop_error \
 _(PROCESSED, "Pkts w/ added ip6 hop-by-hop options")
 
-typedef enum {
+typedef enum
+{
 #define _(sym,str) IP6_ADD_HOP_BY_HOP_ERROR_##sym,
   foreach_ip6_add_hop_by_hop_error
 #undef _
-  IP6_ADD_HOP_BY_HOP_N_ERROR,
+    IP6_ADD_HOP_BY_HOP_N_ERROR,
 } ip6_add_hop_by_hop_error_t;
 
-static char * ip6_add_hop_by_hop_error_strings[] = {
+static char *ip6_add_hop_by_hop_error_strings[] = {
 #define _(sym,string) string,
   foreach_ip6_add_hop_by_hop_error
 #undef _
@@ -265,14 +120,13 @@ static char * ip6_add_hop_by_hop_error_strings[] = {
 
 static uword
 ip6_add_hop_by_hop_node_fn (vlib_main_t * vm,
-		  vlib_node_runtime_t * node,
-		  vlib_frame_t * frame)
+			    vlib_node_runtime_t * node, vlib_frame_t * frame)
 {
-  ip6_hop_by_hop_ioam_main_t * hm = &ip6_hop_by_hop_ioam_main;
-  u32 n_left_from, * from, * to_next;
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
+  u32 n_left_from, *from, *to_next;
   ip_lookup_next_t next_index;
   u32 processed = 0;
-  u8 * rewrite = hm->rewrite;
+  u8 *rewrite = hm->rewrite;
   u32 rewrite_length = vec_len (rewrite);
 
   from = vlib_frame_vector_args (frame);
@@ -283,99 +137,102 @@ ip6_add_hop_by_hop_node_fn (vlib_main_t * vm,
     {
       u32 n_left_to_next;
 
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
       while (n_left_from >= 4 && n_left_to_next >= 2)
-        {
-          u32 bi0, bi1;
-          vlib_buffer_t * b0, *b1;
-          u32 next0, next1;
-          ip6_header_t * ip0, *ip1;
-          ip6_hop_by_hop_header_t * hbh0, *hbh1;
-          u64 * copy_src0, * copy_dst0, *copy_src1, *copy_dst1;
-          u16 new_l0, new_l1;
-
-          /* Prefetch next iteration. */
-          {
-            vlib_buffer_t * p2, * p3;
-
-            p2 = vlib_get_buffer (vm, from[2]);
-            p3 = vlib_get_buffer (vm, from[3]);
-
-            vlib_prefetch_buffer_header (p2, LOAD);
-            vlib_prefetch_buffer_header (p3, LOAD);
-
-            CLIB_PREFETCH (p2->data - rewrite_length, 2 * CLIB_CACHE_LINE_BYTES, STORE);
-            CLIB_PREFETCH (p3->data - rewrite_length, 2 * CLIB_CACHE_LINE_BYTES, STORE);
-          }
-
-          /* speculatively enqueue b0 and b1 to the current next frame */
-          to_next[0] = bi0 = from[0];
-          to_next[1] = bi1 = from[1];
-          from += 2;
-          to_next += 2;
-          n_left_from -= 2;
-          n_left_to_next -= 2;
-
-          b0 = vlib_get_buffer (vm, bi0);
-          b1 = vlib_get_buffer (vm, bi1);
-
-          /* $$$$$ Dual loop: process 2 x packets here $$$$$ */
-          ASSERT (b0->current_data == 0);
-          ASSERT (b1->current_data == 0);
-
-          ip0 = vlib_buffer_get_current (b0);
-          ip1 = vlib_buffer_get_current (b1);
-
-          /* Copy the ip header left by the required amount */
-          copy_dst0 = (u64 *)(((u8 *)ip0) - rewrite_length);
-          copy_dst1 = (u64 *)(((u8 *)ip1) - rewrite_length);
-          copy_src0 = (u64 *) ip0;
-          copy_src1 = (u64 *) ip1;
-
-          copy_dst0 [0] = copy_src0 [0];
-          copy_dst0 [1] = copy_src0 [1];
-          copy_dst0 [2] = copy_src0 [2];
-          copy_dst0 [3] = copy_src0 [3];
-          copy_dst0 [4] = copy_src0 [4];
-
-          copy_dst1 [0] = copy_src1 [0];
-          copy_dst1 [1] = copy_src1 [1];
-          copy_dst1 [2] = copy_src1 [2];
-          copy_dst1 [3] = copy_src1 [3];
-          copy_dst1 [4] = copy_src1 [4];
-
-          vlib_buffer_advance (b0, - (word)rewrite_length);
-          vlib_buffer_advance (b1, - (word)rewrite_length);
-          ip0 = vlib_buffer_get_current (b0);
-          ip1 = vlib_buffer_get_current (b1);
-
-          hbh0 = (ip6_hop_by_hop_header_t *)(ip0 + 1);
-          hbh1 = (ip6_hop_by_hop_header_t *)(ip1 + 1);
-          /* $$$ tune, rewrite_length is a multiple of 8 */
-          clib_memcpy (hbh0, rewrite, rewrite_length);
-          clib_memcpy (hbh1, rewrite, rewrite_length);
-          /* Patch the protocol chain, insert the h-b-h (type 0) header */
-          hbh0->protocol = ip0->protocol;
-          hbh1->protocol = ip1->protocol;
-          ip0->protocol = 0;
-          ip1->protocol = 0;
-          new_l0 = clib_net_to_host_u16 (ip0->payload_length) + rewrite_length;
-          new_l1 = clib_net_to_host_u16 (ip1->payload_length) + rewrite_length;
-          ip0->payload_length = clib_host_to_net_u16 (new_l0);
-          ip1->payload_length = clib_host_to_net_u16 (new_l1);
-
-          /* Populate the (first) h-b-h list elt */
-          next0 = IP6_HBYH_IOAM_INPUT_NEXT_IP6_LOOKUP;
-          next1 = IP6_HBYH_IOAM_INPUT_NEXT_IP6_LOOKUP;
-
-
-          /* $$$$$ End of processing 2 x packets $$$$$ */
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)))
-            {
-              if (b0->flags & VLIB_BUFFER_IS_TRACED)
-                {
+	{
+	  u32 bi0, bi1;
+	  vlib_buffer_t *b0, *b1;
+	  u32 next0, next1;
+	  ip6_header_t *ip0, *ip1;
+	  ip6_hop_by_hop_header_t *hbh0, *hbh1;
+	  u64 *copy_src0, *copy_dst0, *copy_src1, *copy_dst1;
+	  u16 new_l0, new_l1;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t *p2, *p3;
+
+	    p2 = vlib_get_buffer (vm, from[2]);
+	    p3 = vlib_get_buffer (vm, from[3]);
+
+	    vlib_prefetch_buffer_header (p2, LOAD);
+	    vlib_prefetch_buffer_header (p3, LOAD);
+
+	    CLIB_PREFETCH (p2->data - rewrite_length,
+			   2 * CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p3->data - rewrite_length,
+			   2 * CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+	  /* speculatively enqueue b0 and b1 to the current next frame */
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  from += 2;
+	  to_next += 2;
+	  n_left_from -= 2;
+	  n_left_to_next -= 2;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+
+	  /* $$$$$ Dual loop: process 2 x packets here $$$$$ */
+	  ASSERT (b0->current_data == 0);
+	  ASSERT (b1->current_data == 0);
+
+	  ip0 = vlib_buffer_get_current (b0);
+	  ip1 = vlib_buffer_get_current (b1);
+
+	  /* Copy the ip header left by the required amount */
+	  copy_dst0 = (u64 *) (((u8 *) ip0) - rewrite_length);
+	  copy_dst1 = (u64 *) (((u8 *) ip1) - rewrite_length);
+	  copy_src0 = (u64 *) ip0;
+	  copy_src1 = (u64 *) ip1;
+
+	  copy_dst0[0] = copy_src0[0];
+	  copy_dst0[1] = copy_src0[1];
+	  copy_dst0[2] = copy_src0[2];
+	  copy_dst0[3] = copy_src0[3];
+	  copy_dst0[4] = copy_src0[4];
+
+	  copy_dst1[0] = copy_src1[0];
+	  copy_dst1[1] = copy_src1[1];
+	  copy_dst1[2] = copy_src1[2];
+	  copy_dst1[3] = copy_src1[3];
+	  copy_dst1[4] = copy_src1[4];
+
+	  vlib_buffer_advance (b0, -(word) rewrite_length);
+	  vlib_buffer_advance (b1, -(word) rewrite_length);
+	  ip0 = vlib_buffer_get_current (b0);
+	  ip1 = vlib_buffer_get_current (b1);
+
+	  hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
+	  hbh1 = (ip6_hop_by_hop_header_t *) (ip1 + 1);
+	  /* $$$ tune, rewrite_length is a multiple of 8 */
+	  clib_memcpy (hbh0, rewrite, rewrite_length);
+	  clib_memcpy (hbh1, rewrite, rewrite_length);
+	  /* Patch the protocol chain, insert the h-b-h (type 0) header */
+	  hbh0->protocol = ip0->protocol;
+	  hbh1->protocol = ip1->protocol;
+	  ip0->protocol = 0;
+	  ip1->protocol = 0;
+	  new_l0 =
+	    clib_net_to_host_u16 (ip0->payload_length) + rewrite_length;
+	  new_l1 =
+	    clib_net_to_host_u16 (ip1->payload_length) + rewrite_length;
+	  ip0->payload_length = clib_host_to_net_u16 (new_l0);
+	  ip1->payload_length = clib_host_to_net_u16 (new_l1);
+
+	  /* Populate the (first) h-b-h list elt */
+	  next0 = IP6_HBYH_IOAM_INPUT_NEXT_IP6_LOOKUP;
+	  next1 = IP6_HBYH_IOAM_INPUT_NEXT_IP6_LOOKUP;
+
+
+	  /* $$$$$ End of processing 2 x packets $$$$$ */
+
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	    {
+	      if (b0->flags & VLIB_BUFFER_IS_TRACED)
+		{
 		  ip6_add_hop_by_hop_trace_t *t =
 		    vlib_add_trace (vm, node, b0, sizeof (*t));
 		  t->next_index = next0;
@@ -387,23 +244,23 @@ ip6_add_hop_by_hop_node_fn (vlib_main_t * vm,
 		  t->next_index = next1;
 		}
 	    }
-	  processed+=2;
+	  processed += 2;
 	  /* verify speculative enqueues, maybe switch current next frame */
 	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 					   to_next, n_left_to_next,
 					   bi0, bi1, next0, next1);
-        }
+	}
       while (n_left_from > 0 && n_left_to_next > 0)
 	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0;
-          ip6_header_t * ip0;
-          ip6_hop_by_hop_header_t * hbh0;
-          u64 * copy_src0, * copy_dst0;
-          u16 new_l0;
-          
-          /* speculatively enqueue b0 to the current next frame */
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  u32 next0;
+	  ip6_header_t *ip0;
+	  ip6_hop_by_hop_header_t *hbh0;
+	  u64 *copy_src0, *copy_dst0;
+	  u16 new_l0;
+
+	  /* speculatively enqueue b0 to the current next frame */
 	  bi0 = from[0];
 	  to_next[0] = bi0;
 	  from += 1;
@@ -413,43 +270,44 @@ ip6_add_hop_by_hop_node_fn (vlib_main_t * vm,
 
 	  b0 = vlib_get_buffer (vm, bi0);
 
-          ip0 = vlib_buffer_get_current (b0);
-
-          /* Copy the ip header left by the required amount */
-          copy_dst0 = (u64 *)(((u8 *)ip0) - rewrite_length);
-          copy_src0 = (u64 *) ip0;
-
-          copy_dst0 [0] = copy_src0 [0];
-          copy_dst0 [1] = copy_src0 [1];
-          copy_dst0 [2] = copy_src0 [2];
-          copy_dst0 [3] = copy_src0 [3];
-          copy_dst0 [4] = copy_src0 [4];
-          vlib_buffer_advance (b0, - (word)rewrite_length);
-          ip0 = vlib_buffer_get_current (b0);
-
-          hbh0 = (ip6_hop_by_hop_header_t *)(ip0 + 1);
-          /* $$$ tune, rewrite_length is a multiple of 8 */
-          clib_memcpy (hbh0, rewrite, rewrite_length);
-          /* Patch the protocol chain, insert the h-b-h (type 0) header */
-          hbh0->protocol = ip0->protocol;
-          ip0->protocol = 0;
-          new_l0 = clib_net_to_host_u16 (ip0->payload_length) + rewrite_length;
-          ip0->payload_length = clib_host_to_net_u16 (new_l0);
-          
-          /* Populate the (first) h-b-h list elt */
-          next0 = IP6_HBYH_IOAM_INPUT_NEXT_IP6_LOOKUP;
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) 
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED))) 
-            {
-              ip6_add_hop_by_hop_trace_t *t = 
-                 vlib_add_trace (vm, node, b0, sizeof (*t));
-              t->next_index = next0;
-            }
-            
-          processed++;
-
-          /* verify speculative enqueue, maybe switch current next frame */
+	  ip0 = vlib_buffer_get_current (b0);
+
+	  /* Copy the ip header left by the required amount */
+	  copy_dst0 = (u64 *) (((u8 *) ip0) - rewrite_length);
+	  copy_src0 = (u64 *) ip0;
+
+	  copy_dst0[0] = copy_src0[0];
+	  copy_dst0[1] = copy_src0[1];
+	  copy_dst0[2] = copy_src0[2];
+	  copy_dst0[3] = copy_src0[3];
+	  copy_dst0[4] = copy_src0[4];
+	  vlib_buffer_advance (b0, -(word) rewrite_length);
+	  ip0 = vlib_buffer_get_current (b0);
+
+	  hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
+	  /* $$$ tune, rewrite_length is a multiple of 8 */
+	  clib_memcpy (hbh0, rewrite, rewrite_length);
+	  /* Patch the protocol chain, insert the h-b-h (type 0) header */
+	  hbh0->protocol = ip0->protocol;
+	  ip0->protocol = 0;
+	  new_l0 =
+	    clib_net_to_host_u16 (ip0->payload_length) + rewrite_length;
+	  ip0->payload_length = clib_host_to_net_u16 (new_l0);
+
+	  /* Populate the (first) h-b-h list elt */
+	  next0 = IP6_HBYH_IOAM_INPUT_NEXT_IP6_LOOKUP;
+
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
+			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+	    {
+	      ip6_add_hop_by_hop_trace_t *t =
+		vlib_add_trace (vm, node, b0, sizeof (*t));
+	      t->next_index = next0;
+	    }
+
+	  processed++;
+
+	  /* verify speculative enqueue, maybe switch current next frame */
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 					   to_next, n_left_to_next,
 					   bi0, next0);
@@ -458,54 +316,56 @@ ip6_add_hop_by_hop_node_fn (vlib_main_t * vm,
       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
     }
 
-  vlib_node_increment_counter (vm, ip6_add_hop_by_hop_node.index, 
-                               IP6_ADD_HOP_BY_HOP_ERROR_PROCESSED, processed);
+  vlib_node_increment_counter (vm, ip6_add_hop_by_hop_node.index,
+			       IP6_ADD_HOP_BY_HOP_ERROR_PROCESSED, processed);
   return frame->n_vectors;
 }
 
-VLIB_REGISTER_NODE (ip6_add_hop_by_hop_node) = {
-  .function = ip6_add_hop_by_hop_node_fn,
-  .name = "ip6-add-hop-by-hop",
-  .vector_size = sizeof (u32),
-  .format_trace = format_ip6_add_hop_by_hop_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  
-  .n_errors = ARRAY_LEN(ip6_add_hop_by_hop_error_strings),
-  .error_strings = ip6_add_hop_by_hop_error_strings,
-
-  /* See ip/lookup.h */
-  .n_next_nodes = IP6_HBYH_IOAM_INPUT_N_NEXT,
-  .next_nodes = {
+VLIB_REGISTER_NODE (ip6_add_hop_by_hop_node) =	/* *INDENT-OFF* */
+{
+  .function = ip6_add_hop_by_hop_node_fn,.name =
+    "ip6-add-hop-by-hop",.vector_size = sizeof (u32),.format_trace =
+    format_ip6_add_hop_by_hop_trace,.type =
+    VLIB_NODE_TYPE_INTERNAL,.n_errors =
+    ARRAY_LEN (ip6_add_hop_by_hop_error_strings),.error_strings =
+    ip6_add_hop_by_hop_error_strings,
+    /* See ip/lookup.h */
+    .n_next_nodes = IP6_HBYH_IOAM_INPUT_N_NEXT,.next_nodes =
+  {
 #define _(s,n) [IP6_HBYH_IOAM_INPUT_NEXT_##s] = n,
     foreach_ip6_hbyh_ioam_input_next
 #undef _
-  },
-};
+  }
+,};
 
-VLIB_NODE_FUNCTION_MULTIARCH (ip6_add_hop_by_hop_node, ip6_add_hop_by_hop_node_fn)
+/* *INDENT-ON* */
 
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_add_hop_by_hop_node,
+			      ip6_add_hop_by_hop_node_fn)
 /* The main h-b-h tracer was already invoked, no need to do much here */
-typedef struct {
-  u32 next_index;
-} ip6_pop_hop_by_hop_trace_t;
+     typedef struct
+     {
+       u32 next_index;
+     } ip6_pop_hop_by_hop_trace_t;
 
 /* packet trace format function */
-static u8 * format_ip6_pop_hop_by_hop_trace (u8 * s, va_list * args)
+     static u8 *format_ip6_pop_hop_by_hop_trace (u8 * s, va_list * args)
 {
   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  ip6_pop_hop_by_hop_trace_t * t = va_arg (*args, ip6_pop_hop_by_hop_trace_t *);
-  
-  s = format (s, "IP6_POP_HOP_BY_HOP: next index %d",
-              t->next_index);
+  ip6_pop_hop_by_hop_trace_t *t =
+    va_arg (*args, ip6_pop_hop_by_hop_trace_t *);
+
+  s = format (s, "IP6_POP_HOP_BY_HOP: next index %d", t->next_index);
   return s;
 }
 
 int
 ip6_hbh_pop_register_option (u8 option,
-			     int options(ip6_header_t *ip, ip6_hop_by_hop_option_t *opt))
+			     int options (ip6_header_t * ip,
+					  ip6_hop_by_hop_option_t * opt))
 {
-  ip6_hop_by_hop_ioam_main_t * hm = &ip6_hop_by_hop_ioam_main;
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
 
   ASSERT (option < ARRAY_LEN (hm->pop_options));
 
@@ -521,7 +381,7 @@ ip6_hbh_pop_register_option (u8 option,
 int
 ip6_hbh_pop_unregister_option (u8 option)
 {
-  ip6_hop_by_hop_ioam_main_t * hm = &ip6_hop_by_hop_ioam_main;
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
 
   ASSERT (option < ARRAY_LEN (hm->pop_options));
 
@@ -540,32 +400,35 @@ _(PROCESSED, "Pkts w/ removed ip6 hop-by-hop options")  \
 _(NO_HOHO, "Pkts w/ no ip6 hop-by-hop options")         \
 _(OPTION_FAILED, "ip6 pop hop-by-hop failed to process")
 
-typedef enum {
+typedef enum
+{
 #define _(sym,str) IP6_POP_HOP_BY_HOP_ERROR_##sym,
   foreach_ip6_pop_hop_by_hop_error
 #undef _
-  IP6_POP_HOP_BY_HOP_N_ERROR,
+    IP6_POP_HOP_BY_HOP_N_ERROR,
 } ip6_pop_hop_by_hop_error_t;
 
-static char * ip6_pop_hop_by_hop_error_strings[] = {
+static char *ip6_pop_hop_by_hop_error_strings[] = {
 #define _(sym,string) string,
   foreach_ip6_pop_hop_by_hop_error
 #undef _
 };
 
-static inline void ioam_pop_hop_by_hop_processing (vlib_main_t * vm,
-                                                ip6_header_t *ip0,
-                                                ip6_hop_by_hop_header_t *hbh0)
+static inline void
+ioam_pop_hop_by_hop_processing (vlib_main_t * vm,
+				ip6_header_t * ip0,
+				ip6_hop_by_hop_header_t * hbh0)
 {
-  ip6_hop_by_hop_ioam_main_t * hm = &ip6_hop_by_hop_ioam_main;
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
   ip6_hop_by_hop_option_t *opt0, *limit0;
   u8 type0;
 
-  if (!hbh0 || !ip0) return;
+  if (!hbh0 || !ip0)
+    return;
 
-  opt0 = (ip6_hop_by_hop_option_t *)(hbh0+1);
+  opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
   limit0 = (ip6_hop_by_hop_option_t *)
-    ((u8 *)hbh0 + ((hbh0->length+1)<<3));
+    ((u8 *) hbh0 + ((hbh0->length + 1) << 3));
 
   /* Scan the set of h-b-h options, process ones that we understand */
   while (opt0 < limit0)
@@ -573,63 +436,65 @@ static inline void ioam_pop_hop_by_hop_processing (vlib_main_t * vm,
       type0 = opt0->type;
       switch (type0)
 	{
-	case 0: /* Pad1 */
-	  opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
+	case 0:		/* Pad1 */
+	  opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
 	  continue;
-	case 1: /* PadN */
+	case 1:		/* PadN */
 	  break;
 	default:
 	  if (hm->pop_options[type0])
 	    {
-	      if ((*hm->pop_options[type0])(ip0, opt0) < 0)
-	      {
-		vlib_node_increment_counter (vm, ip6_pop_hop_by_hop_node.index, 
-                               IP6_POP_HOP_BY_HOP_ERROR_OPTION_FAILED, 1);
-	      }
+	      if ((*hm->pop_options[type0]) (ip0, opt0) < 0)
+		{
+		  vlib_node_increment_counter (vm,
+					       ip6_pop_hop_by_hop_node.index,
+					       IP6_POP_HOP_BY_HOP_ERROR_OPTION_FAILED,
+					       1);
+		}
 	    }
 	}
-      	opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
+      opt0 =
+	(ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
+				     sizeof (ip6_hop_by_hop_option_t));
     }
 }
 
 static uword
 ip6_pop_hop_by_hop_node_fn (vlib_main_t * vm,
-		  vlib_node_runtime_t * node,
-		  vlib_frame_t * frame)
+			    vlib_node_runtime_t * node, vlib_frame_t * frame)
 {
-  ip6_main_t * im = &ip6_main;
-  ip_lookup_main_t * lm = &im->lookup_main;
-  u32 n_left_from, * from, * to_next;
+  ip6_main_t *im = &ip6_main;
+  ip_lookup_main_t *lm = &im->lookup_main;
+  u32 n_left_from, *from, *to_next;
   ip_lookup_next_t next_index;
   u32 processed = 0;
   u32 no_header = 0;
-  
+
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;
   next_index = node->cached_next_index;
-  
+
   while (n_left_from > 0)
     {
       u32 n_left_to_next;
 
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 
       while (n_left_from >= 4 && n_left_to_next >= 2)
 	{
-          u32 bi0, bi1;
-	  vlib_buffer_t * b0, * b1;
+	  u32 bi0, bi1;
+	  vlib_buffer_t *b0, *b1;
 	  u32 next0, next1;
 	  u32 adj_index0, adj_index1;
-	  ip6_header_t * ip0, *ip1;
-	  ip_adjacency_t * adj0, *adj1;
+	  ip6_header_t *ip0, *ip1;
+	  ip_adjacency_t *adj0, *adj1;
 	  ip6_hop_by_hop_header_t *hbh0, *hbh1;
 	  u64 *copy_dst0, *copy_src0, *copy_dst1, *copy_src1;
 	  u16 new_l0, new_l1;
 
 	  /* Prefetch next iteration. */
 	  {
-	    vlib_buffer_t * p2, * p3;
+	    vlib_buffer_t *p2, *p3;
 
 	    p2 = vlib_get_buffer (vm, from[2]);
 	    p3 = vlib_get_buffer (vm, from[3]);
@@ -641,7 +506,7 @@ ip6_pop_hop_by_hop_node_fn (vlib_main_t * vm,
 	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 	  }
 
-          /* speculatively enqueue b0 and b1 to the current next frame */
+	  /* speculatively enqueue b0 and b1 to the current next frame */
 	  to_next[0] = bi0 = from[0];
 	  to_next[1] = bi1 = from[1];
 	  from += 2;
@@ -652,12 +517,12 @@ ip6_pop_hop_by_hop_node_fn (vlib_main_t * vm,
 	  b0 = vlib_get_buffer (vm, bi0);
 	  b1 = vlib_get_buffer (vm, bi1);
 
-          /* $$$$$ Dual loop: process 2 x packets here $$$$$ */
-          ASSERT (b0->current_data == 0);
-          ASSERT (b1->current_data == 0);
+	  /* $$$$$ Dual loop: process 2 x packets here $$$$$ */
+	  ASSERT (b0->current_data == 0);
+	  ASSERT (b1->current_data == 0);
 
-          ip0 = vlib_buffer_get_current (b0);
-          ip1 = vlib_buffer_get_current (b1);
+	  ip0 = vlib_buffer_get_current (b0);
+	  ip1 = vlib_buffer_get_current (b1);
 	  adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
 	  adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
 	  adj0 = ip_get_adjacency (lm, adj_index0);
@@ -666,19 +531,19 @@ ip6_pop_hop_by_hop_node_fn (vlib_main_t * vm,
 	  next0 = adj0->lookup_next_index;
 	  next1 = adj1->lookup_next_index;
 
-	  hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
-	  hbh1 = (ip6_hop_by_hop_header_t *)(ip1+1);
+	  hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
+	  hbh1 = (ip6_hop_by_hop_header_t *) (ip1 + 1);
 
-	  ioam_pop_hop_by_hop_processing(vm, ip0, hbh0);
-	  ioam_pop_hop_by_hop_processing(vm, ip1, hbh1);
+	  ioam_pop_hop_by_hop_processing (vm, ip0, hbh0);
+	  ioam_pop_hop_by_hop_processing (vm, ip1, hbh1);
 
-	  vlib_buffer_advance (b0, (hbh0->length+1)<<3);
-	  vlib_buffer_advance (b1, (hbh1->length+1)<<3);
+	  vlib_buffer_advance (b0, (hbh0->length + 1) << 3);
+	  vlib_buffer_advance (b1, (hbh1->length + 1) << 3);
 
 	  new_l0 = clib_net_to_host_u16 (ip0->payload_length) -
-	    ((hbh0->length+1)<<3);
+	    ((hbh0->length + 1) << 3);
 	  new_l1 = clib_net_to_host_u16 (ip1->payload_length) -
-	    ((hbh1->length+1)<<3);
+	    ((hbh1->length + 1) << 3);
 
 	  ip0->payload_length = clib_host_to_net_u16 (new_l0);
 	  ip1->payload_length = clib_host_to_net_u16 (new_l1);
@@ -686,58 +551,58 @@ ip6_pop_hop_by_hop_node_fn (vlib_main_t * vm,
 	  ip0->protocol = hbh0->protocol;
 	  ip1->protocol = hbh1->protocol;
 
-	  copy_src0 = (u64 *)ip0;
-	  copy_src1 = (u64 *)ip1;
-	  copy_dst0 = copy_src0 + (hbh0->length+1);
-	  copy_dst0 [4] = copy_src0[4];
-	  copy_dst0 [3] = copy_src0[3];
-	  copy_dst0 [2] = copy_src0[2];
-	  copy_dst0 [1] = copy_src0[1];
-	  copy_dst0 [0] = copy_src0[0];
-	  copy_dst1 = copy_src1 + (hbh1->length+1);
-	  copy_dst1 [4] = copy_src1[4];
-	  copy_dst1 [3] = copy_src1[3];
-	  copy_dst1 [2] = copy_src1[2];
-	  copy_dst1 [1] = copy_src1[1];
-	  copy_dst1 [0] = copy_src1[0];
-	  processed+=2;
-          /* $$$$$ End of processing 2 x packets $$$$$ */
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)))
-            {
-              if (b0->flags & VLIB_BUFFER_IS_TRACED)
-                {
-                    ip6_pop_hop_by_hop_trace_t *t =
-                      vlib_add_trace (vm, node, b0, sizeof (*t));
-                    t->next_index = next0;
-                  }
-                if (b1->flags & VLIB_BUFFER_IS_TRACED)
-                  {
-                    ip6_pop_hop_by_hop_trace_t *t =
-                      vlib_add_trace (vm, node, b1, sizeof (*t));
-                    t->next_index = next1;
-                  }
-              }
-
-            /* verify speculative enqueues, maybe switch current next frame */
-            vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-                                             to_next, n_left_to_next,
-                                             bi0, bi1, next0, next1);
-        }
+	  copy_src0 = (u64 *) ip0;
+	  copy_src1 = (u64 *) ip1;
+	  copy_dst0 = copy_src0 + (hbh0->length + 1);
+	  copy_dst0[4] = copy_src0[4];
+	  copy_dst0[3] = copy_src0[3];
+	  copy_dst0[2] = copy_src0[2];
+	  copy_dst0[1] = copy_src0[1];
+	  copy_dst0[0] = copy_src0[0];
+	  copy_dst1 = copy_src1 + (hbh1->length + 1);
+	  copy_dst1[4] = copy_src1[4];
+	  copy_dst1[3] = copy_src1[3];
+	  copy_dst1[2] = copy_src1[2];
+	  copy_dst1[1] = copy_src1[1];
+	  copy_dst1[0] = copy_src1[0];
+	  processed += 2;
+	  /* $$$$$ End of processing 2 x packets $$$$$ */
+
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	    {
+	      if (b0->flags & VLIB_BUFFER_IS_TRACED)
+		{
+		  ip6_pop_hop_by_hop_trace_t *t =
+		    vlib_add_trace (vm, node, b0, sizeof (*t));
+		  t->next_index = next0;
+		}
+	      if (b1->flags & VLIB_BUFFER_IS_TRACED)
+		{
+		  ip6_pop_hop_by_hop_trace_t *t =
+		    vlib_add_trace (vm, node, b1, sizeof (*t));
+		  t->next_index = next1;
+		}
+	    }
+
+	  /* verify speculative enqueues, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, bi1, next0, next1);
+	}
 
       while (n_left_from > 0 && n_left_to_next > 0)
 	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0;
-          u32 adj_index0;
-          ip6_header_t * ip0;
-          ip_adjacency_t * adj0;
-          ip6_hop_by_hop_header_t *hbh0;
-          u64 * copy_dst0, * copy_src0;
-          u16 new_l0;
-          
-          /* speculatively enqueue b0 to the current next frame */
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  u32 next0;
+	  u32 adj_index0;
+	  ip6_header_t *ip0;
+	  ip_adjacency_t *adj0;
+	  ip6_hop_by_hop_header_t *hbh0;
+	  u64 *copy_dst0, *copy_src0;
+	  u16 new_l0;
+
+	  /* speculatively enqueue b0 to the current next frame */
 	  bi0 = from[0];
 	  to_next[0] = bi0;
 	  from += 1;
@@ -747,42 +612,42 @@ ip6_pop_hop_by_hop_node_fn (vlib_main_t * vm,
 
 	  b0 = vlib_get_buffer (vm, bi0);
 
-          ip0 = vlib_buffer_get_current (b0);
-          adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
-          adj0 = ip_get_adjacency (lm, adj_index0);
+	  ip0 = vlib_buffer_get_current (b0);
+	  adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
+	  adj0 = ip_get_adjacency (lm, adj_index0);
 
 	  /* Default use the next_index from the adjacency. */
 	  next0 = adj0->lookup_next_index;
 
-          /* Perfectly normal to end up here w/ out h-b-h header */
-	  hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
-          
+	  /* Perfectly normal to end up here w/ out h-b-h header */
+	  hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
+
 	  /* TODO:Temporarily doing it here.. do this validation in end_of_path_cb */
-	  ioam_pop_hop_by_hop_processing(vm, ip0, hbh0);
+	  ioam_pop_hop_by_hop_processing (vm, ip0, hbh0);
 	  /* Pop the trace data */
-	  vlib_buffer_advance (b0, (hbh0->length+1)<<3);
+	  vlib_buffer_advance (b0, (hbh0->length + 1) << 3);
 	  new_l0 = clib_net_to_host_u16 (ip0->payload_length) -
-	    ((hbh0->length+1)<<3);
+	    ((hbh0->length + 1) << 3);
 	  ip0->payload_length = clib_host_to_net_u16 (new_l0);
 	  ip0->protocol = hbh0->protocol;
-	  copy_src0 = (u64 *)ip0;
-	  copy_dst0 = copy_src0 + (hbh0->length+1);
-	  copy_dst0 [4] = copy_src0[4];
-	  copy_dst0 [3] = copy_src0[3];
-	  copy_dst0 [2] = copy_src0[2];
-	  copy_dst0 [1] = copy_src0[1];
-	  copy_dst0 [0] = copy_src0[0];
+	  copy_src0 = (u64 *) ip0;
+	  copy_dst0 = copy_src0 + (hbh0->length + 1);
+	  copy_dst0[4] = copy_src0[4];
+	  copy_dst0[3] = copy_src0[3];
+	  copy_dst0[2] = copy_src0[2];
+	  copy_dst0[1] = copy_src0[1];
+	  copy_dst0[0] = copy_src0[0];
 	  processed++;
-              
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) 
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED))) 
-            {
-              ip6_pop_hop_by_hop_trace_t *t = 
-                 vlib_add_trace (vm, node, b0, sizeof (*t));
-              t->next_index = next0;
-            }
-
-          /* verify speculative enqueue, maybe switch current next frame */
+
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
+			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+	    {
+	      ip6_pop_hop_by_hop_trace_t *t =
+		vlib_add_trace (vm, node, b0, sizeof (*t));
+	      t->next_index = next0;
+	    }
+
+	  /* verify speculative enqueue, maybe switch current next frame */
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 					   to_next, n_left_to_next,
 					   bi0, next0);
@@ -791,91 +656,76 @@ ip6_pop_hop_by_hop_node_fn (vlib_main_t * vm,
       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
     }
 
-  vlib_node_increment_counter (vm, ip6_pop_hop_by_hop_node.index, 
-                               IP6_POP_HOP_BY_HOP_ERROR_PROCESSED, processed);
-  vlib_node_increment_counter (vm, ip6_pop_hop_by_hop_node.index, 
-                               IP6_POP_HOP_BY_HOP_ERROR_NO_HOHO, no_header);
+  vlib_node_increment_counter (vm, ip6_pop_hop_by_hop_node.index,
+			       IP6_POP_HOP_BY_HOP_ERROR_PROCESSED, processed);
+  vlib_node_increment_counter (vm, ip6_pop_hop_by_hop_node.index,
+			       IP6_POP_HOP_BY_HOP_ERROR_NO_HOHO, no_header);
   return frame->n_vectors;
 }
 
-VLIB_REGISTER_NODE (ip6_pop_hop_by_hop_node) = {
-  .function = ip6_pop_hop_by_hop_node_fn,
-  .name = "ip6-pop-hop-by-hop",
-  .vector_size = sizeof (u32),
-  .format_trace = format_ip6_pop_hop_by_hop_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  .sibling_of = "ip6-lookup",
-  .n_errors = ARRAY_LEN(ip6_pop_hop_by_hop_error_strings),
-  .error_strings = ip6_pop_hop_by_hop_error_strings,
-
-  /* See ip/lookup.h */
-  .n_next_nodes = 0,
-};
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (ip6_pop_hop_by_hop_node) =
+{
+  .function = ip6_pop_hop_by_hop_node_fn,.name =
+    "ip6-pop-hop-by-hop",.vector_size = sizeof (u32),.format_trace =
+    format_ip6_pop_hop_by_hop_trace,.type =
+    VLIB_NODE_TYPE_INTERNAL,.sibling_of = "ip6-lookup",.n_errors =
+    ARRAY_LEN (ip6_pop_hop_by_hop_error_strings),.error_strings =
+    ip6_pop_hop_by_hop_error_strings,
+    /* See ip/lookup.h */
+.n_next_nodes = 0,};
+
+/* *INDENT-ON* */
 
 VLIB_NODE_FUNCTION_MULTIARCH (ip6_pop_hop_by_hop_node,
 			      ip6_pop_hop_by_hop_node_fn)
-
-static clib_error_t *
-ip6_hop_by_hop_ioam_init (vlib_main_t * vm)
+     static clib_error_t *ip6_hop_by_hop_ioam_init (vlib_main_t * vm)
 {
-  ip6_hop_by_hop_ioam_main_t * hm = &ip6_hop_by_hop_ioam_main;
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
 
   hm->vlib_main = vm;
-  hm->vnet_main = vnet_get_main();
-  hm->unix_time_0 = (u32) time (0); /* Store starting time */
+  hm->vnet_main = vnet_get_main ();
+  hm->unix_time_0 = (u32) time (0);	/* Store starting time */
   hm->vlib_time_0 = vlib_time_now (vm);
   hm->ioam_flag = IOAM_HBYH_MOD;
-  hm->trace_tsp = TSP_MICROSECONDS; /* Micro seconds */
-  memset(hm->add_options, 0, sizeof(hm->add_options));
-  memset(hm->pop_options, 0, sizeof(hm->pop_options));
-  memset(hm->options_size, 0, sizeof(hm->options_size));
-
-  /*
-   * Register the handlers
-   * XXX: This should be done dynamically based on OAM feature being enabled or not.
-   */
-  if (ip6_hbh_register_option(HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST, ip6_hbh_ioam_trace_data_list_handler,
-			      ip6_hbh_ioam_trace_data_list_trace_handler) < 0)
-    return (clib_error_create("registration of HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST failed"));
+  memset (hm->add_options, 0, sizeof (hm->add_options));
+  memset (hm->pop_options, 0, sizeof (hm->pop_options));
+  memset (hm->options_size, 0, sizeof (hm->options_size));
 
   return (0);
 }
 
 VLIB_INIT_FUNCTION (ip6_hop_by_hop_ioam_init);
 
-int ip6_ioam_set_rewrite (u8 **rwp, u32 trace_type, u32 trace_option_elts, 
-                          int has_pot_option, int has_ppc_option)
+int
+ip6_ioam_set_rewrite (u8 ** rwp, int has_trace_option,
+		      int has_pot_option, int has_ppc_option)
 {
-  ip6_hop_by_hop_ioam_main_t * hm = &ip6_hop_by_hop_ioam_main;  
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
   u8 *rewrite = 0;
   u32 size, rnd_size;
   ip6_hop_by_hop_header_t *hbh;
-  ioam_trace_option_t * trace_option;
   u8 *current;
-  u8 trace_data_size = 0;  
+  u8 trace_data_size = 0;
+  u8 pot_data_size = 0;
 
   vec_free (*rwp);
 
-  if (trace_option_elts == 0 && has_pot_option == 0)
+  if (has_trace_option == 0 && has_pot_option == 0)
     return -1;
 
   /* Work out how much space we need */
   size = sizeof (ip6_hop_by_hop_header_t);
 
-  if (trace_option_elts)
+  //if (has_trace_option && hm->get_sizeof_options[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST] != 0)
+  if (has_trace_option
+      && hm->options_size[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST] != 0)
     {
       size += sizeof (ip6_hop_by_hop_option_t);
-
-      trace_data_size = fetch_trace_data_size(trace_type);
-      if (trace_data_size == 0)
-          return VNET_API_ERROR_INVALID_VALUE;
-
-      if (trace_option_elts * trace_data_size > 254)
-          return VNET_API_ERROR_INVALID_VALUE;
-  
-      size += trace_option_elts * trace_data_size;
+      size += hm->options_size[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST];
     }
-  if (has_pot_option && hm->add_options[HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT] != 0)
+  if (has_pot_option
+      && hm->add_options[HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT] != 0)
     {
       size += sizeof (ip6_hop_by_hop_option_t);
       size += hm->options_size[HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT];
@@ -885,92 +735,89 @@ int ip6_ioam_set_rewrite (u8 **rwp, u32 trace_type, u32 trace_option_elts,
   rnd_size = (size + 7) & ~7;
 
   /* allocate it, zero-fill / pad by construction */
-  vec_validate (rewrite, rnd_size-1);
+  vec_validate (rewrite, rnd_size - 1);
 
   hbh = (ip6_hop_by_hop_header_t *) rewrite;
   /* Length of header in 8 octet units, not incl first 8 octets */
-  hbh->length = (rnd_size>>3) - 1;
-  current = (u8 *)(hbh+1);
-  
-  if (trace_option_elts)
+  hbh->length = (rnd_size >> 3) - 1;
+  current = (u8 *) (hbh + 1);
+
+  if (has_trace_option
+      && hm->add_options[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST] != 0)
     {
-      trace_option = (ioam_trace_option_t *)current;
-      trace_option->hdr.type = HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST
-        | HBH_OPTION_TYPE_DATA_CHANGE_ENROUTE;
-      trace_option->hdr.length = 
-               2 /*ioam_trace_type,data_list_elts_left */ + 
-              trace_option_elts * trace_data_size;
-      trace_option->ioam_trace_type = trace_type & TRACE_TYPE_MASK;
-      trace_option->data_list_elts_left = trace_option_elts;
-      current += sizeof (ioam_trace_option_t) + 
-        trace_option_elts * trace_data_size;
+      if (0 != (hm->options_size[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST]))
+	{
+	  trace_data_size =
+	    hm->options_size[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST];
+	  if (0 ==
+	      hm->add_options[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST] (current,
+								     &trace_data_size))
+	    current += hm->options_size[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST];
+	}
     }
-  if (has_pot_option && hm->add_options[HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT] != 0)
+  if (has_pot_option
+      && hm->add_options[HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT] != 0)
     {
-      if (0 == hm->add_options[HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT](current,
-					hm->options_size[HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT]))
-	  current += sizeof (hm->options_size[HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT]);
+      pot_data_size = hm->options_size[HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT];
+      if (0 ==
+	  hm->add_options[HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT] (current,
+								  &pot_data_size))
+	current +=
+	  sizeof (hm->options_size[HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT]);
     }
-  
+
   *rwp = rewrite;
   return 0;
 }
 
 clib_error_t *
-clear_ioam_rewrite_fn(void)
+clear_ioam_rewrite_fn (void)
 {
   ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
 
-  vec_free(hm->rewrite);
+  vec_free (hm->rewrite);
   hm->rewrite = 0;
-  hm->node_id = 0;
-  hm->app_data = 0;
-  hm->trace_type = 0;
-  hm->trace_option_elts = 0;
+  hm->has_trace_option = 0;
   hm->has_pot_option = 0;
   hm->has_ppc_option = 0;
-  hm->trace_tsp = TSP_MICROSECONDS; 
 
   return 0;
 }
 
-clib_error_t * clear_ioam_rewrite_command_fn (vlib_main_t * vm,
-                                 unformat_input_t * input,
-                                 vlib_cli_command_t * cmd)
+clib_error_t *
+clear_ioam_rewrite_command_fn (vlib_main_t * vm,
+			       unformat_input_t * input,
+			       vlib_cli_command_t * cmd)
 {
-  return(clear_ioam_rewrite_fn());
+  return (clear_ioam_rewrite_fn ());
 }
-  
-VLIB_CLI_COMMAND (ip6_clear_ioam_trace_cmd, static) = {
-  .path = "clear ioam rewrite",
-  .short_help = "clear ioam rewrite",
-  .function = clear_ioam_rewrite_command_fn,
-};
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip6_clear_ioam_rewrite_cmd, static) =
+{
+.path = "clear ioam rewrite",.short_help = "clear ioam rewrite",.function =
+    clear_ioam_rewrite_command_fn,};
+/* *INDENT-ON* */
 
 clib_error_t *
-ip6_ioam_trace_profile_set(u32 trace_option_elts, u32 trace_type, u32 node_id,
-                           u32 app_data, int has_pot_option, u32 trace_tsp, 
-                           int has_ppc_option)
+ip6_ioam_enable (int has_trace_option, int has_pot_option, int has_ppc_option)
 {
   int rv;
   ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
-  rv = ip6_ioam_set_rewrite (&hm->rewrite, trace_type, trace_option_elts,
-                             has_pot_option, has_ppc_option);
+  rv = ip6_ioam_set_rewrite (&hm->rewrite, has_trace_option,
+			     has_pot_option, has_ppc_option);
 
   switch (rv)
     {
     case 0:
-      hm->node_id = node_id;
-      hm->app_data = app_data;
-      hm->trace_type = trace_type;
-      hm->trace_option_elts = trace_option_elts;
+      hm->has_trace_option = has_trace_option;
       hm->has_pot_option = has_pot_option;
       hm->has_ppc_option = has_ppc_option;
-      hm->trace_tsp = trace_tsp;
       break;
 
     default:
-      return clib_error_return_code(0, rv, 0, "ip6_ioam_set_rewrite returned %d", rv);
+      return clib_error_return_code (0, rv, 0,
+				     "ip6_ioam_set_rewrite returned %d", rv);
     }
 
   return 0;
@@ -979,126 +826,119 @@ ip6_ioam_trace_profile_set(u32 trace_option_elts, u32 trace_type, u32 node_id,
 
 static clib_error_t *
 ip6_set_ioam_rewrite_command_fn (vlib_main_t * vm,
-                                 unformat_input_t * input,
-                                 vlib_cli_command_t * cmd)
+				 unformat_input_t * input,
+				 vlib_cli_command_t * cmd)
 {
-  u32 trace_option_elts = 0;
-  u32 trace_type = 0, node_id = 0; 
-  u32 app_data = 0, trace_tsp = TSP_MICROSECONDS;
+  int has_trace_option = 0;
   int has_pot_option = 0;
   int has_ppc_option = 0;
-  clib_error_t * rv = 0;
-  
+  clib_error_t *rv = 0;
+
   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
     {
-      if (unformat (input, "trace-type 0x%x trace-elts %d "
-                           "trace-tsp %d node-id 0x%x app-data 0x%x", 
-                      &trace_type, &trace_option_elts, &trace_tsp,
-                      &node_id, &app_data))
-            ;
+      if (unformat (input, "trace"))
+	has_trace_option = 1;
       else if (unformat (input, "pot"))
-        has_pot_option = 1;
+	has_pot_option = 1;
       else if (unformat (input, "ppc encap"))
-        has_ppc_option = PPC_ENCAP;
+	has_ppc_option = PPC_ENCAP;
       else if (unformat (input, "ppc decap"))
-        has_ppc_option = PPC_DECAP;
+	has_ppc_option = PPC_DECAP;
       else if (unformat (input, "ppc none"))
-        has_ppc_option = PPC_NONE;
+	has_ppc_option = PPC_NONE;
       else
-        break;
+	break;
     }
-  
-    
-    rv = ip6_ioam_trace_profile_set(trace_option_elts, trace_type, node_id,
-                           app_data, has_pot_option, trace_tsp, has_ppc_option);
 
-    return rv;
+
+  rv = ip6_ioam_enable (has_trace_option, has_pot_option, has_ppc_option);
+
+  return rv;
 }
 
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip6_set_ioam_rewrite_cmd, static) =
+{
+.path = "set ioam rewrite",.short_help =
+    "set ioam [trace] [pot] [ppc <encap|decap>]",.function =
+    ip6_set_ioam_rewrite_command_fn,};
+/* *INDENT-ON* */
 
-VLIB_CLI_COMMAND (ip6_set_ioam_rewrite_cmd, static) = {
-  .path = "set ioam rewrite",
-  .short_help = "set ioam rewrite trace-type <0x1f|0x3|0x9|0x11|0x19> trace-elts <nn> trace-tsp <0|1|2|3> node-id <node id in hex> app-data <app_data in hex> [pot] [ppc <encap|decap>]",
-  .function = ip6_set_ioam_rewrite_command_fn,
-};
-  
 static clib_error_t *
 ip6_show_ioam_summary_cmd_fn (vlib_main_t * vm,
-                      unformat_input_t * input,
-                      vlib_cli_command_t * cmd)
+			      unformat_input_t * input,
+			      vlib_cli_command_t * cmd)
 {
   ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
   u8 *s = 0;
 
 
-  if (!is_zero_ip6_address(&hm->adj))
-  {
-  s = format(s, "              REWRITE FLOW CONFIGS - \n");
-  s = format(s, "               Destination Address : %U\n",
-            format_ip6_address, &hm->adj, sizeof(ip6_address_t));
-  s = format(s, "                    Flow operation : %d (%s)\n", hm->ioam_flag,
-           (hm->ioam_flag == IOAM_HBYH_ADD) ? "Add" : 
-          ((hm->ioam_flag == IOAM_HBYH_MOD) ? "Mod" : "Pop"));
-  } 
-  else 
-  {
-  s = format(s, "              REWRITE FLOW CONFIGS - Not configured\n");
-  }
-
-  if (hm->trace_option_elts)
-  {
-  s = format(s, " HOP BY HOP OPTIONS - TRACE CONFIG - \n");
-  s = format(s, "                        Trace Type : 0x%x (%d)\n", 
-          hm->trace_type, hm->trace_type);
-  s = format(s, "         Trace timestamp precision : %d (%s)\n", hm->trace_tsp,
-       (hm->trace_tsp == TSP_SECONDS) ? "Seconds" : 
-      ((hm->trace_tsp == TSP_MILLISECONDS) ? "Milliseconds" : 
-     (((hm->trace_tsp == TSP_MICROSECONDS) ? "Microseconds" : "Nanoseconds"))));
-  s = format(s, "                Num of trace nodes : %d\n", 
-          hm->trace_option_elts);
-  s = format(s, "                           Node-id : 0x%x (%d)\n", 
-          hm->node_id, hm->node_id);
-  s = format(s, "                          App Data : 0x%x (%d)\n", 
-          hm->app_data, hm->app_data);
-  }
+  if (!is_zero_ip6_address (&hm->adj))
+    {
+      s = format (s, "              REWRITE FLOW CONFIGS - \n");
+      s = format (s, "               Destination Address : %U\n",
+		  format_ip6_address, &hm->adj, sizeof (ip6_address_t));
+      s =
+	format (s, "                    Flow operation : %d (%s)\n",
+		hm->ioam_flag,
+		(hm->ioam_flag ==
+		 IOAM_HBYH_ADD) ? "Add" : ((hm->ioam_flag ==
+					    IOAM_HBYH_MOD) ? "Mod" : "Pop"));
+    }
   else
-  {
-  s = format(s, " HOP BY HOP OPTIONS - TRACE CONFIG - Not configured\n");
-  }
+    {
+      s = format (s, "              REWRITE FLOW CONFIGS - Not configured\n");
+    }
+
 
-  s = format(s, "                        POT OPTION - %d (%s)\n", 
-          hm->has_pot_option, (hm->has_pot_option?"Enabled":"Disabled"));
+  s = format (s, "                        TRACE OPTION - %d (%s)\n",
+	      hm->has_trace_option,
+	      (hm->has_trace_option ? "Enabled" : "Disabled"));
+  if (hm->has_trace_option)
+    s =
+      format (s,
+	      "Try 'show ioam trace and show ioam-trace profile' for more information\n");
+
+
+  s = format (s, "                        POT OPTION - %d (%s)\n",
+	      hm->has_pot_option,
+	      (hm->has_pot_option ? "Enabled" : "Disabled"));
   if (hm->has_pot_option)
-    s = format(s, "Try 'show ioam pot and show pot profile' for more information\n");
+    s =
+      format (s,
+	      "Try 'show ioam pot and show pot profile' for more information\n");
 
-  s = format(s, "         EDGE TO EDGE - PPC OPTION - %d (%s)\n", 
-         hm->has_ppc_option, ppc_state[hm->has_ppc_option]);
+  s = format (s, "         EDGE TO EDGE - PPC OPTION - %d (%s)\n",
+	      hm->has_ppc_option, ppc_state[hm->has_ppc_option]);
   if (hm->has_ppc_option)
-    s = format(s, "Try 'show ioam ppc' for more information\n");
+    s = format (s, "Try 'show ioam ppc' for more information\n");
 
-  vlib_cli_output(vm, "%v", s);
-  vec_free(s);
+  vlib_cli_output (vm, "%v", s);
+  vec_free (s);
   return 0;
 }
 
-VLIB_CLI_COMMAND (ip6_show_ioam_run_cmd, static) = {
-  .path = "show ioam summary",
-  .short_help = "Summary of IOAM configuration",
-  .function = ip6_show_ioam_summary_cmd_fn,
-};
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip6_show_ioam_run_cmd, static) =
+{
+.path = "show ioam summary",.short_help =
+    "Summary of IOAM configuration",.function =
+    ip6_show_ioam_summary_cmd_fn,};
+/* *INDENT-ON* */
 
-int ip6_ioam_set_destination (ip6_address_t *addr, u32 mask_width, u32 vrf_id,
-                              int is_add, int is_pop, int is_none)
+int
+ip6_ioam_set_destination (ip6_address_t * addr, u32 mask_width, u32 vrf_id,
+			  int is_add, int is_pop, int is_none)
 {
-  ip6_main_t * im = &ip6_main;
-  ip6_hop_by_hop_ioam_main_t * hm = &ip6_hop_by_hop_ioam_main;
-  ip_lookup_main_t * lm = &im->lookup_main;
-  ip_adjacency_t * adj;
+  ip6_main_t *im = &ip6_main;
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
+  ip_lookup_main_t *lm = &im->lookup_main;
+  ip_adjacency_t *adj;
   u32 fib_index;
   u32 len, adj_index;
   int i, rv;
-  uword * p;
-  BVT(clib_bihash_kv) kv, value;
+  uword *p;
+  BVT (clib_bihash_kv) kv, value;
 
   if ((is_add + is_pop + is_none) != 1)
     return VNET_API_ERROR_INVALID_VALUE_2;
@@ -1112,27 +952,28 @@ int ip6_ioam_set_destination (ip6_address_t *addr, u32 mask_width, u32 vrf_id,
   fib_index = p[0];
 
   len = vec_len (im->prefix_lengths_in_search_order);
-  
+
   for (i = 0; i < len; i++)
     {
       int dst_address_length = im->prefix_lengths_in_search_order[i];
-      ip6_address_t * mask = &im->fib_masks[dst_address_length];
-      
+      ip6_address_t *mask = &im->fib_masks[dst_address_length];
+
       if (dst_address_length != mask_width)
-        continue;
+	continue;
 
       kv.key[0] = addr->as_u64[0] & mask->as_u64[0];
       kv.key[1] = addr->as_u64[1] & mask->as_u64[1];
-      kv.key[2] = ((u64)((fib_index))<<32) | dst_address_length;
-      
-      rv = BV(clib_bihash_search_inline_2)(&im->ip6_lookup_table, &kv, &value);
+      kv.key[2] = ((u64) ((fib_index)) << 32) | dst_address_length;
+
+      rv =
+	BV (clib_bihash_search_inline_2) (&im->ip6_lookup_table, &kv, &value);
       if (rv == 0)
-        goto found;
+	goto found;
 
     }
   return VNET_API_ERROR_NO_SUCH_ENTRY;
-  
- found:
+
+found:
 
   /* Got it, modify as directed... */
   adj_index = value.value;
@@ -1150,21 +991,23 @@ int ip6_ioam_set_destination (ip6_address_t *addr, u32 mask_width, u32 vrf_id,
     adj->saved_lookup_next_index = adj->lookup_next_index;
 
   if (is_add)
-    adj->lookup_next_index = (ip_lookup_next_t) IP6_LOOKUP_NEXT_ADD_HOP_BY_HOP;
+    adj->lookup_next_index =
+      (ip_lookup_next_t) IP6_LOOKUP_NEXT_ADD_HOP_BY_HOP;
 
   if (is_pop)
-    adj->lookup_next_index = (ip_lookup_next_t) IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
+    adj->lookup_next_index =
+      (ip_lookup_next_t) IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
 
   hm->adj = *addr;
   hm->ioam_flag = (is_add ? IOAM_HBYH_ADD :
-                  (is_pop ? IOAM_HBYH_POP : IOAM_HBYH_MOD));
+		   (is_pop ? IOAM_HBYH_POP : IOAM_HBYH_MOD));
   return 0;
 }
-                              
+
 static clib_error_t *
 ip6_set_ioam_destination_command_fn (vlib_main_t * vm,
-                                     unformat_input_t * input,
-                                     vlib_cli_command_t * cmd)
+				     unformat_input_t * input,
+				     vlib_cli_command_t * cmd)
 {
   ip6_address_t addr;
   u32 mask_width = ~0;
@@ -1176,19 +1019,18 @@ ip6_set_ioam_destination_command_fn (vlib_main_t * vm,
 
   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
     {
-      if (unformat (input, "%U/%d", 
-                    unformat_ip6_address, &addr, &mask_width))
-        ;
+      if (unformat (input, "%U/%d", unformat_ip6_address, &addr, &mask_width))
+	;
       else if (unformat (input, "vrf-id %d", &vrf_id))
-        ;
+	;
       else if (unformat (input, "add"))
-        is_add = 1;
+	is_add = 1;
       else if (unformat (input, "pop"))
-        is_pop = 1;
+	is_pop = 1;
       else if (unformat (input, "none"))
-        is_none = 1;
+	is_none = 1;
       else
-        break;
+	break;
     }
 
   if ((is_add + is_pop + is_none) != 1)
@@ -1196,30 +1038,42 @@ ip6_set_ioam_destination_command_fn (vlib_main_t * vm,
   if (mask_width == ~0)
     return clib_error_return (0, "<address>/<mask-width> required");
 
-  rv = ip6_ioam_set_destination (&addr, mask_width, vrf_id, 
-                                 is_add, is_pop, is_none);
+  rv = ip6_ioam_set_destination (&addr, mask_width, vrf_id,
+				 is_add, is_pop, is_none);
 
   switch (rv)
     {
     case 0:
       break;
     default:
-      return clib_error_return (0, "ip6_ioam_set_destination returned %d", rv);
+      return clib_error_return (0, "ip6_ioam_set_destination returned %d",
+				rv);
     }
-  
+
   return 0;
 }
 
-VLIB_CLI_COMMAND (ip6_set_ioam_destination_cmd, static) = {
-  .path = "set ioam destination",
-  .short_help = "set ioam destination <ip6-address>/<width> add | pop | none",
-  .function = ip6_set_ioam_destination_command_fn,
-};
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip6_set_ioam_destination_cmd, static) =
+{
+.path = "set ioam destination",.short_help =
+    "set ioam destination <ip6-address>/<width> add | pop | none",.function
+    = ip6_set_ioam_destination_command_fn,};
+/* *INDENT-ON* */
 
 
-void vnet_register_ioam_end_of_path_callback (void *cb)
+void
+vnet_register_ioam_end_of_path_callback (void *cb)
 {
-  ip6_hop_by_hop_ioam_main_t * hm = &ip6_hop_by_hop_ioam_main;
+  ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
 
   hm->ioam_end_of_path_cb = cb;
 }
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/vnet/vnet/ip/ip6_hop_by_hop.h b/vnet/vnet/ip/ip6_hop_by_hop.h
index 0ee24b306a1..dae58619e24 100644
--- a/vnet/vnet/ip/ip6_hop_by_hop.h
+++ b/vnet/vnet/ip/ip6_hop_by_hop.h
@@ -38,12 +38,7 @@ typedef struct {
 
 
   /* Trace option */
-  u8 trace_type;
-  u8 trace_option_elts;
-
-  /* Configured node-id */
-  u32 node_id;
-  u32 app_data;
+  u8 has_trace_option;
 
   /* Pot option */
   u8 has_pot_option;
@@ -57,13 +52,12 @@ typedef struct {
 #define TSP_MILLISECONDS         1
 #define TSP_MICROSECONDS         2
 #define TSP_NANOSECONDS          3
-  /* Time stamp precision. This is enumerated to above four options */
-  u32 trace_tsp;
   
   /* Array of function pointers to ADD and POP HBH option handling routines */
   u8 options_size[256];
-  int (*add_options[256])(u8 *rewrite_string, u8 rewrite_size);
+  int (*add_options[256])(u8 *rewrite_string, u8 *rewrite_size);
   int (*pop_options[256])(ip6_header_t *ip, ip6_hop_by_hop_option_t *opt);
+  int (*get_sizeof_options[256])(u32 *rewrite_size);
   
   /* convenience */
   vlib_main_t * vlib_main;
@@ -74,8 +68,7 @@ extern ip6_hop_by_hop_ioam_main_t ip6_hop_by_hop_ioam_main;
 
 extern u8 * format_path_map(u8 * s, va_list * args);
 extern clib_error_t *
-ip6_ioam_trace_profile_set(u32 trace_option_elts, u32 trace_type, u32 node_id,
-                           u32 app_data, int has_pot_option, u32 trace_tsp,
+ip6_ioam_enable(int has_trace_option, int has_pot_option,
                            int has_e2e_option);
 extern int ip6_ioam_set_destination (ip6_address_t *addr, u32 mask_width,
                   u32 vrf_id, int is_add, int is_pop, int is_none);
@@ -110,12 +103,18 @@ static inline u8 is_zero_ip6_address (ip6_address_t *a)
 
 int ip6_hbh_add_register_option (u8 option,
 				 u8 size,
-				 int rewrite_options(u8 *rewrite_string, u8 size));
+				 int rewrite_options(u8 *rewrite_string, u8 *size));
 int ip6_hbh_add_unregister_option (u8 option);
 
 int ip6_hbh_pop_register_option (u8 option,
 				 int options(ip6_header_t *ip, ip6_hop_by_hop_option_t *opt));
 int ip6_hbh_pop_unregister_option (u8 option);
 
+int
+ip6_hbh_get_sizeof_register_option (u8 option,
+                            int get_sizeof_hdr_options(u32 *rewrite_size));
 
+int
+ip6_ioam_set_rewrite (u8 ** rwp, int has_trace_option,
+		      int has_pot_option, int has_ppc_option);
 #endif /* __included_ip6_hop_by_hop_ioam_h__ */
diff --git a/vnet/vnet/ip/ip6_hop_by_hop_packet.h b/vnet/vnet/ip/ip6_hop_by_hop_packet.h
index 0a2c3d02ac1..8c358334c56 100644
--- a/vnet/vnet/ip/ip6_hop_by_hop_packet.h
+++ b/vnet/vnet/ip/ip6_hop_by_hop_packet.h
@@ -43,134 +43,6 @@ typedef struct {
 #define HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT 60   /* Third highest bit set (change en-route) */
 #define HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE 29
 
-/*
-typedef struct {
-  u32 ttl_node_id;
-  u16 ingress_if;
-  u16 egress_if;
-  u32 timestamp;
-  u32 app_data;
-} ioam_trace_data_list_element_t;
-*/
-
-#define    BIT_TTL_NODEID       (1<<0)
-#define    BIT_ING_INTERFACE    (1<<1) 
-#define    BIT_EGR_INTERFACE    (1<<2) 
-#define    BIT_TIMESTAMP        (1<<3) 
-#define    BIT_APPDATA          (1<<4) 
-#define    TRACE_TYPE_MASK      0x1F /* Mask of all above bits */
-
-/*
-     0x00011111  iOAM-trace-type is 0x00011111 then the format of node
-        data is:
-  
-          0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |   Hop_Lim     |              node_id                          |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |     ingress_if_id             |         egress_if_id          |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         +                           timestamp                           +
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |                            app_data                           |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-  
-*/
-#define   TRACE_TYPE_IF_TS_APP   0x1f
-typedef struct {
-  u32 ttl_node_id;
-  u16 ingress_if;
-  u16 egress_if;
-  u32 timestamp;
-  u32 app_data;
-} ioam_trace_if_ts_app_t;
-
-/*
-     0x00000111  iOAM-trace-type is 0x00000111 then the format is:
-  
-          0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |   Hop_Lim     |              node_id                          |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |     ingress_if_id             |         egress_if_id          |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-  
-*/
-
-#define   TRACE_TYPE_IF   0x03
-typedef struct {
-  u32 ttl_node_id;
-  u16 ingress_if;
-  u16 egress_if;
-} ioam_trace_if_t;
-
-/*
-     0x00001001  iOAM-trace-type is 0x00001001 then the format is:
-  
-          0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |   Hop_Lim     |              node_id                          |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         +                           timestamp                           +
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-  
-*/
-
-#define   TRACE_TYPE_TS   0x09
-typedef struct {
-  u32 ttl_node_id;
-  u32 timestamp;
-} ioam_trace_ts_t;
-
-/*
-     0x00010001  iOAM-trace-type is 0x00010001 then the format is:
-  
-  
-          0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |   Hop_Lim     |              node_id                          |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |                            app_data                           |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-  
-*/
-
-
-#define   TRACE_TYPE_APP   0x11
-typedef struct {
-  u32 ttl_node_id;
-  u32 app_data;
-} ioam_trace_app_t;
-
-/*
-  
-     0x00011001  iOAM-trace-type is 0x00011001 then the format is:
-  
-          0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |   Hop_Lim     |              node_id                          |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         +                           timestamp                           +
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |                            app_data                           |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-*/
-
-#define   TRACE_TYPE_TS_APP   0x19
-typedef struct {
-  u32 ttl_node_id;
-  u32 timestamp;
-  u32 app_data;
-} ioam_trace_ts_app_t;
-
-
-typedef CLIB_PACKED(struct {
-  ip6_hop_by_hop_option_t hdr;
-  u8 ioam_trace_type;
-  u8 data_list_elts_left;
-  u32 elts[0]; /* Variable type. So keep it generic */
-}) ioam_trace_option_t;
-
 
 typedef CLIB_PACKED(struct {
   ip6_hop_by_hop_option_t hdr;
diff --git a/vpp-api-test/vat/api_format.c b/vpp-api-test/vat/api_format.c
index 7458ae3c21d..3b93c985791 100644
--- a/vpp-api-test/vat/api_format.c
+++ b/vpp-api-test/vat/api_format.c
@@ -3407,9 +3407,8 @@ _(want_stats_reply)					\
 _(cop_interface_enable_disable_reply)			\
 _(cop_whitelist_enable_disable_reply)                   \
 _(sw_interface_clear_stats_reply)                       \
-_(trace_profile_add_reply)                              \
-_(trace_profile_apply_reply)                            \
-_(trace_profile_del_reply)                              \
+_(ioam_enable_reply)                              \
+_(ioam_disable_reply)                              \
 _(lisp_add_del_locator_reply)                           \
 _(lisp_add_del_local_eid_reply)                         \
 _(lisp_add_del_remote_mapping_reply)                    \
@@ -3599,9 +3598,8 @@ _(COP_INTERFACE_ENABLE_DISABLE_REPLY, cop_interface_enable_disable_reply) \
 _(COP_WHITELIST_ENABLE_DISABLE_REPLY, cop_whitelist_enable_disable_reply) \
 _(GET_NODE_GRAPH_REPLY, get_node_graph_reply)                           \
 _(SW_INTERFACE_CLEAR_STATS_REPLY, sw_interface_clear_stats_reply)      \
-_(TRACE_PROFILE_ADD_REPLY, trace_profile_add_reply)                   \
-_(TRACE_PROFILE_APPLY_REPLY, trace_profile_apply_reply)               \
-_(TRACE_PROFILE_DEL_REPLY, trace_profile_del_reply)                     \
+_(IOAM_ENABLE_REPLY, ioam_enable_reply)                   \
+_(IOAM_DISABLE_REPLY, ioam_disable_reply)                     \
 _(LISP_ADD_DEL_LOCATOR_SET_REPLY, lisp_add_del_locator_set_reply)       \
 _(LISP_ADD_DEL_LOCATOR_REPLY, lisp_add_del_locator_reply)               \
 _(LISP_ADD_DEL_LOCAL_EID_REPLY, lisp_add_del_local_eid_reply)           \
@@ -7172,24 +7170,20 @@ api_l2_patch_add_del (vat_main_t * vam)
 }
 
 static int
-api_trace_profile_add (vat_main_t * vam)
+api_ioam_enable (vat_main_t * vam)
 {
   unformat_input_t *input = vam->input;
-  vl_api_trace_profile_add_t *mp;
+  vl_api_ioam_enable_t *mp;
   f64 timeout;
   u32 id = 0;
-  u32 trace_option_elts = 0;
-  u32 trace_type = 0, node_id = 0, app_data = 0, trace_tsp = 2;
+  int has_trace_option = 0;
   int has_pow_option = 0;
   int has_ppc_option = 0;
 
   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
     {
-      if (unformat (input, "id %d trace-type 0x%x trace-elts %d "
-		    "trace-tsp %d node-id 0x%x app-data 0x%x",
-		    &id, &trace_type, &trace_option_elts, &trace_tsp,
-		    &node_id, &app_data))
-	;
+      if (unformat (input, "trace"))
+	has_trace_option = 1;
       else if (unformat (input, "pow"))
 	has_pow_option = 1;
       else if (unformat (input, "ppc encap"))
@@ -7201,15 +7195,11 @@ api_trace_profile_add (vat_main_t * vam)
       else
 	break;
     }
-  M (TRACE_PROFILE_ADD, trace_profile_add);
+  M (IOAM_ENABLE, ioam_enable);
   mp->id = htons (id);
-  mp->trace_type = trace_type;
-  mp->trace_num_elt = trace_option_elts;
   mp->trace_ppc = has_ppc_option;
-  mp->trace_app_data = htonl (app_data);
   mp->pow_enable = has_pow_option;
-  mp->trace_tsp = trace_tsp;
-  mp->node_id = htonl (node_id);
+  mp->trace_enable = has_trace_option;
 
   S;
   W;
@@ -7218,78 +7208,14 @@ api_trace_profile_add (vat_main_t * vam)
 
 }
 
-static int
-api_trace_profile_apply (vat_main_t * vam)
-{
-  unformat_input_t *input = vam->input;
-  vl_api_trace_profile_apply_t *mp;
-  f64 timeout;
-  ip6_address_t addr;
-  u32 mask_width = ~0;
-  int is_add = 0;
-  int is_pop = 0;
-  int is_none = 0;
-  u32 vrf_id = 0;
-  u32 id = 0;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "%U/%d", unformat_ip6_address, &addr, &mask_width))
-	;
-      else if (unformat (input, "id %d", &id))
-	;
-      else if (unformat (input, "vrf-id %d", &vrf_id))
-	;
-      else if (unformat (input, "add"))
-	is_add = 1;
-      else if (unformat (input, "pop"))
-	is_pop = 1;
-      else if (unformat (input, "none"))
-	is_none = 1;
-      else
-	break;
-    }
-
-  if ((is_add + is_pop + is_none) != 1)
-    {
-      errmsg ("One of (add, pop, none) required");
-      return -99;
-    }
-  if (mask_width == ~0)
-    {
-      errmsg ("<address>/<mask-width> required");
-      return -99;
-    }
-  M (TRACE_PROFILE_APPLY, trace_profile_apply);
-  clib_memcpy (mp->dest_ipv6, &addr, sizeof (mp->dest_ipv6));
-  mp->id = htons (id);
-  mp->prefix_length = htonl (mask_width);
-  mp->vrf_id = htonl (vrf_id);
-  if (is_add)
-    mp->trace_op = IOAM_HBYH_ADD;
-  else if (is_pop)
-    mp->trace_op = IOAM_HBYH_POP;
-  else
-    mp->trace_op = IOAM_HBYH_MOD;
-
-  if (is_none)
-    mp->enable = 0;
-  else
-    mp->enable = 1;
-
-  S;
-  W;
-
-  return 0;
-}
 
 static int
-api_trace_profile_del (vat_main_t * vam)
+api_ioam_disable (vat_main_t * vam)
 {
-  vl_api_trace_profile_del_t *mp;
+  vl_api_ioam_disable_t *mp;
   f64 timeout;
 
-  M (TRACE_PROFILE_DEL, trace_profile_del);
+  M (IOAM_DISABLE, ioam_disable);
   S;
   W;
   return 0;
@@ -15879,12 +15805,8 @@ _(cop_whitelist_enable_disable, "<intfc> | sw_if_index <nn>\n"		\
   "fib-id <nn> [ip4][ip6][default]")					\
 _(get_node_graph, " ")                                                  \
 _(sw_interface_clear_stats,"<intfc> | sw_if_index <nn>")                \
-_(trace_profile_add, "id <nn> trace-type <0x1f|0x3|0x9|0x11|0x19> "     \
-  "trace-elts <nn> trace-tsp <0|1|2|3> node-id <node id in hex> "       \
-  "app-data <app_data in hex> [pow] [ppc <encap|decap>]")               \
-_(trace_profile_apply, "id <nn> <ip6-address>/<width>"                  \
-  " vrf_id <nn>  add | pop | none")                                     \
-_(trace_profile_del, "")                                                \
+_(ioam_enable, "[trace] [pow] [ppc <encap|decap>]")               \
+_(ioam_disable, "")                                                \
 _(lisp_add_del_locator_set, "locator-set <locator_name> [iface <intf> |"\
                             " sw_if_index <sw_if_index> p <priority> "  \
                             "w <weight>] [del]")                        \
diff --git a/vpp/vpp-api/api.c b/vpp/vpp-api/api.c
index ead5f0cd925..c339cb44c60 100644
--- a/vpp/vpp-api/api.c
+++ b/vpp/vpp-api/api.c
@@ -343,9 +343,8 @@ _(COP_INTERFACE_ENABLE_DISABLE, cop_interface_enable_disable)		\
 _(COP_WHITELIST_ENABLE_DISABLE, cop_whitelist_enable_disable)		\
 _(GET_NODE_GRAPH, get_node_graph)                                       \
 _(SW_INTERFACE_CLEAR_STATS, sw_interface_clear_stats)                   \
-_(TRACE_PROFILE_ADD, trace_profile_add)                                 \
-_(TRACE_PROFILE_APPLY, trace_profile_apply)                             \
-_(TRACE_PROFILE_DEL, trace_profile_del)                                 \
+_(IOAM_ENABLE, ioam_enable)                                 \
+_(IOAM_DISABLE, ioam_disable)                                 \
 _(LISP_ADD_DEL_LOCATOR_SET, lisp_add_del_locator_set)                   \
 _(LISP_ADD_DEL_LOCATOR, lisp_add_del_locator)                           \
 _(LISP_ADD_DEL_LOCAL_EID, lisp_add_del_local_eid)                       \
@@ -7035,56 +7034,30 @@ vl_api_get_node_graph_t_handler (vl_api_get_node_graph_t * mp)
   /* *INDENT-ON* */
 }
 
-static void vl_api_trace_profile_add_t_handler
-  (vl_api_trace_profile_add_t * mp)
+static void
+vl_api_ioam_enable_t_handler (vl_api_ioam_enable_t * mp)
 {
   int rv = 0;
-  vl_api_trace_profile_add_reply_t *rmp;
+  vl_api_ioam_enable_reply_t *rmp;
   clib_error_t *error;
 
   /* Ignoring the profile id as currently a single profile
    * is supported */
-  error = ip6_ioam_trace_profile_set (mp->trace_num_elt, mp->trace_type,
-				      ntohl (mp->node_id),
-				      ntohl (mp->trace_app_data),
-				      mp->pow_enable, mp->trace_tsp,
-				      mp->trace_ppc);
+  error = ip6_ioam_enable (mp->trace_enable, mp->pow_enable, mp->trace_ppc);
   if (error)
     {
       clib_error_report (error);
       rv = clib_error_get_code (error);
     }
 
-  REPLY_MACRO (VL_API_TRACE_PROFILE_ADD_REPLY);
-}
-
-static void vl_api_trace_profile_apply_t_handler
-  (vl_api_trace_profile_apply_t * mp)
-{
-  int rv = 0;
-  vl_api_trace_profile_apply_reply_t *rmp;
-
-  if (mp->enable != 0)
-    {
-      rv = ip6_ioam_set_destination ((ip6_address_t *) (&mp->dest_ipv6),
-				     ntohl (mp->prefix_length),
-				     ntohl (mp->vrf_id),
-				     mp->trace_op == IOAM_HBYH_ADD,
-				     mp->trace_op == IOAM_HBYH_POP,
-				     mp->trace_op == IOAM_HBYH_MOD);
-    }
-  else
-    {
-      //ip6_ioam_clear_destination(&ip6, mp->prefix_length, mp->vrf_id);
-    }
-  REPLY_MACRO (VL_API_TRACE_PROFILE_APPLY_REPLY);
+  REPLY_MACRO (VL_API_IOAM_ENABLE_REPLY);
 }
 
-static void vl_api_trace_profile_del_t_handler
-  (vl_api_trace_profile_del_t * mp)
+static void
+vl_api_ioam_disable_t_handler (vl_api_ioam_disable_t * mp)
 {
   int rv = 0;
-  vl_api_trace_profile_del_reply_t *rmp;
+  vl_api_ioam_disable_reply_t *rmp;
   clib_error_t *error;
 
   error = clear_ioam_rewrite_fn ();
@@ -7094,7 +7067,7 @@ static void vl_api_trace_profile_del_t_handler
       rv = clib_error_get_code (error);
     }
 
-  REPLY_MACRO (VL_API_TRACE_PROFILE_DEL_REPLY);
+  REPLY_MACRO (VL_API_IOAM_DISABLE_REPLY);
 }
 
 static void
diff --git a/vpp/vpp-api/vpe.api b/vpp/vpp-api/vpe.api
index 51862f744a7..5c1502a890a 100644
--- a/vpp/vpp-api/vpe.api
+++ b/vpp/vpp-api/vpe.api
@@ -3962,87 +3962,50 @@ define sw_interface_clear_stats_reply
   i32 retval;
 };
 
-/** \brief IOAM Trace : Set TRACE profile
+/** \brief IOAM enable : Enable in-band OAM
     @param id - profile id
-    @param trace_type - Trace type
-    @param trace_num_elt - Number of nodes in trace path
     @param trace_ppc - Trace PPC (none/encap/decap)
-    @param trace_tsp - Trace timestamp precision (0-sec,1-ms,2-us,3-ns)
-    @param trace_app_data - Trace application data, can be any 4 bytes
     @param pow_enable - Proof of Work enabled or not flag
-    @param node_id - Id of this node
+    @param trace_enable - iOAM Trace enabled or not flag
 */
-define trace_profile_add
+define ioam_enable
 {
   u32 client_index;
   u32 context;
   u16 id;
-  u8 trace_type;
-  u8 trace_num_elt;
   u8 trace_ppc;
-  u8 trace_tsp;
-  u32 trace_app_data;
   u8 pow_enable;
+  u8 trace_enable;
   u32 node_id;
 };
 
-/** \brief Trace profile add / del response
+/** \brief iOAM Trace profile add / del response
     @param context - sender context, to match reply w/ request
     @param retval - return value for request
 */
-define trace_profile_add_reply
+define ioam_enable_reply
 {
   u32 context;
   i32 retval;
 };
 
-/** \brief IOAM Trace enable trace profile for a flow
-    @param id - id of the trace profile to be applied
-    @param dest_ipv6 - Destination IPv6 address
-    @param prefix_length - prefix mask
-    @param vrf_id - VRF ID
-    @param trace_op - Trace operation (add/mod/del)
-    @param enable - apply/remove the trace profile for the flow
-*/
-define trace_profile_apply
-{
-  u32 client_index;
-  u32 context;
-  u16 id;
-  u8 dest_ipv6[16];
-  u32 prefix_length;
-  u32 vrf_id;
-  u8 trace_op;
-  u8 enable;
-};
-
-/** \brief Trace profile apply response
-    @param context - sender context, to match reply w/ request   
-    @param retval - return value for request
-*/
-define trace_profile_apply_reply
-{
-  u32 context;
-  i32 retval;
-};
-
-/** \brief Delete Trace Profile 
+/** \brief iOAM disable
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
     @param index - MAP Domain index
 */
-define trace_profile_del
+define ioam_disable
 {
   u32 client_index;
   u32 context;
   u16 id;
 };
 
-/** \brief Trace profile add / del response
+/** \brief iOAM disable response
     @param context - sender context, to match reply w/ request
     @param retval - return value for request
 */
-define trace_profile_del_reply
+define ioam_disable_reply
 {
   u32 context;
   i32 retval;