mdata: buffer metadata change tracker plugin

A handy tool in case you need to know which metadata will be changed
when a packet visits a certain node.

Reflect metadata changes into format functions used by the
vpp-specific wireshark dissector.

Type: feature

Signed-off-by: Dave Barach <dave@barachs.net>
Change-Id: I96fe8a24db4082bb29fe2a33cc522e8616a3a1bb

9 files changed, 794 insertions, 23 deletions

diff --git a/src/plugins/mdata/CMakeLists.txt b/src/plugins/mdata/CMakeLists.txt
new file mode 100644
index 00000000000..6f45281e2b3
--- /dev/null
+++ b/src/plugins/mdata/CMakeLists.txt
@@ -0,0 +1,24 @@
+# Copyright (c) 2019 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_vpp_plugin(mdata
+  SOURCES
+  mdata.c
+  mdata.h
+
+  API_FILES
+  mdata.api
+
+  API_TEST_SOURCES
+  mdata_test.c
+)
diff --git a/src/plugins/mdata/FEATURE.yaml b/src/plugins/mdata/FEATURE.yaml
new file mode 100644
index 00000000000..c71281d777b
--- /dev/null
+++ b/src/plugins/mdata/FEATURE.yaml
@@ -0,0 +1,13 @@
+name: Buffer Metadata Change Tracker (mdata)
+maintainer: Dave Barach <dave@barachs.net>
+features:
+  - Buffer Metadata Change Tracker
+description: "Buffer Metadata Change Tracker
+              Uses the before / after graph node main loop performance
+              callback hooks to snapshoot buffer metadata, then
+              compare and summarize results per-node.
+              Answers the question "what buffer metadata does a particular
+              graph node change?" by direct observation.
+              Zero performance impact until enabled."
+state: production
+properties: [API, CLI, MULTITHREAD]
diff --git a/src/plugins/mdata/mdata.api b/src/plugins/mdata/mdata.api
new file mode 100644
index 00000000000..de1c8e4c957
--- /dev/null
+++ b/src/plugins/mdata/mdata.api
@@ -0,0 +1,46 @@
+/*
+ * mdata.api - binary API skeleton
+ *
+ * Copyright (c) <current-year> <your-organization>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @file mdata.api
+ * @brief VPP control-plane API messages.
+ *
+ * This file defines VPP control-plane binary API messages which are generally
+ * called through a shared memory interface.
+ */
+
+/* Version and type recitations */
+
+option version = "0.1.0";
+import "vnet/interface_types.api";
+
+/** @brief API to enable / disable mdata on an interface
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param enable_disable - 1 to enable, 0 to disable the feature
+*/
+
+autoreply define mdata_enable_disable {
+  /* Client identifier, set from api_main.my_client_index */
+  u32 client_index;
+
+  /* Arbitrary context, so client can match reply to request */
+  u32 context;
+
+  /* Enable / disable the feature */
+  bool enable_disable;
+};
diff --git a/src/plugins/mdata/mdata.c b/src/plugins/mdata/mdata.c
new file mode 100644
index 00000000000..830673edeb8
--- /dev/null
+++ b/src/plugins/mdata/mdata.c
@@ -0,0 +1,493 @@
+/*
+ * mdata.c - Buffer metadata change tracker
+ *
+ * Copyright (c) 2019 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/vnet.h>
+#include <vnet/plugin/plugin.h>
+#include <mdata/mdata.h>
+
+#include <vlibapi/api.h>
+#include <vlibmemory/api.h>
+#include <vpp/app/version.h>
+#include <stdbool.h>
+
+#include <mdata/mdata.api_enum.h>
+#include <mdata/mdata.api_types.h>
+
+#define REPLY_MSG_ID_BASE mmp->msg_id_base
+#include <vlibapi/api_helper_macros.h>
+
+mdata_main_t mdata_main;
+
+/** @file buffer metadata change tracker
+ */
+
+static mdata_t mdata_none;
+
+/** Metadata tracking callback
+    before_or_after: 0 => before, 1=> after
+*/
+static void
+mdata_trace_callback (vlib_main_t * vm, u64 * c0, u64 * c1,
+		      vlib_node_runtime_t * node,
+		      vlib_frame_t * frame, int before_or_after)
+{
+  int i;
+  mdata_main_t *mm = &mdata_main;
+  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
+  u32 *from;
+  u32 n_left_from;
+  mdata_t *before, *modifies;
+  u8 *after;
+
+  /* Input nodes don't have frames, etc. */
+  if (frame == 0)
+    return;
+
+  n_left_from = frame->n_vectors;
+
+  if (n_left_from == 0)
+    return;
+
+  from = vlib_frame_vector_args (frame);
+
+  vlib_get_buffers (vm, from, bufs, n_left_from);
+  b = bufs;
+
+  if (before_or_after == 1 /* after */ )
+    goto after_pass;
+
+  /* Resize the per-thread "before" vector to cover the current frame */
+  vec_reset_length (mm->before_per_thread[vm->thread_index]);
+  vec_validate (mm->before_per_thread[vm->thread_index], n_left_from - 1);
+  before = mm->before_per_thread[vm->thread_index];
+  before->node_index = ~0;
+
+  /* Before we call the dispatch fn, copy metadata. */
+  while (n_left_from > 0)
+    {
+      clib_memcpy_fast (before->mdata, b[0], sizeof (before->mdata));
+      b++;
+      before++;
+      n_left_from--;
+    }
+  return;
+
+after_pass:
+
+  /* Recover the metadata copy we saved a moment ago */
+  before = mm->before_per_thread[vm->thread_index];
+
+  /* We'd better have the same number of buffers... */
+  ASSERT (n_left_from == vec_len (before));
+  ASSERT (node->node_index);
+
+  clib_spinlock_lock_if_init (&mm->modify_lock);
+
+  /*
+   * Resize the per-node accumulator vector as needed
+   * Paint the "no data" patter across any nodes we haven't seen yet
+   */
+  vec_validate_init_empty (mm->modifies, node->node_index, mdata_none);
+  modifies = vec_elt_at_index (mm->modifies, node->node_index);
+  modifies->node_index = node->node_index;
+  before = mm->before_per_thread[vm->thread_index];
+
+  /* Walk the frame */
+  while (n_left_from > 0)
+    {
+      after = (u8 *) b[0];
+
+      /* Compare metadata before and after node dispatch fn */
+      for (i = 0; i < ARRAY_LEN (before->mdata); i++)
+	{
+	  /* Mark mdata octet changed */
+	  if (before->mdata[i] != after[i])
+	    modifies->mdata[i] = 0xff;
+	}
+
+      b++;
+      before++;
+      n_left_from--;
+    }
+
+  clib_spinlock_unlock_if_init (&mm->modify_lock);
+}
+
+int
+mdata_enable_disable (mdata_main_t * mmp, int enable_disable)
+{
+  int rv = 0;
+  vlib_thread_main_t *thread_main = vlib_get_thread_main ();
+  int i;
+
+  if (mmp->modify_lock == 0 && thread_main->n_vlib_mains > 1)
+    clib_spinlock_init (&mmp->modify_lock);
+
+  if (vec_len (mmp->before_per_thread) == 0)
+    {
+      mdata_none.node_index = ~0;
+      vec_validate (mmp->before_per_thread, vec_len (vlib_mains) - 1);
+    }
+
+  /* Reset the per-node accumulator, see vec_validate_init_empty above */
+  vec_reset_length (mmp->modifies);
+
+  for (i = 0; i < vec_len (vlib_mains); i++)
+    {
+      if (vlib_mains[i] == 0)
+	continue;
+
+      clib_callback_enable_disable
+	(vlib_mains[i]->vlib_node_runtime_perf_counter_cbs,
+	 vlib_mains[i]->vlib_node_runtime_perf_counter_cb_tmp,
+	 vlib_mains[i]->worker_thread_main_loop_callback_lock,
+	 (void *) mdata_trace_callback, enable_disable);
+    }
+
+  return rv;
+}
+
+static clib_error_t *
+mdata_enable_disable_command_fn (vlib_main_t * vm,
+				 unformat_input_t * input,
+				 vlib_cli_command_t * cmd)
+{
+  mdata_main_t *mmp = &mdata_main;
+  int enable_disable = 1;
+
+  int rv;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "disable") || unformat (input, "off"))
+	enable_disable = 0;
+      if (unformat (input, "enable") || unformat (input, "on"))
+	enable_disable = 1;
+      else
+	break;
+    }
+
+  rv = mdata_enable_disable (mmp, enable_disable);
+
+  switch (rv)
+    {
+    case 0:
+      break;
+
+    default:
+      return clib_error_return (0, "mdata_enable_disable returned %d", rv);
+    }
+  return 0;
+}
+
+/*?
+ * This command enables or disables buffer metadata change tracking
+ *
+ *@cliexpar
+ * To enable buffer metadata change tracking:
+ *@cliexstart{buffer metadata tracking on}
+ * Tracking enabled
+ *@cliexend
+ *
+ *@cliexstart{buffer metadata tracking off}
+ * Tracking disabled
+ *@cliexend
+?*/
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (mdata_enable_disable_command, static) =
+{
+  .path = "buffer metadata tracking",
+  .short_help = "buffer metadata tracking [on][off]",
+  .function = mdata_enable_disable_command_fn,
+};
+/* *INDENT-ON* */
+
+/* API message handler */
+static void vl_api_mdata_enable_disable_t_handler
+  (vl_api_mdata_enable_disable_t * mp)
+{
+  vl_api_mdata_enable_disable_reply_t *rmp;
+  mdata_main_t *mmp = &mdata_main;
+  int rv;
+
+  rv = mdata_enable_disable (mmp, (int) (mp->enable_disable));
+
+  REPLY_MACRO (VL_API_MDATA_ENABLE_DISABLE_REPLY);
+}
+
+/* API definitions */
+#include <mdata/mdata.api.c>
+
+static clib_error_t *
+mdata_init (vlib_main_t * vm)
+{
+  mdata_main_t *mmp = &mdata_main;
+  clib_error_t *error = 0;
+
+  mmp->vlib_main = vm;
+  mmp->vnet_main = vnet_get_main ();
+
+  /* Add our API messages to the global name_crc hash table */
+  mmp->msg_id_base = setup_message_id_table ();
+
+  return error;
+}
+
+VLIB_INIT_FUNCTION (mdata_init);
+
+/* *INDENT-OFF* */
+VLIB_PLUGIN_REGISTER () =
+{
+  .version = VPP_BUILD_VER,
+  .description = "Buffer metadata change tracker."
+};
+/* *INDENT-ON* */
+
+
+#define foreach_primary_metadata_field          \
+_(current_data)                                 \
+_(current_length)                               \
+_(flags)                                        \
+_(flow_id)                                      \
+_(ref_count)                                    \
+_(buffer_pool_index)                            \
+_(error)                                        \
+_(next_buffer)                                  \
+_(current_config_index)                         \
+_(punt_reason)
+
+#define foreach_opaque_metadata_field           \
+_(sw_if_index[0])                               \
+_(sw_if_index[1])                               \
+_(l2_hdr_offset)                                \
+_(l3_hdr_offset)                                \
+_(l4_hdr_offset)                                \
+_(feature_arc_index)                            \
+_(ip.adj_index[0])                              \
+_(ip.adj_index[1])                              \
+_(ip.flow_hash)                                 \
+_(ip.save_protocol)                             \
+_(ip.fib_index)                                 \
+_(ip.icmp.type)                                 \
+_(ip.icmp.code)                                 \
+_(ip.icmp.data)                                 \
+_(ip.reass.next_index)                          \
+_(ip.reass.error_next_index)                    \
+_(ip.reass.owner_thread_index)                  \
+_(ip.reass.ip_proto)                            \
+_(ip.reass.l4_src_port)                         \
+_(ip.reass.l4_dst_port)                         \
+_(ip.reass.estimated_mtu)                       \
+_(ip.reass.fragment_first)                      \
+_(ip.reass.fragment_last)                       \
+_(ip.reass.range_first)                         \
+_(ip.reass.range_last)                          \
+_(ip.reass.next_range_bi)                       \
+_(ip.reass.ip6_frag_hdr_offset)                 \
+_(mpls.ttl)                                     \
+_(mpls.exp)                                     \
+_(mpls.first)                                   \
+_(mpls.save_rewrite_length)                     \
+_(mpls.mpls_hdr_length)                         \
+_(mpls.bier.n_bytes)                            \
+_(l2.feature_bitmap)                            \
+_(l2.bd_index)                                  \
+_(l2.l2fib_sn)                                  \
+_(l2.l2_len)                                    \
+_(l2.shg)                                       \
+_(l2.bd_age)                                    \
+_(l2t.next_index)                               \
+_(l2t.session_index)                            \
+_(l2_classify.table_index)                      \
+_(l2_classify.opaque_index)                     \
+_(l2_classify.hash)                             \
+_(policer.index)                                \
+_(ipsec.sad_index)                              \
+_(ipsec.protect_index)                          \
+_(map.mtu)                                      \
+_(map_t.map_domain_index)			\
+_(map_t.v6.saddr)                               \
+_(map_t.v6.daddr)                               \
+_(map_t.v6.frag_offset)                         \
+_(map_t.v6.l4_offset)                           \
+_(map_t.v6.l4_protocol)                         \
+_(map_t.checksum_offset)			\
+_(map_t.mtu)                                    \
+_(ip_frag.mtu)                                  \
+_(ip_frag.next_index)                           \
+_(ip_frag.flags)                                \
+_(cop.current_config_index)                     \
+_(lisp.overlay_afi)                             \
+_(tcp.connection_index)                         \
+_(tcp.seq_number)                               \
+_(tcp.next_node_opaque)                         \
+_(tcp.seq_end)                                  \
+_(tcp.ack_number)                               \
+_(tcp.hdr_offset)                               \
+_(tcp.data_offset)                              \
+_(tcp.data_len)                                 \
+_(tcp.flags)                                    \
+_(snat.flags)
+
+#define foreach_opaque2_metadata_field          \
+_(qos.bits)                                     \
+_(qos.source)                                   \
+_(loop_counter)                                 \
+_(gbp.flags)                                    \
+_(gbp.sclass)                                   \
+_(gso_size)                                     \
+_(gso_l4_hdr_sz)                                \
+_(pg_replay_timestamp)
+
+static u8 *
+format_buffer_metadata_changes (u8 * s, va_list * args)
+{
+  mdata_main_t *mm = va_arg (*args, mdata_main_t *);
+  int verbose = va_arg (*args, int);
+  mdata_t *modifies;
+  vlib_buffer_t *b;
+  vnet_buffer_opaque_t *o;
+  vnet_buffer_opaque2_t *o2;
+  vlib_node_t *node;
+  int i, j;
+  int printed;
+
+  clib_spinlock_lock_if_init (&mm->modify_lock);
+
+  for (i = 0; i < vec_len (mm->modifies); i++)
+    {
+      modifies = vec_elt_at_index (mm->modifies, i);
+      node = vlib_get_node (mm->vlib_main, i);
+
+      /* No data for this node? */
+      if (modifies->node_index == ~0)
+	{
+	  if (verbose)
+	    s = format (s, "\n%v: no data\n", node->name);
+	  continue;
+	}
+
+      /* We visited the node, but it may not have changed any metadata... */
+      for (j = 0; j < ARRAY_LEN (modifies->mdata); j++)
+	{
+	  if (modifies->mdata[j])
+	    goto found;
+	}
+      s = format (s, "\n%v: no metadata changes\n", node->name);
+      continue;
+
+    found:
+      /* Fields which the node modifies will be non-zero */
+      b = (vlib_buffer_t *) (modifies->mdata);
+
+      /* Dump primary metadata changes */
+      s = format (s, "\n%v: ", node->name);
+
+      printed = 0;
+#define _(n) if (b->n) {s = format (s, "%s ", #n); printed = 1;}
+      foreach_primary_metadata_field;
+#undef _
+
+      if (printed == 0)
+	s = format (s, "no vlib_buffer_t metadata changes");
+
+      vec_add1 (s, '\n');
+
+      /*
+       * Dump opaque union changes.
+       * Hopefully this will give folks a clue about opaque
+       * union data conflicts. That's the point of the exercise...
+       */
+      o = vnet_buffer (b);
+      printed = 0;
+      s = format (s, "  vnet_buffer_t: ");
+
+#define _(n) if (o->n) {s = format (s, "%s ", #n); printed = 1;}
+      foreach_opaque_metadata_field;
+#undef _
+
+      if (printed == 0)
+	s = format (s, "no changes");
+
+      vec_add1 (s, '\n');
+
+      o2 = vnet_buffer2 (b);
+      printed = 0;
+      s = format (s, "  vnet_buffer2_t: ");
+
+#define _(n) if (o2->n) {s = format (s, "%s ", #n); printed = 1;}
+      foreach_opaque2_metadata_field;
+#undef _
+      if (printed == 0)
+	s = format (s, "no changes");
+
+      vec_add1 (s, '\n');
+
+    }
+
+  clib_spinlock_unlock_if_init (&mm->modify_lock);
+
+  return s;
+}
+
+static clib_error_t *
+show_metadata_command_fn (vlib_main_t * vm,
+			  unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  int verbose = 0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "verbose %=", &verbose, 1))
+	;
+      else
+	break;
+    }
+
+  vlib_cli_output (vm, "%U", format_buffer_metadata_changes, &mdata_main,
+		   verbose);
+  return 0;
+}
+
+/*?
+ * This command displays buffer metadata change information
+ *@cliexpar
+ * How to display buffer metadata change information
+ *@cliexstart{show buffer metadata}
+ * ethernet-input: current_data current_length flags error
+ * vnet_buffer_t: l2_hdr_offset l3_hdr_offset
+ * vnet_buffer2_t: no changes
+ *@cliexend
+?*/
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_metadata_command, static) =
+{
+  .path = "show buffer metadata",
+  .short_help = "show buffer metadata",
+  .function = show_metadata_command_fn,
+};
+/* *INDENT-OFF* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/mdata/mdata.h b/src/plugins/mdata/mdata.h
new file mode 100644
index 00000000000..6f38918ec25
--- /dev/null
+++ b/src/plugins/mdata/mdata.h
@@ -0,0 +1,66 @@
+
+/*
+ * mdata.h - Buffer metadata change tracker
+ *
+ * Copyright (c) 2019 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_mdata_h__
+#define __included_mdata_h__
+
+#include <vnet/vnet.h>
+
+#include <vppinfra/hash.h>
+#include <vppinfra/error.h>
+
+/** @file buffer metadata change tracker definitions
+ */
+
+typedef struct
+{
+  /** Node index, ~0 means no data from this run */
+  u32 node_index;
+  /** buffer metadata, cast to vlib_buffer_t as needed */
+  u8 mdata[128];
+} mdata_t;
+
+typedef struct
+{
+  /** API message ID base */
+  u16 msg_id_base;
+
+  /** Per-thread buffer metadata before calling node fcn */
+  mdata_t **before_per_thread;
+
+  /** Spinlock to protect modified metadata by node */
+  clib_spinlock_t modify_lock;
+
+  /** Modified metadata by node */
+  mdata_t *modifies;
+
+  /* convenience */
+  vlib_main_t *vlib_main;
+  vnet_main_t *vnet_main;
+} mdata_main_t;
+
+extern mdata_main_t mdata_main;
+
+#endif /* __included_mdata_h__ */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/mdata/mdata_doc.md b/src/plugins/mdata/mdata_doc.md
new file mode 100644
index 00000000000..cbbfb012183
--- /dev/null
+++ b/src/plugins/mdata/mdata_doc.md
@@ -0,0 +1,24 @@
+# Buffer metadata change tracker {#mdata_doc}
+
+## Introduction
+
+The mdata plugin uses the vlib main loop "before" performance counter
+hook to snapshoot buffer metadata before calling the node dispatch
+function. Similarly, the plugin uses the main loop "after" hook to
+compare a vectors' worth of buffer metadata after the fact.
+
+The comparison function is a simple octet-by-octet A != B check. We
+accumulate changed octets per-node across the entire run, using a
+single spinlock-protected accumulator.
+
+The "show buffer metadata" command produces a report of all fields
+whose values are changed by nodes visited during a given run.
+
+Since many fields in the vnet_buffer_opaque_t are union members,
+it may appear that a certain node changes numerous fields. The entire
+point of the exercise is to warn developers that if a packet visits
+node N, data placed into opaque union field F *will* be affected.
+
+One should never assume much about buffer metadata immutability across
+arbitrary subgraphs. This tool generates accurate reports, to the
+extent that one exercises the required subgraph trajectories.
diff --git a/src/plugins/mdata/mdata_test.c b/src/plugins/mdata/mdata_test.c
new file mode 100644
index 00000000000..57ec6846655
--- /dev/null
+++ b/src/plugins/mdata/mdata_test.c
@@ -0,0 +1,80 @@
+/*
+ * mdata.c - buffer metadata change tracker vpp-api-test plug-in
+ *
+ * Copyright (c) 2019 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <vat/vat.h>
+#include <vlibapi/api.h>
+#include <vlibmemory/api.h>
+#include <vppinfra/error.h>
+#include <stdbool.h>
+
+#define __plugin_msg_base mdata_test_main.msg_id_base
+#include <vlibapi/vat_helper_macros.h>
+
+/* Declare message IDs */
+#include <mdata/mdata.api_enum.h>
+#include <mdata/mdata.api_types.h>
+
+typedef struct
+{
+  /* API message ID base */
+  u16 msg_id_base;
+  vat_main_t *vat_main;
+} mdata_test_main_t;
+
+mdata_test_main_t mdata_test_main;
+
+static int
+api_mdata_enable_disable (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  int enable_disable = 1;
+  vl_api_mdata_enable_disable_t *mp;
+  int ret;
+
+  /* Parse args required to build the message */
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "disable"))
+	enable_disable = 0;
+      else
+	break;
+    }
+
+  /* Construct the API message */
+  M (MDATA_ENABLE_DISABLE, mp);
+  mp->enable_disable = enable_disable;
+
+  /* send it... */
+  S (mp);
+
+  /* Wait for a reply... */
+  W (ret);
+  return ret;
+}
+
+/*
+ * List of messages that the mdata test plugin sends,
+ * and that the data plane plugin processes
+ */
+#include <mdata/mdata.api_test.c>
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vlib/main.c b/src/vlib/main.c
index 90ce53de5a0..4223474367e 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -1004,14 +1004,17 @@ format_buffer_metadata (u8 * s, va_list * args)
   s = format (s, "flags: %U\n", format_vnet_buffer_flags, b);
   s = format (s, "current_data: %d, current_length: %d\n",
 	      (i32) (b->current_data), (i32) (b->current_length));
-  s = format (s, "current_config_index: %d, flow_id: %x, next_buffer: %x\n",
-	      b->current_config_index, b->flow_id, b->next_buffer);
-  s = format (s, "error: %d, ref_count: %d, buffer_pool_index: %d\n",
-	      (u32) (b->error), (u32) (b->ref_count),
-	      (u32) (b->buffer_pool_index));
-  s = format (s,
-	      "trace_handle: 0x%x, len_not_first_buf: %d\n",
-	      b->trace_handle, b->total_length_not_including_first_buffer);
+  s = format
+    (s,
+     "current_config_index/punt_reason: %d, flow_id: %x, next_buffer: %x\n",
+     b->current_config_index, b->flow_id, b->next_buffer);
+  s =
+    format (s, "error: %d, ref_count: %d, buffer_pool_index: %d\n",
+	    (u32) (b->error), (u32) (b->ref_count),
+	    (u32) (b->buffer_pool_index));
+  s =
+    format (s, "trace_handle: 0x%x, len_not_first_buf: %d\n", b->trace_handle,
+	    b->total_length_not_including_first_buffer);
   return s;
 }
 
diff --git a/src/vnet/interface_format.c b/src/vnet/interface_format.c
index 347dd4940d4..69f481c303d 100644
--- a/src/vnet/interface_format.c
+++ b/src/vnet/interface_format.c
@@ -444,6 +444,17 @@ format_vnet_buffer_opaque (u8 * s, va_list * args)
 	      "ip.reass.next_index: %d, ip.reass.estimated_mtu: %d",
 	      o->ip.reass.next_index, (u32) (o->ip.reass.estimated_mtu));
   vec_add1 (s, '\n');
+  s = format (s,
+	      "ip.reass.error_next_index: %d, ip.reass.owner_thread_index: %d",
+	      o->ip.reass.error_next_index,
+	      (u32) (o->ip.reass.owner_thread_index));
+  vec_add1 (s, '\n');
+  s = format (s,
+	      "ip.reass.ip_proto: %d, ip.reass.l4_src_port: %d",
+	      o->ip.reass.ip_proto, (u32) (o->ip.reass.l4_src_port));
+  vec_add1 (s, '\n');
+  s = format (s, "ip.reass.l4_dst_port: %d", o->ip.reass.l4_dst_port);
+  vec_add1 (s, '\n');
 
   s = format (s,
 	      "ip.reass.fragment_first: %d ip.reass.fragment_last: %d",
@@ -469,13 +480,15 @@ format_vnet_buffer_opaque (u8 * s, va_list * args)
 	      (u32) (o->mpls.ttl), (u32) (o->mpls.exp), (u32) (o->mpls.first),
 	      o->mpls.save_rewrite_length, (u32) (o->mpls.bier.n_bytes));
   vec_add1 (s, '\n');
+  s = format (s, "mpls.mpls_hdr_length: %d", (u32) (o->mpls.mpls_hdr_length));
+  vec_add1 (s, '\n');
 
   s = format (s,
-	      "l2.feature_bitmap: %08x, l2.bd_index: %d, l2.l2_len: %d, "
-	      "l2.shg: %d, l2.l2fib_sn: %d, l2.bd_age: %d",
-	      o->l2.feature_bitmap, (u32) (o->l2.bd_index),
-	      (u32) (o->l2.l2_len), (u32) (o->l2.shg), (u32) (o->l2.l2fib_sn),
-	      (u32) (o->l2.bd_age));
+	      "l2.feature_bitmap: %08x, l2.bd_index: %d, l2.l2fib_sn %d, "
+	      "l2.l2_len: %d, l2.shg: %d, l2.bd_age: %d",
+	      (u32) (o->l2.feature_bitmap), (u32) (o->l2.bd_index),
+	      (u32) (o->l2.l2fib_sn),
+	      (u32) (o->l2.l2_len), (u32) (o->l2.shg), (u32) (o->l2.bd_age));
   vec_add1 (s, '\n');
 
   s = format (s,
@@ -499,18 +512,24 @@ format_vnet_buffer_opaque (u8 * s, va_list * args)
   s = format (s, "policer.index: %d", o->policer.index);
   vec_add1 (s, '\n');
 
-  s = format (s, "ipsec.sad_index: %d", o->ipsec.sad_index);
+  s = format (s, "ipsec.sad_index: %d, ipsec.protect_index",
+	      o->ipsec.sad_index, o->ipsec.protect_index);
   vec_add1 (s, '\n');
 
   s = format (s, "map.mtu: %d", (u32) (o->map.mtu));
   vec_add1 (s, '\n');
 
   s = format (s,
-	      "map_t.v6.saddr: 0x%x, map_t.v6.daddr: 0x%x, "
-	      "map_t.v6.frag_offset: %d, map_t.v6.l4_offset: %d",
+	      "map_t.map_domain_index: %d, map_t.v6.saddr: 0x%x, "
+	      "map_t.v6.daddr: 0x%x, map_t.v6.frag_offset: %d, "
+	      "map_t.v6.l4_offset: %d, map_t.v6.l4_protocol: %d, "
+	      "map.t.checksum_offset: %d",
+	      o->map_t.map_domain_index,
 	      o->map_t.v6.saddr,
 	      o->map_t.v6.daddr,
-	      (u32) (o->map_t.v6.frag_offset), (u32) (o->map_t.v6.l4_offset));
+	      (u32) (o->map_t.v6.frag_offset), (u32) (o->map_t.v6.l4_offset),
+	      (u32) (o->map_t.v6.l4_protocol),
+	      (u32) (o->map_t.checksum_offset));
   vec_add1 (s, '\n');
 
   s = format (s,
@@ -533,12 +552,11 @@ format_vnet_buffer_opaque (u8 * s, va_list * args)
   vec_add1 (s, '\n');
 
   s = format
-    (s, "tcp.connection_index: %d, tcp.seq_number: %d, tcp.seq_end: %d, "
-     "tcp.ack_number: %d, tcp.hdr_offset: %d, tcp.data_offset: %d",
-     o->tcp.connection_index,
-     o->tcp.seq_number,
-     o->tcp.seq_end,
-     o->tcp.ack_number,
+    (s,
+     "tcp.connection_index: %d, tcp.seq_number: %d, tcp.next_node_opaque: %d "
+     "tcp.seq_end: %d, tcp.ack_number: %d, tcp.hdr_offset: %d, "
+     "tcp.data_offset: %d", o->tcp.connection_index, o->tcp.next_node_opaque,
+     o->tcp.seq_number, o->tcp.seq_end, o->tcp.ack_number,
      (u32) (o->tcp.hdr_offset), (u32) (o->tcp.data_offset));
   vec_add1 (s, '\n');
 
@@ -585,6 +603,10 @@ format_vnet_buffer_opaque2 (u8 * s, va_list * args)
 	      (u32) (o->gbp.flags), (u32) (o->gbp.sclass));
   vec_add1 (s, '\n');
 
+  s = format (s, "gso_size: %d, gso_l4_hdr_sz: %d",
+	      (u32) (o->gso_size), (u32) (o->gso_l4_hdr_sz));
+  vec_add1 (s, '\n');
+
   s = format (s, "pg_replay_timestamp: %llu", (u32) (o->pg_replay_timestamp));
   vec_add1 (s, '\n');